Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix(pnpm): url and git package specifiers
Browse files Browse the repository at this point in the history
jbedard committed May 23, 2024
1 parent ce5eefb commit 564526b
Showing 12 changed files with 321 additions and 109 deletions.
8 changes: 0 additions & 8 deletions e2e/pnpm_lockfiles/README.md
Original file line number Diff line number Diff line change
@@ -2,14 +2,6 @@

TODO:

- http references: `"hello": "https://gitpkg.vercel.app/EqualMa/gitpkg-hello/packages/hello"`

Has inconsistencies across pnpm lockfile versions, issues with pnpm9

- file references: `"@scoped/c": "file:../projects/c"`

Has inconsistencies across pnpm lockfile versions, issues with pnpm9

- npm: references: `@aspect-test/a2": "npm:@aspect-test/a"`

No :node_modules/\* targets are generated for aliases to npm packages.
2 changes: 2 additions & 0 deletions e2e/pnpm_lockfiles/base/package.json
Original file line number Diff line number Diff line change
@@ -3,6 +3,8 @@
"private": true,
"dependencies": {
"@aspect-test/a": "^5.0.2",
"debug": "ngokevin/debug#9742c5f383a6f8046241920156236ade8ec30d53",
"hello": "https://gitpkg.vercel.app/EqualMa/gitpkg-hello/packages/hello",
"rollup": "3.2.5",
"meaning-of-life": "1.0.0",
"uvu": "0.5.6",
13 changes: 12 additions & 1 deletion e2e/pnpm_lockfiles/lockfile-test.bzl
Original file line number Diff line number Diff line change
@@ -77,9 +77,20 @@ def lockfile_test(name = None):
# link:, workspace:, file:, ./rel/path
":node_modules/@scoped/a",
":node_modules/@scoped/b",
# ":node_modules/@scoped/c", TODO: see README
":node_modules/@scoped/c",
":node_modules/@scoped/d",

# http:
# ":node_modules/debug",
# TODO: differs across lockfile versions
":.aspect_rules_js/node_modules/{}".format("[email protected]+ngokevin+debug+tar.gz+9742c5f383a6f8046241920156236ade8ec30d53" if lock_version == "v90" else "[email protected]+ngokevin+debug+9742c5f383a6f8046241920156236ade8ec30d53"),

# repo#hash
# (no protocol, assumed to be github repo + hash):
# ":node_modules/hello",
# TODO: differs across lockfile versions
":.aspect_rules_js/node_modules/{}".format("[email protected]+EqualMa+gitpkg-hello+packages+hello" if lock_version == "v90" else "@gitpkg.vercel.app+EqualMa+gitpkg-hello+packages+hello"),

# npm:
# ":node_modules/@aspect-test/c2",

27 changes: 27 additions & 0 deletions e2e/pnpm_lockfiles/v54/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 30 additions & 18 deletions e2e/pnpm_lockfiles/v54/snapshots/defs.bzl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 29 additions & 0 deletions e2e/pnpm_lockfiles/v60/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 30 additions & 18 deletions e2e/pnpm_lockfiles/v60/snapshots/defs.bzl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 29 additions & 0 deletions e2e/pnpm_lockfiles/v61/pnpm-lock.yaml
48 changes: 30 additions & 18 deletions e2e/pnpm_lockfiles/v61/snapshots/defs.bzl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

30 changes: 30 additions & 0 deletions e2e/pnpm_lockfiles/v90/pnpm-lock.yaml
48 changes: 27 additions & 21 deletions e2e/pnpm_lockfiles/v90/snapshots/defs.bzl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

100 changes: 75 additions & 25 deletions npm/private/utils.bzl
Original file line number Diff line number Diff line change
@@ -242,7 +242,7 @@ def _convert_pnpm_v6_v9_version_peer_dep(version):
# "File name too long) build failures.
peer_dep = "_" + _hash(peer_dep)
version = version[0:peer_dep_index] + _sanitize_string(peer_dep)
version = version.rstrip("_")
version = version.strip("_")
return version

def _convert_pnpm_v6_importer_dependency_map(deps):
@@ -251,7 +251,7 @@ def _convert_pnpm_v6_importer_dependency_map(deps):
result[name] = _convert_pnpm_v6_v9_version_peer_dep(attributes.get("version"))
return result

def _convert_pnpm_v6_v9_package_dependency_map(deps):
def _convert_pnpm_v6_package_dependency_map(deps):
result = {}
for name, version in deps.items():
result[name] = _convert_pnpm_v6_v9_version_peer_dep(version)
@@ -335,9 +335,9 @@ def _convert_v6_packages(packages):
name = name,
version = version,
friendly_version = friendly_version,
dependencies = _convert_pnpm_v6_v9_package_dependency_map(package_snapshot.get("dependencies", {})),
optional_dependencies = _convert_pnpm_v6_v9_package_dependency_map(package_snapshot.get("optionalDependencies", {})),
peer_dependencies = _convert_pnpm_v6_v9_package_dependency_map(package_snapshot.get("peerDependencies", {})),
dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("dependencies", {})),
optional_dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("optionalDependencies", {})),
peer_dependencies = _convert_pnpm_v6_package_dependency_map(package_snapshot.get("peerDependencies", {})),
dev = package_snapshot.get("dev", False),
has_bin = package_snapshot.get("hasBin", False),
optional = package_snapshot.get("optional", False),
@@ -355,21 +355,73 @@ def _convert_v6_packages(packages):

return result

def _convert_pnpm_v9_dependency_version(version):
# Strip URL-style protocols from version to align more with pnpm <v9 and rules_js
# Note this does NOT strip file: or link: prefixes.
proto_end = version.find("://")
if proto_end != -1:
version = version[proto_end + 3:]

return version

def _convert_pnpm_v9_package_dependency_map(deps):
result = {}
for name, version in deps.items():
# Raw version or version(peers) or file reference
if version[0].isdigit() or version.startswith("file:") or version.startswith("link:"):
result[name] = _convert_pnpm_v6_v9_version_peer_dep(version)
else:
# Otherwise assume a reference to another package@version(...maybe peers...)
i = version.find("@", 1)
result[version[:i]] = _convert_pnpm_v6_v9_version_peer_dep(version[i + 1:])
result[name] = _convert_pnpm_v6_v9_version_peer_dep(_convert_pnpm_v9_dependency_version(version))

return result

def _convert_pnpm_v9_importer_dependency_map(deps):
result = {}
for name, attributes in deps.items():
result[name] = _convert_pnpm_v6_v9_version_peer_dep(_convert_pnpm_v9_dependency_version(attributes.get("version")))

return result

def _convert_v9_importers(importers):
# Convert pnpm lockfile v9 importers to a rules_js compatible ~v5 format.
#
# v9 structure is the same as v6, but the importer dependency version format
# has changed such that {"name": "version"} exactly aligns with the lockfile
# "packages" field {"name@version": {...package info...}}.
#
# (Unlike <=v6 where name/versions were a mess of registry/url/name/versions)
#
# The structure is the same as v6:
#
# deps:
# pkg-a:
# specifier: 1.2.3
# version: 1.2.3
# devDeps:
# pkg-b:
# specifier: ^4.5.6
# version: 4.10.1

result = {}
for import_path, importer in importers.items():
result[import_path] = _new_import_info(
# TODO: normalize edge cases such as:
# - deps with protocols
# - ?
dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("dependencies", {})),
dev_dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("devDependencies", {})),
optional_dependencies = _convert_pnpm_v9_importer_dependency_map(importer.get("optionalDependencies", {})),
)
return result

# v9 importers are the same as v6 importers
_convert_v9_importers = _convert_v6_importers
def _parse_v9_snapshot_key(key):
peers = None

# Split a snapshot key of the form name@version[(scoped-data)]
if key[-1] == ")":
peer_meta_index = key.index("(")
peers = _convert_pnpm_v6_v9_version_peer_dep(key[peer_meta_index:])
key = key[:peer_meta_index]

name, version = key.rsplit("@", 1)

return name, _convert_pnpm_v9_dependency_version(version), "{}@{}".format(name, version), peers

def _convert_v9_packages(packages, snapshots):
# Convert pnpm lockfile v9 importers to a rules_js compatible format.
@@ -402,8 +454,8 @@ def _convert_v9_packages(packages, snapshots):

# Snapshots contains the packages with the keys (which include peers) to return
for package_key, package_snapshot in snapshots.items():
peer_meta_index = package_key.find("(")
static_key = package_key[:peer_meta_index] if peer_meta_index > 0 else package_key
name, version, static_key, peers_key = _parse_v9_snapshot_key(package_key)

if not static_key in packages:
msg = "package {} not found in pnpm 'packages'".format(static_key)
fail(msg)
@@ -414,16 +466,14 @@ def _convert_v9_packages(packages, snapshots):
msg = "package {} has no resolution field".format(static_key)
fail(msg)

# the raw name + version are the key, not including peerDeps+patch
name, friendly_version = static_key.rsplit("@", 1)
package_key = _convert_pnpm_v6_v9_version_peer_dep(package_key)

# Extract the version including peerDeps+patch from the key
version = package_key.rsplit("@", 1)[1]

# package_data can have the resolved "version" for things like https:// deps
if "version" in package_data:
friendly_version = package_data["version"]
# otherwise use the calculated version from the snapshot key, *excluding the peers*
friendly_version = package_data["version"] if "version" in package_data else version

package_key = "{}@{}".format(name, _sanitize_string(version))
if peers_key:
package_key = "{}_{}".format(package_key, peers_key)
version = "{}_{}".format(version, peers_key)

package_info = _new_package_info(
id = package_data.get("id", None),

0 comments on commit 564526b

Please sign in to comment.