diff --git a/docs/npm_translate_lock.md b/docs/npm_translate_lock.md index 61d60a1ba..ce2338757 100644 --- a/docs/npm_translate_lock.md +++ b/docs/npm_translate_lock.md @@ -67,7 +67,7 @@ npm_translate_lock(name, lifecycle_hooks_use_default_shell_env, replace_packages, bins, verify_node_modules_ignored, verify_patches, quiet, external_repository_action_cache, link_workspace, pnpm_version, use_pnpm, - npm_package_target_name, use_starlark_yaml_parser, kwargs) + npm_package_target_name, kwargs) Repository macro to generate starlark code from a lock file. @@ -132,7 +132,6 @@ For more about how to use npm_translate_lock, read [pnpm and rules_js](/docs/pnp | pnpm_version | pnpm version to use when generating the @pnpm repository. Set to None to not create this repository.

Can be left unspecified and the rules_js default LATEST_PNPM_VERSION will be used.

Use use_pnpm for bzlmod. | "8.15.3" | | use_pnpm | label of the pnpm extension to use.

Can be left unspecified and the rules_js default pnpm extension (with the LATEST_PNPM_VERSION) will be used.

Use pnpm_version for non-bzlmod. | None | | npm_package_target_name | The name of linked npm_package targets. When npm_package targets are linked as pnpm workspace packages, the name of the target must align with this value.

The {dirname} placeholder is replaced with the directory name of the target.

By default the directory name of the target is used.

Default: {dirname} | "{dirname}" | -| use_starlark_yaml_parser | Opt-out of using yq to parse the pnpm-lock file which was added in https://github.com/aspect-build/rules_js/pull/1458 and use the legacy starlark yaml parser instead.

This opt-out is a return safety in cases where yq is not able to parse the pnpm generated yaml file. For example, this has been observed to happen due to a line such as the following in the pnpm generated lock file:

 resolution: {tarball: https://gitpkg.vercel.app/blockprotocol/blockprotocol/packages/%40blockprotocol/type-system-web?6526c0e}


where the ? character in the tarball value causes yq to fail with:

 $ yq pnpm-lock.yaml -o=json Error: bad file 'pnpm-lock.yaml': yaml: line 7129: did not find expected ',' or '}'


If the tarball value is quoted or escaped then yq would accept it but as of this writing, the latest version of pnpm (8.14.3) does not quote or escape such a value and the latest version of yq (4.40.5) does not handle it as is.

Possibly related to https://github.com/pnpm/pnpm/issues/5414. | False | | kwargs | Internal use only | none | diff --git a/e2e/npm_translate_lock/WORKSPACE b/e2e/npm_translate_lock/WORKSPACE index 7f014805b..1a8d239b8 100644 --- a/e2e/npm_translate_lock/WORKSPACE +++ b/e2e/npm_translate_lock/WORKSPACE @@ -19,7 +19,6 @@ npm_translate_lock( npmrc = "//:.npmrc", pnpm_lock = "//:pnpm-lock.yaml", update_pnpm_lock = True, - use_starlark_yaml_parser = True, # test coverage for this flag verify_node_modules_ignored = "//:.bazelignore", ) diff --git a/e2e/webpack_devserver/MODULE.bazel b/e2e/webpack_devserver/MODULE.bazel index aff87e261..7d5521409 100644 --- a/e2e/webpack_devserver/MODULE.bazel +++ b/e2e/webpack_devserver/MODULE.bazel @@ -12,7 +12,6 @@ npm.npm_translate_lock( name = "npm", npmrc = "//:.npmrc", pnpm_lock = "//:pnpm-lock.yaml", - use_starlark_yaml_parser = True, # test coverage for this flag verify_node_modules_ignored = "//:.bazelignore", ) use_repo(npm, "npm") diff --git a/npm/extensions.bzl b/npm/extensions.bzl index 17e569009..6c197b135 100644 --- a/npm/extensions.bzl +++ b/npm/extensions.bzl @@ -79,21 +79,19 @@ def _npm_lock_imports_bzlmod(module_ctx, attr): lock_packages = {} lock_patched_dependencies = {} lock_parse_err = None - if attr.use_starlark_yaml_parser: - lock_importers, lock_packages, lock_patched_dependencies, lock_parse_err = utils.parse_pnpm_lock_yaml(module_ctx.read(attr.pnpm_lock)) + + is_windows = repo_utils.is_windows(module_ctx) + host_yq = Label("@{}_{}//:yq{}".format(attr.yq_toolchain_prefix, repo_utils.platform(module_ctx), ".exe" if is_windows else "")) + yq_args = [ + str(module_ctx.path(host_yq)), + str(module_ctx.path(attr.pnpm_lock)), + "-o=json", + ] + result = module_ctx.execute(yq_args) + if result.return_code: + lock_parse_err = "failed to parse pnpm lock file with yq. '{}' exited with {}: \nSTDOUT:\n{}\nSTDERR:\n{}".format(" ".join(yq_args), result.return_code, result.stdout, result.stderr) else: - is_windows = repo_utils.is_windows(module_ctx) - host_yq = Label("@{}_{}//:yq{}".format(attr.yq_toolchain_prefix, repo_utils.platform(module_ctx), ".exe" if is_windows else "")) - yq_args = [ - str(module_ctx.path(host_yq)), - str(module_ctx.path(attr.pnpm_lock)), - "-o=json", - ] - result = module_ctx.execute(yq_args) - if result.return_code: - lock_parse_err = "failed to parse pnpm lock file with yq. '{}' exited with {}: \nSTDOUT:\n{}\nSTDERR:\n{}".format(" ".join(yq_args), result.return_code, result.stdout, result.stderr) - else: - lock_importers, lock_packages, lock_patched_dependencies, lock_parse_err = utils.parse_pnpm_lock_json(result.stdout if result.stdout != "null" else None) # NB: yq will return the string "null" if the yaml file is empty + lock_importers, lock_packages, lock_patched_dependencies, lock_parse_err = utils.parse_pnpm_lock_json(result.stdout if result.stdout != "null" else None) # NB: yq will return the string "null" if the yaml file is empty if lock_parse_err != None: msg = """ diff --git a/npm/private/BUILD.bazel b/npm/private/BUILD.bazel index 2a4d4ea61..117f037c5 100644 --- a/npm/private/BUILD.bazel +++ b/npm/private/BUILD.bazel @@ -211,7 +211,6 @@ bzl_library( srcs = ["utils.bzl"], visibility = ["//npm:__subpackages__"], deps = [ - ":yaml", "@aspect_bazel_lib//lib:paths", "@aspect_bazel_lib//lib:repo_utils", "@aspect_bazel_lib//lib:utils", @@ -304,9 +303,3 @@ bzl_library( srcs = ["versions.bzl"], visibility = ["//npm:__subpackages__"], ) - -bzl_library( - name = "yaml", - srcs = ["yaml.bzl"], - visibility = ["//npm:__subpackages__"], -) diff --git a/npm/private/npm_translate_lock.bzl b/npm/private/npm_translate_lock.bzl index 3fe5a694c..98c8810c5 100644 --- a/npm/private/npm_translate_lock.bzl +++ b/npm/private/npm_translate_lock.bzl @@ -76,7 +76,6 @@ _ATTRS = { "root_package": attr.string(default = DEFAULT_ROOT_PACKAGE), "update_pnpm_lock": attr.bool(), "use_home_npmrc": attr.bool(), - "use_starlark_yaml_parser": attr.bool(), "verify_node_modules_ignored": attr.label(), "verify_patches": attr.label(), "yarn_lock": attr.label(), @@ -186,7 +185,6 @@ def npm_translate_lock( pnpm_version = LATEST_PNPM_VERSION, use_pnpm = None, npm_package_target_name = "{dirname}", - use_starlark_yaml_parser = False, **kwargs): """Repository macro to generate starlark code from a lock file. @@ -514,31 +512,6 @@ def npm_translate_lock( Default: `{dirname}` - use_starlark_yaml_parser: Opt-out of using `yq` to parse the pnpm-lock file which was added - in https://github.com/aspect-build/rules_js/pull/1458 and use the legacy starlark yaml - parser instead. - - This opt-out is a return safety in cases where yq is not able to parse the pnpm generated - yaml file. For example, this has been observed to happen due to a line such as the following - in the pnpm generated lock file: - - ``` - resolution: {tarball: https://gitpkg.vercel.app/blockprotocol/blockprotocol/packages/%40blockprotocol/type-system-web?6526c0e} - ``` - - where the `?` character in the `tarball` value causes `yq` to fail with: - - ``` - $ yq pnpm-lock.yaml -o=json - Error: bad file 'pnpm-lock.yaml': yaml: line 7129: did not find expected ',' or '}' - ``` - - If the tarball value is quoted or escaped then yq would accept it but as of this writing, the latest - version of pnpm (8.14.3) does not quote or escape such a value and the latest version of yq (4.40.5) - does not handle it as is. - - Possibly related to https://github.com/pnpm/pnpm/issues/5414. - **kwargs: Internal use only """ @@ -632,7 +605,6 @@ def npm_translate_lock( use_pnpm = use_pnpm, yq_toolchain_prefix = yq_toolchain_prefix, npm_package_target_name = npm_package_target_name, - use_starlark_yaml_parser = use_starlark_yaml_parser, bzlmod = bzlmod, ) diff --git a/npm/private/npm_translate_lock_state.bzl b/npm/private/npm_translate_lock_state.bzl index 3f5cb8384..32e6ed969 100644 --- a/npm/private/npm_translate_lock_state.bzl +++ b/npm/private/npm_translate_lock_state.bzl @@ -504,19 +504,17 @@ def _load_lockfile(priv, rctx, label_store): packages = {} patched_dependencies = {} lock_parse_err = None - if rctx.attr.use_starlark_yaml_parser: - importers, packages, patched_dependencies, lock_parse_err = utils.parse_pnpm_lock_yaml(rctx.read(label_store.path("pnpm_lock"))) + + yq_args = [ + str(label_store.path("host_yq")), + str(label_store.path("pnpm_lock")), + "-o=json", + ] + result = rctx.execute(yq_args) + if result.return_code: + lock_parse_err = "failed to parse pnpm lock file with yq. '{}' exited with {}: \nSTDOUT:\n{}\nSTDERR:\n{}".format(" ".join(yq_args), result.return_code, result.stdout, result.stderr) else: - yq_args = [ - str(label_store.path("host_yq")), - str(label_store.path("pnpm_lock")), - "-o=json", - ] - result = rctx.execute(yq_args) - if result.return_code: - lock_parse_err = "failed to parse pnpm lock file with yq. '{}' exited with {}: \nSTDOUT:\n{}\nSTDERR:\n{}".format(" ".join(yq_args), result.return_code, result.stdout, result.stderr) - else: - importers, packages, patched_dependencies, lock_parse_err = utils.parse_pnpm_lock_json(result.stdout if result.stdout != "null" else None) # NB: yq will return the string "null" if the yaml file is empty + importers, packages, patched_dependencies, lock_parse_err = utils.parse_pnpm_lock_json(result.stdout if result.stdout != "null" else None) # NB: yq will return the string "null" if the yaml file is empty priv["importers"] = importers priv["packages"] = packages diff --git a/npm/private/test/BUILD.bazel b/npm/private/test/BUILD.bazel index c5fb073c1..3df12c61e 100644 --- a/npm/private/test/BUILD.bazel +++ b/npm/private/test/BUILD.bazel @@ -9,7 +9,6 @@ load(":pkg_glob_tests.bzl", "pkg_glob_tests") load(":transitive_closure_tests.bzl", "transitive_closure_tests") load(":translate_lock_helpers_tests.bzl", "translate_lock_helpers_tests") load(":utils_tests.bzl", "utils_tests") -load(":yaml_tests.bzl", "yaml_tests") # gazelle:exclude **/snapshots/**/* @@ -26,8 +25,6 @@ transitive_closure_tests(name = "test_transitive_closure") translate_lock_helpers_tests(name = "test_translate_lock") -yaml_tests(name = "test_yaml") - parse_pnpm_lock_tests(name = "test_parse_pnpm_lock") generated_pkg_json_test(name = "test_generated_pkg_json") diff --git a/npm/private/test/parse_pnpm_lock_tests.bzl b/npm/private/test/parse_pnpm_lock_tests.bzl index 4f1ee5822..1361c68b3 100644 --- a/npm/private/test/parse_pnpm_lock_tests.bzl +++ b/npm/private/test/parse_pnpm_lock_tests.bzl @@ -6,55 +6,18 @@ load("//npm/private:utils.bzl", "utils") def _parse_empty_lock_test_impl(ctx): env = unittest.begin(ctx) - parsed_yaml = utils.parse_pnpm_lock_yaml("") parsed_json_a = utils.parse_pnpm_lock_json("") parsed_json_b = utils.parse_pnpm_lock_json("{}") expected = ({}, {}, {}, None) - asserts.equals(env, expected, parsed_yaml) asserts.equals(env, expected, parsed_json_a) asserts.equals(env, expected, parsed_json_b) return unittest.end(env) -def _parse_merge_conflict_test_impl(ctx): - env = unittest.begin(ctx) - - parsed = utils.parse_pnpm_lock_yaml(""" -importers: - .: - dependencies: -<<<<< HEAD""") - expected = ({}, {}, {}, "expected lockfileVersion key in lockfile") - - asserts.equals(env, expected, parsed) - - return unittest.end(env) - def _parse_lockfile_v5_test_impl(ctx): env = unittest.begin(ctx) - parsed_yaml = utils.parse_pnpm_lock_yaml("""\ -lockfileVersion: 5.4 - -specifiers: - '@aspect-test/a': 5.0.0 - -dependencies: - '@aspect-test/a': 5.0.0 - -packages: - - /@aspect-test/a/5.0.0: - resolution: {integrity: sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==} - hasBin: true - dependencies: - '@aspect-test/b': 5.0.0 - '@aspect-test/c': 1.0.0 - '@aspect-test/d': 2.0.0_@aspect-test+c@1.0.0 - dev: false -""") - parsed_json = utils.parse_pnpm_lock_json("""\ { "lockfileVersion": 5.4, @@ -109,7 +72,6 @@ packages: None, ) - asserts.equals(env, expected, parsed_yaml) asserts.equals(env, expected, parsed_json) return unittest.end(env) @@ -117,26 +79,6 @@ packages: def _parse_lockfile_v6_test_impl(ctx): env = unittest.begin(ctx) - parsed_yaml = utils.parse_pnpm_lock_yaml("""\ -lockfileVersion: '6.0' - -dependencies: - '@aspect-test/a': - specifier: 5.0.0 - version: 5.0.0 - -packages: - - /@aspect-test/a@5.0.0: - resolution: {integrity: sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==} - hasBin: true - dependencies: - '@aspect-test/b': 5.0.0 - '@aspect-test/c': 1.0.0 - '@aspect-test/d': 2.0.0(@aspect-test/c@1.0.0) - dev: false -""") - parsed_json = utils.parse_pnpm_lock_json("""\ { "lockfileVersion": "6.0", @@ -192,7 +134,6 @@ packages: None, ) - asserts.equals(env, expected, parsed_yaml) asserts.equals(env, expected, parsed_json) return unittest.end(env) @@ -200,13 +141,11 @@ packages: a_test = unittest.make(_parse_empty_lock_test_impl, attrs = {}) b_test = unittest.make(_parse_lockfile_v5_test_impl, attrs = {}) c_test = unittest.make(_parse_lockfile_v6_test_impl, attrs = {}) -d_test = unittest.make(_parse_merge_conflict_test_impl, attrs = {}) TESTS = [ a_test, b_test, c_test, - d_test, ] def parse_pnpm_lock_tests(name): diff --git a/npm/private/test/yaml_tests.bzl b/npm/private/test/yaml_tests.bzl deleted file mode 100644 index c6d05a848..000000000 --- a/npm/private/test/yaml_tests.bzl +++ /dev/null @@ -1,579 +0,0 @@ -"Unit tests for yaml.bzl" - -load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load("//npm/private:yaml.bzl", "parse") - -def _parse_basic_test_impl(ctx): - env = unittest.begin(ctx) - - # Scalars - asserts.equals(env, (True, None), parse("true")) - asserts.equals(env, (False, None), parse("false")) - asserts.equals(env, (1, None), parse("1")) - asserts.equals(env, (3.14, None), parse("3.14")) - asserts.equals(env, ("foo", None), parse("foo")) - asserts.equals(env, ("foo", None), parse("'foo'")) - asserts.equals(env, ("foo", None), parse("\"foo\"")) - asserts.equals(env, ("foo", None), parse(" foo")) - asserts.equals(env, ("foo", None), parse(" foo ")) - asserts.equals(env, ("foo", None), parse("foo ")) - asserts.equals(env, ("foo", None), parse("\nfoo")) - asserts.equals(env, ("foo", None), parse("foo\n")) - asserts.equals(env, ("-foo", None), parse("-foo")) - asserts.equals(env, ("foo{[]}", None), parse("foo{[]}")) - - return unittest.end(env) - -def _parse_sequences_test_impl(ctx): - env = unittest.begin(ctx) - - # Sequences (- notation) - asserts.equals(env, (["foo"], None), parse("- foo")) - asserts.equals(env, (["foo - bar"], None), parse("- foo - bar")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -- foo -- bar -""")) - asserts.equals(env, (["foo"], None), parse("""\ -- - foo -""")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -- - foo -- - bar -""")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -- foo -- - bar -""")) - - # Sequences ([] notation) - asserts.equals(env, ([], None), parse("[]")) - asserts.equals(env, (["foo"], None), parse("[foo]")) - asserts.equals(env, (["fo o"], None), parse("[fo o]")) - asserts.equals(env, (["fo\no"], None), parse("[fo\no]")) - asserts.equals(env, (["foo", "bar"], None), parse("[foo,bar]")) - asserts.equals(env, (["foo", "bar"], None), parse("[foo, bar]")) - asserts.equals(env, ([1, True, "false"], None), parse("[1, true, \"false\"]")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -[ - foo, - bar -] -""")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -[ - foo, - bar, -] -""")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -[ - 'foo', - "bar", -] -""")) - asserts.equals(env, (["foo", "bar"], None), parse("""\ -[ - foo, - - bar -] -""")) - asserts.equals(env, ([["foo", "bar"]], None), parse("[[foo,bar]]")) - asserts.equals(env, ([["foo", [1, True]]], None), parse("[[foo,[1, true]]]")) - asserts.equals(env, ([["foo", "bar"]], None), parse("[[foo, bar]]")) - asserts.equals(env, ([["foo", "bar"]], None), parse("""\ -[ - [ - foo, - bar - ] -] -""")) - - return unittest.end(env) - -def _parse_maps_test_impl(ctx): - env = unittest.begin(ctx) - - # Maps - scalar properties - asserts.equals(env, ({"foo": "bar"}, None), parse("foo: bar")) - asserts.equals(env, ({"foo": "bar"}, None), parse("foo: 'bar'")) - asserts.equals(env, ({"foo": "bar"}, None), parse("foo: \"bar\"")) - asserts.equals(env, ({"foo": "bar"}, None), parse("foo: bar ")) - asserts.equals(env, ({"foo": "bar"}, None), parse("'foo': bar")) - asserts.equals(env, ({"foo": "bar"}, None), parse("\"foo\": bar")) - asserts.equals(env, ({"foo": 1.5}, None), parse("foo: 1.5")) - asserts.equals(env, ({"foo": True}, None), parse("foo: true")) - asserts.equals(env, ({"foo": False}, None), parse("foo: false")) - asserts.equals(env, ({"foo": "bar:baz"}, None), parse("foo: bar:baz")) - - # Maps - flow notation - asserts.equals(env, ({}, None), parse("{}")) - asserts.equals(env, ({"foo": "bar"}, None), parse("{foo: bar}")) - asserts.equals(env, ({"foo": "bar"}, None), parse("{foo: 'bar'}")) - asserts.equals(env, ({"foo": "bar"}, None), parse("{foo: \"bar\"}")) - asserts.equals(env, ({"foo": "bar"}, None), parse("{foo: bar }")) - asserts.equals(env, ({"foo": "bar"}, None), parse("{'foo': bar}")) - asserts.equals(env, ({"foo": "bar"}, None), parse("{\"foo\": bar}")) - asserts.equals(env, ({"foo": 1.5}, None), parse("{foo: 1.5}")) - asserts.equals(env, ({"foo": True}, None), parse("{foo: true}")) - asserts.equals(env, ({"foo": False}, None), parse("{foo: false}")) - asserts.equals(env, ({"foo": "bar:baz"}, None), parse("{foo: bar:baz}")) - asserts.equals(env, ({"foo": {"bar": 5}}, None), parse("{foo: {bar: 5}}")) - asserts.equals(env, ({"foo": 5, "bar": 6}, None), parse("{foo: 5, bar: 6}")) - asserts.equals(env, ({"foo": 5, "bar": {"moo": "cow"}, "faz": "baz"}, None), parse("{foo: 5, bar: {moo: cow}, faz: baz}")) - asserts.equals(env, ({"foo": {"bar": 5}}, None), parse("""\ -{ - foo: - { - bar: 5 - } -} -""")) - - return unittest.end(env) - -def _parse_multiline_test_impl(ctx): - env = unittest.begin(ctx) - - # Literal multiline strings (|), strip (-) and keep (+) - asserts.equals(env, ({"foo": "bar\n"}, None), parse("""\ -foo: | - bar -""")) - asserts.equals(env, ({"foo": "bar\n", "moo": "cow\n"}, None), parse("""\ -foo: | - bar -moo: | - cow -""")) - asserts.equals(env, ({"foo": {"bar": "baz \n faz\n"}}, None), parse("""\ -foo: - bar: | - baz - faz -""")) - asserts.equals(env, ({"a": "b\nc\nd\n"}, None), parse("""\ -a: | - b - c - d - - -""")) - asserts.equals(env, ({"a": "\n\nb\n\nc\n\nd\n"}, None), parse("""\ -a: | - - - b - - c - - d - - -""")) - asserts.equals(env, ({"foo": "bar"}, None), parse("""\ -foo: |- - bar -""")) - asserts.equals(env, ({"foo": "bar", "moo": "cow"}, None), parse("""\ -foo: |- - bar -moo: |- - cow -""")) - asserts.equals(env, ({"foo": "bar\n"}, None), parse("""\ -foo: |+ - bar -""")) - asserts.equals(env, ({"a": "\n\nb\n\nc\n\nd\n\n\n"}, None), parse("""\ -a: |+ - - - b - - c - - d - - -""")) - asserts.equals(env, ({"foo": "bar\n", "moo": "cow", "faz": "baz\n\n"}, None), parse("""\ -foo: | - bar -moo: |- - cow - -faz: |+ - baz - -""")) - - return unittest.end(env) - -def _parse_mixed_test_impl(ctx): - env = unittest.begin(ctx) - - # Mixed sequence and map flows - asserts.equals(env, ({"foo": ["moo"]}, None), parse("{foo: [moo]}")) - asserts.equals(env, (["foo", {"moo": "cow"}], None), parse("[foo, {moo: cow}]")) - asserts.equals(env, ({"foo": {"moo": "cow", "faz": ["baz", 123]}}, None), parse("{foo: {moo: cow, faz: [baz, 123]}}")) - asserts.equals(env, ([{"foo": ["bar", {"moo": ["cow"]}], "json": "bearded"}], None), parse("[{foo: [bar, {moo: [cow]}], json: bearded}]")) - - # Multi-level maps - asserts.equals(env, ({"foo": {"moo": "cow"}}, None), parse("""\ -foo: - moo: cow -""")) - asserts.equals(env, ({"foo": {"bar": {"moo": 5, "cow": True}}}, None), parse("""\ -foo: - bar: - moo: 5 - cow: true -""")) - asserts.equals(env, ({"a": {"b": {"c": {"d": 1, "e": 2, "f": 3}, "g": {"h": 4}}}}, None), parse("""\ -a: - b: - c: - d: 1 - e: 2 - f: 3 - g: - h: 4 -""")) - - # More than one root property - asserts.equals(env, ({"a": True, "b": False}, None), parse("""\ -a: true -b: false -""")) - asserts.equals(env, ({"a": {"b": True}, "c": {"d": False}}, None), parse("""\ -a: - b: true -c: - d: false -""")) - - # Value begins on next line at an indent - asserts.equals(env, ({"moo": "cow"}, None), parse("""\ -moo: - cow -""")) - - # Mixed flow and non-flow maps - asserts.equals(env, ({"foo": {"bar": {"moo": 5, "cow": True}}}, None), parse("""\ -foo: {bar: {moo: 5, cow: true}} -""")) - asserts.equals(env, ({"foo": {"bar": {"moo": 5, "cow": True}, "baz": "faz"}}, None), parse("""\ -foo: {bar: {moo: 5, cow: true}, baz: faz} -""")) - asserts.equals(env, ({"foo": {"bar": {"moo": 5, "cow": True}}}, None), parse("""\ -foo: - bar: {moo: 5, cow: true} -""")) - asserts.equals(env, ({"foo": {"bar": {"moo": 5, "cow": True}, "baz": "faz"}, "json": ["bearded"]}, None), parse("""\ -foo: - bar: - { - moo: 5, cow: true - } - baz: - faz -json: [bearded] -""")) - asserts.equals(env, ({"foo": {"bar": {"moo": [{"cow": True}]}}}, None), parse("""\ -foo: - bar: {moo: [ - {cow: true} - ]} -""")) - - # Miscellaneous - asserts.equals(env, ({"foo": {"bar": "b-ar", "moo": ["cow"]}, "loo": ["roo", "goo"]}, None), parse("""\ -foo: - bar: b-ar - moo: [cow] -loo: - - roo - - goo -""")) - - asserts.equals(env, ({"foo": "bar\n", "baz": 5}, None), parse("""\ -foo: | - bar -baz: 5 -""")) - - return unittest.end(env) - -def _parse_complex_map_test_impl(ctx): - env = unittest.begin(ctx) - - # Basic complex-object - asserts.equals(env, ({"a": True}, None), parse("""\ -? a -: true -""")) - - # Multiple bsic complex-object - asserts.equals(env, ({"a": True, "b": False}, None), parse("""\ -? a -: true -? b -: false -""")) - - # Whitespace in various places - asserts.equals(env, ({"a": True, "b": False, "c": "foo", " d ": " e "}, None), parse("""\ -? a -: true - -? b - -: false - -? 'c' -: foo - -? " d " -: " e " -""")) - - # Object in complex mapping key - asserts.equals(env, ({"a": {"b": {"c": 1, "d": True}}}, None), parse("""\ -a: - ? b - : c: 1 - d: true -""")) - - # Array in complex mapping key - asserts.equals(env, ({"a": {"b": [1, True]}}, None), parse("""\ -a: - ? b - : [1, true] -""")) - - # Arrays in complex mapping key - asserts.equals(env, ({"a": [1, 2]}, None), parse("""\ -? a -: - 1 - - 2 -""")) - - # Multiple nesting, arrays in objects - asserts.equals(env, ({"a": {"b": {"c": [1, 2]}}, "d": 3}, None), parse("""\ -? a -: ? b - : ? c - : - 1 - - 2 -? d -: 3 -""")) - - # Popping in/out of nested maps - asserts.equals(env, ({ - "a": { - "b": { - "c": 1, - "d": { - "e": "f", - }, - "g": 3, - }, - "e": 3, - }, - }, None), parse("""\ -a: - ? b - : c: 1 - d: - ? e - : f - g: 3 - ? e - : 3 -""")) - - return unittest.end(env) - -def _parse_lockfile_test_impl(ctx): - env = unittest.begin(ctx) - - # Partial lock file - asserts.equals(env, ({ - "lockfileVersion": 5.4, - "specifiers": { - "@aspect-test/a": "5.0.0", - }, - "dependencies": { - "@aspect-test/a": "5.0.0", - }, - "packages": { - "/@aspect-test/a/5.0.0": { - "resolution": { - "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", - }, - "hasBin": True, - "dependencies": { - "@aspect-test/b": "5.0.0", - "@aspect-test/c": "1.0.0", - "@aspect-test/d": "2.0.0_@aspect-test+c@1.0.0", - }, - "dev": False, - }, - }, - }, None), parse("""\ -lockfileVersion: 5.4 - -specifiers: - '@aspect-test/a': 5.0.0 - -dependencies: - '@aspect-test/a': 5.0.0 - -packages: - - /@aspect-test/a/5.0.0: - resolution: {integrity: sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==} - hasBin: true - dependencies: - '@aspect-test/b': 5.0.0 - '@aspect-test/c': 1.0.0 - '@aspect-test/d': 2.0.0_@aspect-test+c@1.0.0 - dev: false -""")) - - asserts.equals(env, ({ - "lockfileVersion": "6.0", - "packages": { - "/pkg-a@1.2.3": { - "resolution": { - "integrity": "sha512-asdf", - }, - "dev": False, - }, - "/pkg-c@3.2.1": { - "resolution": { - "integrity": "sha512-fdsa", - }, - "dev": True, - }, - }, - }, None), parse("""\ -lockfileVersion: '6.0' - -packages: - ? /pkg-a@1.2.3 - : resolution: {integrity: sha512-asdf} - dev: false - - ? /pkg-c@3.2.1 - : resolution: {integrity: sha512-fdsa} - dev: true -""")) - - return unittest.end(env) - -def _parse_conflict(ctx): - env = unittest.begin(ctx) - - # Similar test to https://github.com/pnpm/pnpm/blob/37fffbefa5a9136f2e189c01a5edf3c00ac48018/packages/supi/test/lockfile.ts#L1237C1-L1249C13 - asserts.equals(env, (None, "Unknown result state: <<<<< HEAD for value 100.0.0"), parse("""\ -importers: - .: - dependencies: -<<<<< HEAD - dep-of-pkg-with-1-dep: 100.0.0 -===== - dep-of-pkg-with-1-dep: 100.1.0 ->>>>> next - specifiers: - dep-of-pkg-with-1-dep: '>100.0.0' -lockfileVersion: 123 -packages: -<<<<<<< HEAD -""")) - - return unittest.end(env) - -def _parse_errors(ctx): - env = unittest.begin(ctx) - - asserts.equals(env, (None, "Unknown result state: sdlkfdslkjf for value +"), parse("""\ -asdljfk - sdlkfdslkjf - - - + - [- - -- sldkjf -""")) - - asserts.equals(env, (None, "Unexpected EOF"), parse("""\ -[asdljfk -""")) - - asserts.equals(env, (None, "Unexpected EOF"), parse("""\ -a: [ -""")) - - return unittest.end(env) - -parse_basic_test = unittest.make( - _parse_basic_test_impl, - attrs = {}, -) -parse_sequences_test = unittest.make( - _parse_sequences_test_impl, - attrs = {}, -) -parse_maps_test = unittest.make( - _parse_maps_test_impl, - attrs = {}, -) -parse_multiline_test = unittest.make( - _parse_multiline_test_impl, - attrs = {}, -) - -parse_mixed_test = unittest.make( - _parse_mixed_test_impl, - attrs = {}, -) -parse_complex_map_test = unittest.make( - _parse_complex_map_test_impl, - attrs = {}, -) -parse_lockfile_test = unittest.make( - _parse_lockfile_test_impl, - attrs = {}, -) -parse_conflict_test = unittest.make( - _parse_conflict, - attrs = {}, -) -parse_errors_test = unittest.make( - _parse_errors, - attrs = {}, -) - -def yaml_tests(name): - unittest.suite( - name, - parse_basic_test, - parse_sequences_test, - parse_maps_test, - parse_multiline_test, - parse_mixed_test, - parse_complex_map_test, - parse_lockfile_test, - parse_conflict_test, - parse_errors_test, - ) diff --git a/npm/private/utils.bzl b/npm/private/utils.bzl index 08bbb4456..5952b2d68 100644 --- a/npm/private/utils.bzl +++ b/npm/private/utils.bzl @@ -5,7 +5,6 @@ load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils") load("@aspect_bazel_lib//lib:utils.bzl", bazel_lib_utils = "utils") load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:types.bzl", "types") -load(":yaml.bzl", _parse_yaml = "parse") INTERNAL_ERROR_MSG = "ERROR: rules_js internal error, please file an issue: https://github.com/aspect-build/rules_js/issues" DEFAULT_REGISTRY_PROTOCOL = "https" @@ -135,18 +134,6 @@ def _convert_v6_packages(packages): result[_convert_pnpm_v6_package_name(package)] = package_info return result -def _parse_pnpm_lock_yaml(content): - """Parse the content of a pnpm-lock.yaml file. - - Args: - content: lockfile content - - Returns: - A tuple of (importers dict, packages dict, patched_dependencies dict, error string) - """ - parsed, err = _parse_yaml(content) - return _parse_pnpm_lock_common(parsed, err) - def _parse_pnpm_lock_json(content): """Parse the content of a pnpm-lock.yaml file. @@ -159,7 +146,7 @@ def _parse_pnpm_lock_json(content): return _parse_pnpm_lock_common(json.decode(content) if content else None, None) def _parse_pnpm_lock_common(parsed, err): - """Helper function used by _parse_pnpm_lock_yaml and _parse_pnpm_lock_json. + """Helper function used by _parse_pnpm_lock_json. Args: parsed: lockfile content object @@ -431,7 +418,6 @@ utils = struct( pnpm_name = _pnpm_name, assert_lockfile_version = _assert_lockfile_version, parse_pnpm_package_key = _parse_pnpm_package_key, - parse_pnpm_lock_yaml = _parse_pnpm_lock_yaml, parse_pnpm_lock_json = _parse_pnpm_lock_json, friendly_name = _friendly_name, package_store_name = _package_store_name, diff --git a/npm/private/yaml.bzl b/npm/private/yaml.bzl deleted file mode 100644 index 0639e9683..000000000 --- a/npm/private/yaml.bzl +++ /dev/null @@ -1,602 +0,0 @@ -"An experimental (and incomplete) yaml parser for starlark" -# https://github.com/bazelbuild/starlark/issues/219 - -_STATE = struct( - # Consume extraneous space and keep track of indent level - CONSUME_SPACE = 0, - # Consume whitespace within a flow, where indentation isn't a factor - CONSUME_SPACE_FLOW = 1, - # Parse the next key or value - PARSE_NEXT = 2, - # Parse a multiline string - PARSE_MULTILINE_STRING = 3, - # Parse the next key or value inside of a flow - PARSE_NEXT_FLOW = 4, - # Pseudo-states that don't perform any logic but indicate the current - # hierarchical result being formed in the starlark output. - KEY = 5, - SEQUENCE = 6, -) - -EOF = {} - -def parse(yaml): - """Parse yaml into starlark - - Args: - yaml: string, the yaml content to parse - - Returns: - A tuple containing an equivalent mapping to native starlark types and error. - """ - yaml = _normalize_yaml(yaml) - - starlark = {"result": None, "error": None} - stack = [] - stack.append(_new_CONSUME_SPACE(indent = "")) - - inputs = yaml.elems()[:] - inputs.append(EOF) - - for input in inputs: - state = _peek(stack) - - if state["id"] == _STATE.CONSUME_SPACE: - _handle_CONSUME_SPACE(state, input, stack, starlark) - elif state["id"] == _STATE.CONSUME_SPACE_FLOW: - _handle_CONSUME_SPACE_FLOW(state, input, stack, starlark) - elif state["id"] == _STATE.PARSE_NEXT: - _handle_PARSE_NEXT(state, input, stack, starlark) - elif state["id"] == _STATE.PARSE_MULTILINE_STRING: - _handle_PARSE_MULTILINE_STRING(state, input, stack, starlark) - elif state["id"] == _STATE.PARSE_NEXT_FLOW: - _handle_PARSE_NEXT_FLOW(state, input, stack, starlark) - else: - msg = "Unknown state {}".format(state["id"]) - fail(msg) - - if starlark["error"] != None: - return None, starlark["error"] - - return starlark["result"], None - -def _handle_CONSUME_SPACE(state, input, stack, starlark): - if input == EOF: - return - - if input == "\n": - # Reset the indentation - state["indent"] = "" - elif input.isspace(): - # Count the leading indentation - state["indent"] += input - elif input == "{": - stack.pop() - - # We are at the beginning of a new flow map - stack.append(_new_KEY( - key = "", - flow = True, - )) - - _initialize_result_value(starlark, stack) - - # Consume any space following the { - stack.append(_new_CONSUME_SPACE_FLOW()) - - elif input == "?": - # Complex mapping keys are indicated by a "?" at the beginning of the line. - # NOTE: currently only supports string value keys. - stack.pop() - - # We just started parsing a key - _pop_higher_indented_states(stack, state["indent"]) - - if len(stack) < 1 or len(state["indent"]) > len(_peek(stack, _STATE.KEY)["indent"]): - # The key is part of a new map - stack.append(_new_KEY( - key = "", - indent = state["indent"], - flow = False, - complex = True, - )) - else: - # The key is a sibling in the map - _peek(stack, _STATE.KEY)["key"] = "" - _peek(stack, _STATE.KEY)["complex"] = True - - _initialize_result_value(starlark, stack) - - # Consume any space following the ?. Add an extra space in place of the "?" - stack.append(_new_CONSUME_SPACE(indent = state["indent"] + " ")) - - elif input == ":" and len(stack) > 1 and stack[-2]["id"] == _STATE.KEY and stack[-2]["complex"]: - stack.pop() - - # Add an extra space in place of the ":" - stack.append(_new_CONSUME_SPACE(indent = state["indent"] + " ")) - - elif input == "[": - stack.pop() - - # We are at the beginning of a new flow sequence - stack.append(_new_SEQUENCE( - index = 0, - flow = True, - )) - - _initialize_result_value(starlark, stack) - - # Consume any space following the [ - stack.append(_new_CONSUME_SPACE_FLOW()) - elif input == "|": - stack.pop() - - # We are at the beginning of a multiline string - stack.append(_new_PARSE_MULTILINE_STRING( - indent = state["indent"], - )) - else: - # Reached the beginning of a value or key - stack.pop() - stack.append(_new_PARSE_NEXT( - indent = state["indent"], - buffer = input, - )) - -def _handle_PARSE_NEXT(state, input, stack, starlark): - if input == EOF: - stack.pop() - - # We just parsed a scalar - _set_result_value(starlark, stack, _parse_scalar(state["buffer"])) - - # Consume any space following the scalar - stack.append(_new_CONSUME_SPACE( - indent = "", - )) - elif input.isspace(): - if state["buffer"].endswith(":"): - stack.pop() - - # We just parsed a key - _pop_higher_indented_states(stack, state["indent"]) - - if len(stack) < 1 or len(state["indent"]) > len(_peek(stack, _STATE.KEY)["indent"]): - # The key is part of a new map - stack.append(_new_KEY( - key = _parse_key(state["buffer"][0:-1]), - indent = state["indent"], - flow = False, - complex = False, - )) - else: - # The key is a sibling in the map - _peek(stack, _STATE.KEY)["key"] = _parse_key(state["buffer"][0:-1]) - _peek(stack, _STATE.KEY)["complex"] = False - - _initialize_result_value(starlark, stack) - - # Consume any space following the key - stack.append(_new_CONSUME_SPACE( - indent = state["indent"] if not input == "\n" else "", - )) - elif state["buffer"] == "-": - stack.pop() - - if len(stack) > 0 and stack[-1]["id"] == _STATE.SEQUENCE: - # We are at the next item in a non-flow sequence - stack[-1]["index"] += 1 - else: - # We are at the beginning of a non-flow sequence - stack.append(_new_SEQUENCE( - indent = state["indent"], - index = 0, - flow = False, - )) - - _initialize_result_value(starlark, stack) - - # Consume any space following the sequence marker - stack.append(_new_CONSUME_SPACE( - indent = state["indent"] if not input == "\n" else "", - )) - elif input == "\n": - stack.pop() - - top = _peek(stack) if len(stack) > 0 else None - if top and top["id"] == _STATE.KEY and top["complex"] and not top["key"]: - # We just parsed a ?: key - top["key"] = _parse_scalar(state["buffer"]) - else: - # We just parsed a scalar - _set_result_value(starlark, stack, _parse_scalar(state["buffer"])) - - # Consume any space following the scalar - stack.append(_new_CONSUME_SPACE( - indent = "", - )) - else: - # Accumulate the space as part of the next thing to parse - state["buffer"] += input - else: - # Accumulate the current text until we know what to do with it - state["buffer"] += input - -def _handle_PARSE_MULTILINE_STRING(state, input, stack, starlark): - if not state["consumed_first_newline"]: - # Consume the rest of the line after the multiline indicator - if input == "+": - state["keep"] = True - elif input == "-": - state["strip"] = True - elif input == "\n": - state["consumed_first_newline"] = True - elif input.isspace(): - pass - else: - msg = "Unexpected input '{}' after start of multiline string".format(input) - fail(msg) - - elif state["value_indent"] == None: - # Establish the indent of the value - if input == "\n" and state["buffer"] == "": - state["value"] += "\n" - elif input.isspace(): - state["buffer"] += input - else: - state["value_indent"] = state["buffer"] - if not state["value_indent"].startswith(state["indent"]): - fail("Value indent of multiline string does not match indent of property") - elif len(state["value_indent"]) <= len(state["indent"]): - fail("Value indent of multiline is not greater than the indent of the property") - state["buffer"] += input - - elif input == EOF: - # Parse all lines of the string value - state["value"] = _finalize_multiline_string(state["value"], state["keep"], state["strip"]) - _set_result_value(starlark, stack, state["value"]) - - elif input == "\n": - state["buffer"] += input - if state["buffer"] == "\n": - state["value"] += "\n" - else: - state["value"] += state["buffer"][len(state["value_indent"]):] - state["buffer"] = "" - - elif not input.isspace() and not state["buffer"].startswith(state["value_indent"]): - state["value"] = _finalize_multiline_string(state["value"], state["keep"], state["strip"]) - _set_result_value(starlark, stack, state["value"]) - - stack.pop() - - # Parse the next thing - stack.append(_new_PARSE_NEXT( - indent = state["buffer"], - buffer = input, - )) - - else: - state["buffer"] += input - -def _handle_CONSUME_SPACE_FLOW(_, input, stack, starlark): - if input == EOF: - starlark["error"] = "Unexpected EOF" - return - - if input.isspace(): - pass - elif input == "[": - # We started a new inner flow sequence - stack.pop() - stack.append(_new_SEQUENCE( - index = 0, - flow = True, - )) - - _initialize_result_value(starlark, stack) - - # Consume any space following the [ - stack.append(_new_CONSUME_SPACE_FLOW()) - elif input == "{": - # We started a new inner flow map - stack.pop() - stack.append(_new_KEY( - key = "", - flow = True, - )) - - _initialize_result_value(starlark, stack) - - # Consume any space following the { - stack.append(_new_CONSUME_SPACE_FLOW()) - elif input == "]" and _in_flow_sequence(stack): - # We are at the end of a flow sequence - stack.pop() - _pop(stack, _STATE.SEQUENCE) - - # Consume any space before the next thing to parse and escape the flow if needed - stack.append(_new_CONSUME_SPACE_FLOW() if _in_flow(stack) else _new_CONSUME_SPACE( - indent = _peek(stack)["indent"] if len(stack) > 0 else "", - )) - elif input == "}" and _in_flow_map(stack): - # We are at the end of a flow map - stack.pop() - _pop(stack, _STATE.KEY) - - # Consume any space before the next thing to parse and escape the flow if needed - stack.append(_new_CONSUME_SPACE_FLOW() if _in_flow(stack) else _new_CONSUME_SPACE( - indent = _peek(stack)["indent"] if len(stack) > 0 else "", - )) - elif input == "," and _in_flow_map(stack): - # If we come across a comma but we are in the consume space state, - # then it means we just parsed a non-scalar which is already in the - # result, so just move on (I think...) - stack.pop() - - # Consume any space before the next sequence value - stack.append(_new_CONSUME_SPACE_FLOW()) - else: - # Reached the beginning of a value or key - stack.pop() - stack.append(_new_PARSE_NEXT_FLOW( - buffer = input, - )) - -def _handle_PARSE_NEXT_FLOW(state, input, stack, starlark): - if input == EOF: - starlark["error"] = "Unexpected EOF" - return - - if input == "[": - fail("Unhandled case") - elif input == "," and _in_flow_sequence(stack): - # We parsed the next value in a flow sequence - _set_result_value(starlark, stack, _parse_scalar(state["buffer"])) - stack.pop() - - sequence_flow_state = _peek(stack, _STATE.SEQUENCE) - sequence_flow_state["index"] += 1 - - # Consume any space before the next sequence value - stack.append(_new_CONSUME_SPACE_FLOW()) - elif input == "," and _in_flow_map(stack): - # We parsed the a value corresponding to the current key - _set_result_value(starlark, stack, _parse_scalar(state["buffer"])) - stack.pop() - - # Reset the key - map_flow_state = _peek(stack, _STATE.KEY) - map_flow_state["key"] = "" - - # Consume any space before the next sequence value - stack.append(_new_CONSUME_SPACE_FLOW()) - elif input == "]" and _in_flow_sequence(stack): - # We are at the end of a flow sequence - _set_result_value(starlark, stack, _parse_scalar(state["buffer"])) - stack.pop() - _pop(stack, _STATE.SEQUENCE) - - # Consume any space before the next thing to parse and escape the flow if needed - stack.append(_new_CONSUME_SPACE_FLOW() if _in_flow(stack) else _new_CONSUME_SPACE( - indent = _peek(stack)["indent"] if len(stack) > 0 else "", - )) - elif input == "}" and _in_flow_map(stack): - # We are at the end of a flow map - _set_result_value(starlark, stack, _parse_scalar(state["buffer"])) - stack.pop() - _pop(stack, _STATE.KEY) - - # Consume any space before the next thing to parse and escape the flow if needed - stack.append(_new_CONSUME_SPACE_FLOW() if _in_flow(stack) else _new_CONSUME_SPACE( - indent = _peek(stack)["indent"] if len(stack) > 0 else "", - )) - elif input.isspace() and state["buffer"].endswith(":") and _in_flow_map(stack): - # We just parsed a key - stack.pop() - _peek(stack, _STATE.KEY)["key"] = _parse_key(state["buffer"][0:-1]) - - stack.append(_new_CONSUME_SPACE_FLOW()) - else: - state["buffer"] += input - -def _new_CONSUME_SPACE(indent): - return { - "id": _STATE.CONSUME_SPACE, - "indent": indent, - } - -def _new_CONSUME_SPACE_FLOW(): - return { - "id": _STATE.CONSUME_SPACE_FLOW, - } - -def _new_PARSE_NEXT(indent, buffer): - return { - "id": _STATE.PARSE_NEXT, - "indent": indent, - "buffer": buffer, - } - -def _new_PARSE_MULTILINE_STRING(indent): - return { - "id": _STATE.PARSE_MULTILINE_STRING, - "type": "literal", # In case we support "folded" (>) later - "indent": indent, - "value_indent": None, - "buffer": "", - "value": "", - "strip": False, - "keep": False, - "consumed_first_newline": False, - } - -def _new_PARSE_NEXT_FLOW(buffer): - return { - "id": _STATE.PARSE_NEXT_FLOW, - "buffer": buffer, - } - -def _new_KEY(key, flow, complex = False, indent = None): - return { - "id": _STATE.KEY, - "key": key, - "indent": indent if not flow else None, - "flow": flow, - "complex": complex, - } - -def _new_SEQUENCE(index, flow, indent = None): - return { - "id": _STATE.SEQUENCE, - "indent": indent if not flow else None, - "index": index, - "flow": flow, - } - -def _normalize_yaml(yaml): - yaml = yaml.replace("\r", "") - return yaml - -def _initialize_result_value(starlark, stack): - "Initialize empty starlark maps or list values for the current pseudostates in the stack" - kns_states = _get_key_and_sequence_states(stack) - - if len(kns_states) == 0: - return - else: - if starlark["result"] == None: - starlark["result"] = _empty_value_for_state(kns_states[0]) - curr_result = starlark["result"] - for (i, state) in enumerate(kns_states[0:-1]): - if type(curr_result) == "dict": - curr_result = curr_result.setdefault(state["key"], _empty_value_for_state(kns_states[i + 1])) - elif type(curr_result) == "list": - if not "index" in state: - fail("Invalid yaml state under {}".format(_stack_path(stack))) - - if state["index"] >= len(curr_result): - curr_result.append(_empty_value_for_state(kns_states[i + 1])) - curr_result = curr_result[state["index"]] - else: - starlark["error"] = "Unknown result state: " + curr_result - return - -def _stack_path(stack): - p = [] - for s in stack: - p.append(s["key"]) - return p - -def _set_result_value(starlark, stack, value): - "Add a new value to the starlark result corresponding to the last pseudostate in the stack" - kns_states = _get_key_and_sequence_states(stack) - if len(kns_states) == 0: - starlark["result"] = value - else: - curr_result = starlark["result"] - for state in kns_states[0:-1]: - if type(curr_result) == "dict": - curr_result = curr_result[state["key"]] - else: - curr_result = curr_result[state["index"]] - if type(curr_result) == "dict": - curr_result[kns_states[-1]["key"]] = value - elif type(curr_result) == "list": - curr_result.append(value) - else: - starlark["error"] = "Unknown result state: " + curr_result + " for value " + value - return - -def _empty_value_for_state(state): - if state["id"] == _STATE.KEY: - return {} - elif state["id"] == _STATE.SEQUENCE: - return [] - else: - msg = "State {} has no empty type".format(state["id"]) - fail(msg) - -def _peek(stack, expected = False): - top = stack[-1] - if expected and top and top["id"] != expected: - fail("Expected state {} but got {}".format(expected, top["id"])) - - return top - -def _parse_scalar(value): - value = value.strip() - if _is_int(value): - return int(value) - elif _is_float(value): - return float(value) - elif _is_bool(value): - return _to_bool(value) - elif value.startswith("'"): - return value.strip("'") - else: - return value.strip("\"") - -def _finalize_multiline_string(value, keep, strip): - if keep and strip: - fail("Error: a multiline string cannot both keep and strip trailing newlines. This is probably a bug.") - if not keep and not strip: - value = value.rstrip(" \t\n") + "\n" - elif strip: - value = value.rstrip(" \t\n") - - return value - -def _pop(stack, *types): - for t in types: - if _peek(stack)["id"] != t: - fail("Expected state {} but found {}".format(t, _peek(stack)["id"])) - stack.pop() - -def _is_float(value): - return value.replace(".", "", 1).isdigit() - -def _is_int(value): - return value.isdigit() - -def _is_bool(value): - return value == "true" or value == "false" - -def _to_bool(value): - if value == "true": - return True - elif value == "false": - return False - msg = "Cannot convert scalar {} to a starlark boolean".format(value) - fail(msg) - -def _parse_key(key): - if key.startswith("'"): - return key.strip("'") - elif key.startswith("\""): - return key.strip("\"") - return key - -def _get_key_and_sequence_states(stack): - return [state for state in stack if state["id"] in [_STATE.KEY, _STATE.SEQUENCE]] - -def _in_flow(stack): - kns_states = _get_key_and_sequence_states(stack) - return len(kns_states) > 0 and kns_states[-1]["id"] in [_STATE.SEQUENCE, _STATE.KEY] and kns_states[-1]["flow"] - -def _in_flow_sequence(stack): - kns_states = _get_key_and_sequence_states(stack) - return len(kns_states) > 0 and kns_states[-1]["id"] == _STATE.SEQUENCE and kns_states[-1]["flow"] - -def _in_flow_map(stack): - kns_states = _get_key_and_sequence_states(stack) - return len(kns_states) > 0 and kns_states[-1]["id"] == _STATE.KEY and kns_states[-1]["flow"] - -def _pop_higher_indented_states(stack, indent): - remove = [] - for state in stack: - if state["id"] in [_STATE.KEY, _STATE.SEQUENCE] and len(state["indent"]) > len(indent): - remove.append(state) - for state in remove: - stack.remove(state)