From f63b850f27bbd8479b0fddb2fc3a2e7b62bf795f Mon Sep 17 00:00:00 2001 From: Simon Friis Vindum Date: Mon, 19 Oct 2026 11:59:10 +0200 Subject: [PATCH 1/5] Build: Support architecture overrides for imported ZIP archives Allow imported ZIP prefixes in arch_overrides so native dependencies can be distributed separately from architecture-independent files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9f12f7cb-26aa-405b-a1f0-9dcc239778cb --- misc/bazel/pkg.bzl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 684bcbb8c3ba..1410e12330bb 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -84,7 +84,7 @@ CodeQLPackInfo = provider( "pack_prefix": "A prefix to add to all paths, IF the user requests so. We omit it for local installation targets of single packs (but not pack groups)", "files": "PackageFilegroupInfo provider with list of all files in this pack (CODEQL_PLATFORM in paths unresolved)", "zips": "A _ZipInfo provider to include in the pack, (CODEQL_PLATFORM unresolved).", - "arch_overrides": "A list of files that should be included in the arch-specific bit, even though the path doesn't contain CODEQL_PLATFORM.", + "arch_overrides": "Files or imported zip prefixes to include in the arch-specific bit, even without CODEQL_PLATFORM in their paths.", }, ) @@ -220,7 +220,7 @@ _codeql_pack_info = rule( allow_files = [".zip"], ), "prefix": attr.string(doc = "Prefix to add to all files."), - "arch_overrides": attr.string_list(doc = "A list of files that should be included in the arch package regardless of the path, specify the path _without_ `prefix`."), + "arch_overrides": attr.string_list(doc = "Files or imported zip prefixes to include in the arch package regardless of the path, specified _without_ `prefix`."), }, provides = [CodeQLPackInfo], ) @@ -293,6 +293,8 @@ def _codeql_pack_group_extract_zips_impl(ctx): # for each zip file, resolve whether it's filtered out or not by the current kind, and add the pack prefix for zip, zip_prefix in src.zips.zips_to_prefixes.items(): zip_kind, expanded_prefix = _expand_path(paths.join(prefix, zip_prefix), platform) + if zip_kind == "common" and zip_prefix in src.arch_overrides: + zip_kind = "arch" if include_all_files or zip_kind == ctx.attr.kind: zips_to_prefixes[zip] = expanded_prefix @@ -465,13 +467,13 @@ def codeql_pack( The distinction between arch-specific and common contents is made based on whether the paths (including possible prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also be replaced by the appropriate platform (`linux64`, `linux-arm64`, `win64` or `osx64`). - Specific file paths can be placed in the arch-specific package by adding them to `arch_overrides`, even if their - path doesn't contain the `CODEQL_PLATFORM` placeholder. + Specific file paths or imported zip prefixes can be placed in the arch-specific package by adding them to + `arch_overrides`, even if their path doesn't contain the `CODEQL_PLATFORM` placeholder. The codeql pack rules will expand the `{CODEQL_PLATFORM}` marker in paths, and use that to split the files into a common and an arch-specific part. This placeholder will be replaced by the appropriate platform (`linux64`, `linux-arm64`, `win64` or `osx64`). - `arch_overrides` is a list of files that should be included in the arch-specific bits of the pack, even if their path doesn't - contain the `{CODEQL_PLATFORM}` marker. + `arch_overrides` is a list of files or imported zip prefixes that should be included in the arch-specific bits of the + pack, even if their path doesn't contain the `{CODEQL_PLATFORM}` marker. All files in the pack will be prefixed with `name`, unless `pack_prefix` is set, then is used instead. This rule also provides a convenient installer target named `-installer`, with a path governed by `install_dest`, From dba7357fe1bc0c654cee1dd681bdff293412f695 Mon Sep 17 00:00:00 2001 From: Simon Friis Vindum Date: Mon, 19 Oct 2026 11:59:10 +0200 Subject: [PATCH 2/5] JS: Package native TypeScript runtimes by distribution Split the wrapper and native runtimes into common and architecture-specific archives. Select and verify pinned platform packages, preserve executable permissions, and use ZIP response files on Windows. Add installed-package smoke tests covering platform selection, archive layout, runtime compatibility, and the merged distribution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9f12f7cb-26aa-405b-a1f0-9dcc239778cb --- javascript/BUILD.bazel | 6 +- .../extractor/lib/typescript/BUILD.bazel | 104 +++++++- .../lib/typescript/package-runtime.mjs | 75 ++++++ .../lib/typescript/test/package_test.py | 236 ++++++++++++++++++ 4 files changed, 411 insertions(+), 10 deletions(-) create mode 100644 javascript/extractor/lib/typescript/package-runtime.mjs create mode 100644 javascript/extractor/lib/typescript/test/package_test.py diff --git a/javascript/BUILD.bazel b/javascript/BUILD.bazel index 85b2024c283c..8f4a001265ae 100644 --- a/javascript/BUILD.bazel +++ b/javascript/BUILD.bazel @@ -33,5 +33,9 @@ codeql_pack( "//javascript/extractor:tools-extractor", "//javascript/resources", ], - zips = {"//javascript/extractor/lib/typescript": "tools"}, + arch_overrides = ["tools/typescript-parser-wrapper/node_modules/@typescript"], + zips = { + "//javascript/extractor/lib/typescript": "tools", + "//javascript/extractor/lib/typescript:native": "tools/typescript-parser-wrapper/node_modules/@typescript", + }, ) diff --git a/javascript/extractor/lib/typescript/BUILD.bazel b/javascript/extractor/lib/typescript/BUILD.bazel index 28003162673e..9a85c1a04474 100644 --- a/javascript/extractor/lib/typescript/BUILD.bazel +++ b/javascript/extractor/lib/typescript/BUILD.bazel @@ -1,13 +1,28 @@ -# Builds a zip file of the compiled typscript-parser-wrapper and its dependencies. +load("@rules_python//python:defs.bzl", "py_test") +load("//misc/bazel:os.bzl", "codeql_platform_select") +load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pack_group") + +DISTRIBUTION = codeql_platform_select( + linux64 = "linux64", + linux_arm64 = "linux-arm64", + osx64 = "osx64", + win64 = "win64", +) + +# Keep the JavaScript API common, but ship native executables in the arch package. genrule( - name = "typescript", + name = "build", srcs = [ "tsconfig.json", "package.json", "package-lock.json", + "package-runtime.mjs", ] + glob(["src/**"]), - outs = ["javascript.zip"], - cmd = "\n".join([ + outs = [ + "javascript.zip", + "native.zip", + ], + cmd = "DISTRIBUTION=" + DISTRIBUTION + "\n" + "\n".join([ # the original working directory is not preserved anywhere, but needs to be accessible, as # all paths are relative to this # unfortunately, we need to change the working directory to run npm. @@ -25,20 +40,35 @@ genrule( "rm -rf $(RULEDIR)/inputs", "cp -L -R $$(dirname $(execpath package.json)) $(RULEDIR)/inputs", "cd $(RULEDIR)/inputs", - "$$NPM install", + "$$NPM ci", "$$NPM run build", "rm -rf node_modules", # Install again with only runtime deps - "$$NPM install --prod", + "$$NPM ci --omit=dev --omit=optional --ignore-scripts", + # Optional dependencies are host-filtered even with --force. Install the + # selected packages explicitly, retaining the lockfile's + # versions/integrities. + "$$NPM install --omit=dev --ignore-scripts --no-save --force $$(node package-runtime.mjs packages $$DISTRIBUTION)", + # Copy native binaries into `native/` and remove the explicitly + # installed native packages from `node_modules`. + "node package-runtime.mjs stage $$DISTRIBUTION", "mv node_modules build/", + # Node needs the ESM package boundary in the installed wrapper too. + "cp package.json build/", "mkdir -p typescript-parser-wrapper", "mv build/* typescript-parser-wrapper", - "OUT=$$BAZEL_ROOT/$@", + "OUT=$$BAZEL_ROOT/$(execpath javascript.zip)", + "NATIVE_OUT=$$BAZEL_ROOT/$(execpath native.zip)", "case $$OSTYPE in", - " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;", + " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT); NATIVE_OUT=$$(cygpath -w $$NATIVE_OUT);;", "esac", "", - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)", + # Use response files to stay below Windows' command-line length limit. + "find typescript-parser-wrapper -print > wrapper-files", + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT @wrapper-files", + "cd native", + "find typescript-* -print > native-files", + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$NATIVE_OUT @native-files", "rm -rf $$TEMP", ]), tools = [ @@ -48,3 +78,59 @@ genrule( ], visibility = ["//visibility:public"], ) + +filegroup( + name = "typescript", + srcs = ["javascript.zip"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "native", + srcs = ["native.zip"], + visibility = ["//visibility:public"], +) + +codeql_pack( + name = "package-test-pack", + srcs = [], + arch_overrides = ["tools/typescript-parser-wrapper/node_modules/@typescript"], + install_dest = None, + pack_prefix = "", + zips = { + ":native": "tools/typescript-parser-wrapper/node_modules/@typescript", + ":typescript": "tools", + }, +) + +codeql_pack_group( + name = "package-test-distribution", + srcs = [":package-test-pack"], + skip_installer = True, +) + +py_test( + name = "package-test", + size = "small", + srcs = ["test/package_test.py"], + data = [ + "package-runtime.mjs", + ":native", + ":package-test-distribution-arch-zip", + ":package-test-distribution-common-zip", + ":typescript", + "//misc/ripunzip", + "@nodejs//:node_bin", + ], + env = { + "NODE_BIN": "$(rlocationpath @nodejs//:node_bin)", + "PACKAGE_RUNTIME": "$(rlocationpath package-runtime.mjs)", + "RIPUNZIP": "$(rlocationpath //misc/ripunzip)", + "TS_ARCH_ZIP": "$(rlocationpath :package-test-distribution-arch-zip)", + "TS_COMMON_ZIP": "$(rlocationpath :package-test-distribution-common-zip)", + "TS_NATIVE_ZIP": "$(rlocationpath :native)", + "TS_WRAPPER_ZIP": "$(rlocationpath :typescript)", + }, + main = "test/package_test.py", + deps = ["@rules_python//python/runfiles"], +) diff --git a/javascript/extractor/lib/typescript/package-runtime.mjs b/javascript/extractor/lib/typescript/package-runtime.mjs new file mode 100644 index 000000000000..69c6fd0b770d --- /dev/null +++ b/javascript/extractor/lib/typescript/package-runtime.mjs @@ -0,0 +1,75 @@ +// This script is used in `BUILD.bazel` to manage the TypeScript native +// executables. +// +// Run it with an `action` argument and a `distribution` argument. On +// - packages it prints the pinned native packages to install. +// - stage it moves the relevant native executables into +// `native/`, leaving `node_modules` with only the common JavaScript +// dependencies. + +import { + chmodSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + statSync, +} from "node:fs"; +import { join } from "node:path"; + +const platformsByDistribution = { + linux64: ["linux-x64"], + "linux-arm64": ["linux-arm64"], + osx64: ["darwin-arm64", "darwin-x64"], + win64: ["win32-x64"], +}; +const [action, distribution] = process.argv.slice(2); +if (process.argv.length !== 4 || !["packages", "stage"].includes(action)) { + throw new Error( + "Usage: node package-runtime.mjs ", + ); +} +const platforms = platformsByDistribution[distribution]; +if (platforms == null) { + throw new Error("Unsupported CodeQL distribution: " + distribution); +} +const readJson = (filename) => JSON.parse(readFileSync(filename, "utf8")); +const typescript = readJson("node_modules/typescript/package.json"); + +if (action === "packages") { + // Print the native packages for the specified distribution + console.log( + platforms + .map( + (platform) => `@typescript/typescript-${platform}@${typescript.version}`, + ) + .join(" "), + ); + process.exit(0); +} else { + // Move the native Go executables for the specified distribution into the + // `native/` directory + mkdirSync("native"); + for (const platform of platforms) { + const name = "typescript-" + platform; + const source = join("node_modules", "@typescript", name); + const manifest = readJson(join(source, "package.json")); + if ( + manifest.name !== "@typescript/" + name || + manifest.version !== typescript.version || + typescript.optionalDependencies[manifest.name] !== manifest.version + ) { + throw new Error( + "Native runtime does not match TypeScript: " + manifest.name, + ); + } + const executableName = platform.startsWith("win32-") ? "tsc.exe" : "tsc"; + const executable = join(source, "lib", executableName); + if (!statSync(executable).isFile()) { + throw new Error("Native executable is not a file: " + executable); + } + chmodSync(executable, 0o755); + renameSync(source, join("native", name)); + } + rmSync("node_modules/@typescript", { recursive: true }); +} diff --git a/javascript/extractor/lib/typescript/test/package_test.py b/javascript/extractor/lib/typescript/test/package_test.py new file mode 100644 index 000000000000..8ab3e1330eb5 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/package_test.py @@ -0,0 +1,236 @@ +import json +import os +from pathlib import Path +import stat +import subprocess +import tempfile +import unittest +import zipfile + +from python.runfiles import runfiles + + +PLATFORMS = { + "linux64": ["linux-x64"], + "linux-arm64": ["linux-arm64"], + "osx64": ["darwin-arm64", "darwin-x64"], + "win64": ["win32-x64"], +} + + +class PackageTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + files = runfiles.Create() + cls.node = files.Rlocation(os.environ["NODE_BIN"]) + cls.ripunzip = files.Rlocation(os.environ["RIPUNZIP"]) + cls.script = files.Rlocation(os.environ["PACKAGE_RUNTIME"]) + cls.wrapper_zip = files.Rlocation(os.environ["TS_WRAPPER_ZIP"]) + cls.native_zip = files.Rlocation(os.environ["TS_NATIVE_ZIP"]) + cls.common_zip = files.Rlocation(os.environ["TS_COMMON_ZIP"]) + cls.arch_zip = files.Rlocation(os.environ["TS_ARCH_ZIP"]) + with zipfile.ZipFile(cls.wrapper_zip) as archive: + cls.version = json.loads( + archive.read("typescript-parser-wrapper/package.json") + )["dependencies"]["typescript"] + + def stage(self, directory, distribution, mismatch=False): + modules = directory / "node_modules" + typescript = modules / "typescript" + typescript.mkdir(parents=True) + dependencies = {} + for platform in {p for ps in PLATFORMS.values() for p in ps}: + name = "typescript-" + platform + package = modules / "@typescript" / name + (package / "lib").mkdir(parents=True) + manifest = { + "name": "@typescript/" + name, + "version": "invalid" if mismatch else self.version, + } + (package / "package.json").write_text(json.dumps(manifest)) + executable = package / "lib" / ( + "tsc.exe" if platform.startswith("win32-") else "tsc" + ) + executable.write_text("native executable fixture") + executable.chmod(0o644) + dependencies[manifest["name"]] = self.version + (typescript / "package.json").write_text( + json.dumps({"version": self.version, "optionalDependencies": dependencies}) + ) + packages = subprocess.run( + [self.node, self.script, "packages", distribution], + cwd=directory, + capture_output=True, + text=True, + timeout=30, + ) + if distribution in PLATFORMS: + self.assertEqual(packages.returncode, 0, packages.stderr) + self.assertEqual( + packages.stdout.split(), + [ + "@typescript/typescript-" + platform + "@" + self.version + for platform in PLATFORMS[distribution] + ], + ) + else: + self.assertNotEqual(packages.returncode, 0) + self.assertIn("Unsupported CodeQL distribution", packages.stderr) + return subprocess.run( + [self.node, self.script, "stage", distribution], + cwd=directory, + capture_output=True, + text=True, + timeout=30, + ) + + def test_invalid_arguments_fail(self): + for arguments in [ + [], + ["stage"], + ["unsupported", "linux64"], + ["stage", "linux64", "extra"], + ]: + with self.subTest(arguments=arguments): + result = subprocess.run( + [self.node, self.script, *arguments], + capture_output=True, + text=True, + timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertIn( + "Usage: node package-runtime.mjs ", + result.stderr, + ) + + def test_distribution_selection_is_independent_of_host(self): + for distribution, platforms in PLATFORMS.items(): + with ( + self.subTest(distribution=distribution), + tempfile.TemporaryDirectory() as temp, + ): + directory = Path(temp) + result = self.stage(directory, distribution) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + self.assertEqual( + sorted(p.name for p in (directory / "native").iterdir()), + sorted("typescript-" + p for p in platforms), + ) + self.assertFalse((directory / "node_modules/@typescript").exists()) + if os.name != "nt": + for executable in (directory / "native").glob("*/lib/tsc"): + self.assertEqual(stat.S_IMODE(executable.stat().st_mode), 0o755) + + def test_invalid_distribution_and_mismatched_runtime_fail(self): + for distribution, mismatch, message in [ + ("unsupported", False, "Unsupported CodeQL distribution"), + ("osx64", True, "Native runtime does not match TypeScript"), + ]: + with ( + self.subTest(distribution=distribution), + tempfile.TemporaryDirectory() as temp, + ): + result = self.stage(Path(temp), distribution, mismatch) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertIn(message, result.stderr) + + def extract(self, filename, destination): + destination.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + [self.ripunzip, "unzip-file", filename, "-d", str(destination)], + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_installed_archives_parse_without_external_typescript(self): + host = subprocess.check_output( + [self.node, "-p", "process.platform + '-' + process.arch"], text=True + ).strip() + platforms = next(ps for ps in PLATFORMS.values() if host in ps) + with zipfile.ZipFile(self.wrapper_zip) as archive: + self.assertFalse(any("/@typescript/" in n for n in archive.namelist())) + with zipfile.ZipFile(self.native_zip) as archive: + names = archive.namelist() + self.assertEqual( + sorted({n.removeprefix("./").split("/")[0] for n in names}), + sorted("typescript-" + p for p in platforms), + ) + for platform in platforms: + prefix = "typescript-" + platform + manifest = json.loads(archive.read(prefix + "/package.json")) + self.assertEqual(manifest["version"], self.version) + binary = prefix + "/lib/" + ( + "tsc.exe" if platform.startswith("win32-") else "tsc" + ) + self.assertGreater(archive.getinfo(binary).file_size, 0) + if not platform.startswith("win32-"): + self.assertEqual( + archive.getinfo(binary).external_attr >> 16 & 0o777, 0o755 + ) + + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + with zipfile.ZipFile(self.common_zip) as archive: + self.assertFalse(any("/@typescript/" in n for n in archive.namelist())) + self.assertIn( + "tools/typescript-parser-wrapper/main.js", archive.namelist() + ) + with zipfile.ZipFile(self.arch_zip) as archive: + self.assertEqual( + sorted(n for n in archive.namelist() if not n.endswith("/")), + sorted( + "tools/typescript-parser-wrapper/node_modules/@typescript/" + n + for n in names + if not n.endswith("/") + ), + ) + self.extract(self.common_zip, directory) + self.extract(self.arch_zip, directory) + wrapper = directory / "tools/typescript-parser-wrapper" + environment = dict(os.environ) + environment.pop("NODE_PATH", None) + commands = [] + for name, source in [ + ("valid.ts", "const value: number = 1;"), + ("valid.tsx", "const value =
Hello
;"), + ("invalid.ts", "const value = ;"), + ]: + filename = directory / name + filename.write_text(source) + commands.extend( + [ + {"command": "parse", "filename": str(filename)}, + {"command": "reset"}, + ] + ) + commands.append({"command": "quit"}) + result = subprocess.run( + [self.node, str(wrapper / "main.js")], + input="".join(json.dumps(command) + "\n" for command in commands), + cwd=directory, + env=environment, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, "") + responses = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(len(responses), 6) + for response in responses[::2]: + self.assertEqual(response["type"], "ast") + for response in responses[1::2]: + self.assertEqual(response, {"type": "reset-done"}) + self.assertEqual(responses[0]["ast"]["parseDiagnostics"], []) + self.assertEqual(responses[2]["ast"]["parseDiagnostics"], []) + self.assertTrue(responses[4]["ast"]["parseDiagnostics"]) + + +if __name__ == "__main__": + unittest.main() From 9c7f5824c330ef0a26d009c009d08ac75d125f3c Mon Sep 17 00:00:00 2001 From: Simon Friis Vindum Date: Mon, 19 Oct 2026 11:59:10 +0200 Subject: [PATCH 3/5] JS: Migrate the parser to native TypeScript 7.1 Pin the native compiler and adopt its ESM API. Project native AST nodes into the existing JSON protocol and adapt Java conversion to native fields, token kinds, namespaces, and import attributes. Reuse one lazy API instance across files and prepare-files batches, disposing each program after serialization. Close the API on reset, quit, and EOF; rely on process-exit cleanup for fatal errors. Add wrapper, lifecycle, and Java regression coverage, teach the Java test harness to install native runtimes, and regenerate import-attribute expectations. Cover source-phase import bindings with a QL fixture. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8f505060-5341-43f3-b8e9-1b3d60171a2d --- .../lib/typescript/package-lock.json | 376 +++- .../extractor/lib/typescript/package.json | 13 +- .../lib/typescript/src/ast_extractor.ts | 312 ++-- .../extractor/lib/typescript/src/main.ts | 338 +--- .../lib/typescript/test/lifecycle.test.mjs | 304 ++++ .../lib/typescript/test/wrapper.test.mjs | 343 ++++ .../extractor/lib/typescript/tsconfig.json | 10 +- .../ts/extractor/TypeScriptASTConverter.java | 62 +- .../semmle/js/extractor/test/AllTests.java | 51 +- .../com/semmle/js/extractor/test/BUILD.bazel | 3 + .../js/extractor/test/TypeScriptTests.java | 99 + .../ts/output/trap/import-attributes.ts.trap | 1592 ++++++++++------- .../TypeScript/SourceImports/test.expected | 5 + .../TypeScript/SourceImports/test.ql | 5 + .../TypeScript/SourceImports/tst.ts | 3 + 15 files changed, 2401 insertions(+), 1115 deletions(-) create mode 100644 javascript/extractor/lib/typescript/test/lifecycle.test.mjs create mode 100644 javascript/extractor/lib/typescript/test/wrapper.test.mjs create mode 100644 javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java create mode 100644 javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected create mode 100644 javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql create mode 100644 javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts diff --git a/javascript/extractor/lib/typescript/package-lock.json b/javascript/extractor/lib/typescript/package-lock.json index 68923719ff8d..662d844b9a38 100644 --- a/javascript/extractor/lib/typescript/package-lock.json +++ b/javascript/extractor/lib/typescript/package-lock.json @@ -6,10 +6,13 @@ "": { "name": "typescript-parser-wrapper", "dependencies": { - "typescript": "5.9" + "typescript": "7.1.0-beta" }, "devDependencies": { "@types/node": "^24.3.1" + }, + "engines": { + "node": ">=16.20.0" } }, "node_modules/@types/node": { @@ -21,16 +24,375 @@ "undici-types": "~7.10.0" } }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.1.0-beta.tgz", + "integrity": "sha512-DXDG+jIWxBHGdoO70tcHON33rCnSzsEZj9P13oyf/r/KEGIo6awsdSNVNjAw0uMwEpqRu75IrgCcW58YcVkZHg==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-android-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-android-arm64/-/typescript-android-arm64-7.1.0-beta.tgz", + "integrity": "sha512-UZx6PxoQavuvaVTrSf3aw0Oy/J/uJpRylipA0kzc038jMvZXh0UtE8vnQp20FqtdgD5bhXZ0aJrRJyoTSUO58Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.1.0-beta.tgz", + "integrity": "sha512-q7mxblcyX0E7hXsY4k+Q6KWKK5mGmfVq+BDesdTgRVgjodk27aIxER77Zyj9+TgSTlCje8F7+JM6f6hl2BuNdw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.1.0-beta.tgz", + "integrity": "sha512-D5EYqG5Wm/guw9+0j8+Mn1Jbr/gfS/VtFTuoH8PJtFbcqeDaQGMulc7SW8q51QG6Y3wGOGX3nbsTOw6fIKWWAA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.1.0-beta.tgz", + "integrity": "sha512-9A0XX0tcnH9qPdolSKKzSjTb93dlB0UEcAJbHYEsrFX2fkFNSum9+dvkQQJ9vLc+7vxHetDPujq9Qe6wx+Xhow==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.1.0-beta.tgz", + "integrity": "sha512-EDf/dBHfY0TxX3NmUTek0hbffeyZlh3+ls6WQW3ICB8vYxwYLFnzg7wkgEUvl9eRlpiJ/nmc2S8arPMzINJ7rg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.1.0-beta.tgz", + "integrity": "sha512-AvN9HmzOQh46G4312ooSgMHhbB7DtnLwg8D9mo324X49B7weMwtJrBG1MxT9IyI8kzDd7YRtW+1hWGun7WG7Qw==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.1.0-beta.tgz", + "integrity": "sha512-hNqQBgQ5b5oGqN76QqbAU6CwYhJZ6iOcWc2kPTUs2OentgpxZktETziH6KIyRFJPaiBncognwn2ZLu3DNhHiWw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.1.0-beta.tgz", + "integrity": "sha512-xbUustlBrFflt2pNEyG5ye+d4WGPc/rfr2xK29MXeobPSw297JqwuKRrvbKM5o3fE/eUM/c+urVm6jQ3IaMVTw==", + "cpu": [ + "loong64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.1.0-beta.tgz", + "integrity": "sha512-fS4/G7/0bXZ3NSq3nBMAXyg65I3tO4S7j08VNQ2g1aNC/qq79gE7Qne1BNjVsjMFfHBVBFjQYoTQCjmVU2iFlg==", + "cpu": [ + "mips64el" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.1.0-beta.tgz", + "integrity": "sha512-0upCCxYPX8X6gOwkvfv+Ee7/ufyy7k97mTrGE0e5BxLlnUEa+w+rtgnv2dsKcxFdwdeZYVlZKCqgN/xCBV08+A==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.1.0-beta.tgz", + "integrity": "sha512-0jKU9YRgOPRgiy5V/UWv37kB53PaTpIbwGCbbOMUPPAkrjVRNOIAJaz7Wa7s3H7mdDebX1LopUEPEJR+7Xa7zg==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.1.0-beta.tgz", + "integrity": "sha512-j+d04s2kxaPQh6DLPb01sknAQfzsrdSZy15jBRHyLEZoaaQ19dVjbFRlkNVvYwUG7ZdP8+y5b/7RmPxdtHUKdQ==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.1.0-beta.tgz", + "integrity": "sha512-kcD51ihf+pnI5KT4uQQYG04OpwznaZYBlIQhDhl8fflFd27NktxXsUeguxZyvUMyWfum7IF2bBS6Jv7ox0AgFQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.1.0-beta.tgz", + "integrity": "sha512-8W85YyoGK+rgGoSU9jJQUpELbrKcZcbdJk2jPNQ6UKDKI7ryPlPFTrV2jCE5RQuv4Tt96hMDTc33s4YfRrLFBA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.1.0-beta.tgz", + "integrity": "sha512-iGsTE0ve2PG0AcyZinwAR0LnGEZhu/y4dceS0hZsYCaKdYETvQtvtlNWlmYKempsBr6C7Il7dMXHVk+VFm9u0g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.1.0-beta.tgz", + "integrity": "sha512-Z2FERdHB9fAKI+KGqdWn04O7uNlnB+VYiEm6INE1aZuzzLNd3KBqQ6IpKUJF2ny4GigGU6nVu75XD9tAUODPng==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.1.0-beta.tgz", + "integrity": "sha512-NVG3qeyCq/rJEOhaRjR3LsWgjrjo5vJW25omVLeHAVnpKIG4pE4cenwwKGR06DiJ1s0QTOymcJeKXMEpsiNBVg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.1.0-beta.tgz", + "integrity": "sha512-ovAdoWXEn2B5/fyWG69pLU3FWmQwPWOjQV5TFbcxtpnOv+R3g8ySGHl1NYRq7Lb/GYdFDvaGcZ+SiU7iKuiMsA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.1.0-beta.tgz", + "integrity": "sha512-lJbo71IEOfL7ZZIZnArRaOLqjn9BAfoSzyUQn0aSjfSpDZL4ru0afUsdBfo6PQEjD6ssdmi0n/RwaPADX6bsvQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.1.0-beta.tgz", + "integrity": "sha512-UKcHvbybY2tDTvFW2b1vwiAvDlIRooo69lh8CXWVhOQejcvpM5+IgbiFW9EIwR5Wi5X/v+BdN7gdWbigBuJ3rw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/typescript": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", - "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "version": "7.1.0-beta", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.1.0-beta.tgz", + "integrity": "sha512-cjWjZAJDsmZzsNC5GwHNlW0YPM65jZa5oR0KyQcYX3KSVIfTvtBq//vewISyatKk4OLi5grZjHhIVE9fdrw1UQ==", + "license": "Apache-2.0", "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" + "tsc": "bin/tsc" }, "engines": { - "node": ">=14.17" + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.1.0-beta", + "@typescript/typescript-android-arm64": "7.1.0-beta", + "@typescript/typescript-darwin-arm64": "7.1.0-beta", + "@typescript/typescript-darwin-x64": "7.1.0-beta", + "@typescript/typescript-freebsd-arm64": "7.1.0-beta", + "@typescript/typescript-freebsd-x64": "7.1.0-beta", + "@typescript/typescript-linux-arm": "7.1.0-beta", + "@typescript/typescript-linux-arm64": "7.1.0-beta", + "@typescript/typescript-linux-loong64": "7.1.0-beta", + "@typescript/typescript-linux-mips64el": "7.1.0-beta", + "@typescript/typescript-linux-ppc64": "7.1.0-beta", + "@typescript/typescript-linux-riscv64": "7.1.0-beta", + "@typescript/typescript-linux-s390x": "7.1.0-beta", + "@typescript/typescript-linux-x64": "7.1.0-beta", + "@typescript/typescript-netbsd-arm64": "7.1.0-beta", + "@typescript/typescript-netbsd-x64": "7.1.0-beta", + "@typescript/typescript-openbsd-arm64": "7.1.0-beta", + "@typescript/typescript-openbsd-x64": "7.1.0-beta", + "@typescript/typescript-sunos-x64": "7.1.0-beta", + "@typescript/typescript-win32-arm64": "7.1.0-beta", + "@typescript/typescript-win32-x64": "7.1.0-beta" } }, "node_modules/undici-types": { diff --git a/javascript/extractor/lib/typescript/package.json b/javascript/extractor/lib/typescript/package.json index f23f197e870b..8fb036890778 100644 --- a/javascript/extractor/lib/typescript/package.json +++ b/javascript/extractor/lib/typescript/package.json @@ -1,15 +1,20 @@ { "name": "typescript-parser-wrapper", "private": true, + "type": "module", + "engines": { + "node": ">=16.20.0" + }, "dependencies": { - "typescript": "5.9" + "typescript": "7.1.0-beta" }, "scripts": { - "build": "tsc --project tsconfig.json", - "check": "tsc --noEmit --project . && tslint --project .", + "build": "node node_modules/typescript/lib/tsc.js --project tsconfig.json", + "check": "node node_modules/typescript/lib/tsc.js --noEmit --project . && tslint --project .", + "test": "npm run build && node --test test/lifecycle.test.mjs test/wrapper.test.mjs", "lint": "tslint --project .", "lint-fix": "tslint --project . --fix", - "watch": "tsc -p . -w --sourceMap" + "watch": "node node_modules/typescript/lib/tsc.js -p . -w --sourceMap" }, "devDependencies": { "@types/node": "^24.3.1" diff --git a/javascript/extractor/lib/typescript/src/ast_extractor.ts b/javascript/extractor/lib/typescript/src/ast_extractor.ts index ac3b8eb69ed6..bdac77e11a40 100644 --- a/javascript/extractor/lib/typescript/src/ast_extractor.ts +++ b/javascript/extractor/lib/typescript/src/ast_extractor.ts @@ -1,143 +1,217 @@ -import * as ts from "typescript"; - -/** - * Interface that exposes some internal properties we rely on, as well as - * some properties we add ourselves. - */ -export interface AugmentedSourceFile extends ts.SourceFile { - parseDiagnostics?: any[]; - /** Internal property that we expose as a workaround. */ - redirectInfo?: object | null; - $tokens?: Token[]; - $lineStarts?: ReadonlyArray; -} +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { SyntaxKind } from "typescript/ast"; +import * as ts from "typescript/ast"; +import type { Diagnostic } from "typescript/sync"; -export interface AugmentedNode extends ts.Node { - $pos?: any; - $end?: any; - $declarationKind?: string; +export type JsonValue = string | number | boolean | JsonObject | JsonValue[]; +export interface JsonObject { + [property: string]: JsonValue; } -export type AugmentedPos = number; +/** Child and scalar properties consumed by the Java AST converter. */ +const astProperties = [ + "argument", + "argumentExpression", + "arguments", + "assertsModifier", + "asteriskToken", + "attributes", + "block", + "body", + "caseBlock", + "catchClause", + "checkType", + "children", + "clauses", + "closingElement", + "closingFragment", + "condition", + "constraint", + "declarationList", + "declarations", + "defaultType", + "dotDotDotToken", + "elements", + "elementType", + "elseStatement", + "exclamationToken", + "exportClause", + "expression", + "exprName", + "extendsType", + "falseType", + "finallyBlock", + "head", + "heritageClauses", + "importClause", + "incrementor", + "indexType", + "initializer", + "keywordToken", + "label", + "left", + "literal", + "members", + "modifiers", + "moduleReference", + "moduleSpecifier", + "name", + "namedBindings", + "objectType", + "openingElement", + "openingFragment", + "operand", + "operator", + "operatorToken", + "parameterName", + "parameters", + "phaseModifier", + "postfixToken", + "properties", + "propertyName", + "qualifier", + "questionDotToken", + "questionToken", + "right", + "statement", + "statements", + "tag", + "tagName", + "template", + "templateSpans", + "text", + "thenStatement", + "token", + "trueType", + "tryBlock", + "type", + "typeArguments", + "typeName", + "typeParameter", + "typeParameters", + "types", + "value", + "variableDeclaration", + "whenFalse", + "whenTrue", +]; -export interface Token { - kind: ts.SyntaxKind; - tokenPos: AugmentedPos; - text: string; -} +const require = createRequire(import.meta.url); +const packageRoot = dirname(require.resolve("typescript/package.json")); -let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; +// NOTE: Importing this file directly from the `typescript` package is a bit +// hacky, but the child properties are not exposed through the public API. +const { childProperties } = await import( + pathToFileURL(join(packageRoot, "dist/api/node/protocol.generated.js")).href); -/** - * Invokes the given callback for every AST node in the given tree. - */ -function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) { - function visit(node: ts.Node) { - ts.forEachChild(node, visit); - callback(node); +const scalarAstProperties = ["keywordToken", "operator", "phaseModifier", "text", "token"]; +const astPropertiesByKind = {} as Record; +for (const kind of Object.values(SyntaxKind)) { + if (typeof kind === "number" && kind !== SyntaxKind.Count) { + const children = childProperties[kind] ?? []; + astPropertiesByKind[kind] = astProperties.filter(property => + scalarAstProperties.includes(property) || children.includes(property)); } - visit(ast); } -export function augmentAst(ast: AugmentedSourceFile, code: string) { - ast.$lineStarts = ast.getLineStarts(); - - /** - * Converts a numeric offset to the value expected by the Java counterpart of the extractor. - */ - function augmentPos(pos: number, shouldSkipWhitespace?: boolean): AugmentedPos { - // skip over leading spaces/comments - if (shouldSkipWhitespace) { - skipWhiteSpace.lastIndex = pos; - pos += skipWhiteSpace.exec(code)[0].length; - } - return pos; +const skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; + +export function extractAst(ast: ts.SourceFile, diagnostics: readonly Diagnostic[]): JsonObject { + const code = ast.text; + // The native parser still builds assertion ASTs, but now diagnoses this formerly accepted syntax. + const parseDiagnostics = diagnostics.filter(diagnostic => diagnostic.code !== 2880); + + function augmentPos(pos: number): number { + skipWhiteSpace.lastIndex = pos; + return pos + skipWhiteSpace.exec(code)![0].length; } - // Find the position of all tokens where the scanner requires parse-tree information. - // At these offsets, a call to `scanner.reScanX` is required. - type RescanEvent = () => void; - let reScanEvents: RescanEvent[] = []; - let reScanEventPos: number[] = []; - let scanner = ts.createScanner(ts.ScriptTarget.ES2015, false, 1, code); - let reScanSlashToken = scanner.reScanSlashToken.bind(scanner); - let reScanTemplateToken = scanner.reScanTemplateToken.bind(scanner); - let reScanGreaterToken = scanner.reScanGreaterToken.bind(scanner); - if (!ast.parseDiagnostics || ast.parseDiagnostics.length === 0) { - forEachNode(ast, node => { - if (ts.isRegularExpressionLiteral(node)) { - reScanEventPos.push(node.getStart(ast, false)); - reScanEvents.push(reScanSlashToken); - } - if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { - reScanEventPos.push(node.getStart(ast, false)); - reScanEvents.push(reScanTemplateToken); - } - if (ts.isBinaryExpression(node)) { - let operator = node.operatorToken; - switch (operator.kind) { - case ts.SyntaxKind.GreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: - case ts.SyntaxKind.GreaterThanGreaterThanToken: - reScanEventPos.push(operator.getStart(ast, false)); - reScanEvents.push(reScanGreaterToken); - break; - } - } - }); + function projectValue(value: unknown): JsonValue { + if (Array.isArray(value)) return Array.from(value, projectValue); + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return value; + } + if (value != null && typeof value === "object" && "forEachChild" in value) { + return projectNode(value as ts.Node); + } + throw new Error("Unexpected TypeScript AST property value"); } - reScanEventPos.push(Infinity); + function projectNode(node: ts.Node): JsonObject { + const result: JsonObject = { + kind: node.kind, + flags: node.flags, + $pos: augmentPos(node.pos), + $end: node.end, + }; + const properties = node as ts.Node & { [property: string]: unknown }; + for (const property of astPropertiesByKind[node.kind]) { + const value = properties[property]; + if (value != null) result[property] = projectValue(value); + } - // add tokens and comments to the AST - ast.$tokens = []; - let rescanEventIndex = 0; - let nextRescanPosition = reScanEventPos[0]; - let tk; - do { - tk = scanner.scan(); - if (scanner.getTokenPos() === nextRescanPosition) { - let callback = reScanEvents[rescanEventIndex]; - callback(); - ++rescanEventIndex; - nextRescanPosition = reScanEventPos[rescanEventIndex]; + // Native nodes share boolean getters even on kinds where they are not meaningful. + if (ts.isExportAssignment(node)) result.isExportEquals = node.isExportEquals; + if (ts.isExportDeclaration(node) || ts.isImportEqualsDeclaration(node) + || ts.isImportSpecifier(node) || ts.isExportSpecifier(node)) { + result.isTypeOnly = node.isTypeOnly; } - ast.$tokens.push({ - kind: tk, - tokenPos: augmentPos(scanner.getTokenPos()), - text: scanner.getTokenText(), - }); - } while (tk !== ts.SyntaxKind.EndOfFileToken); + if (ts.isImportTypeNode(node)) result.isTypeOf = node.isTypeOf; - if (ast.parseDiagnostics) { - ast.parseDiagnostics.forEach(d => { - delete d.file; - d.$pos = augmentPos(d.start); - }); + return result; } - visitAstNode(ast); - function visitAstNode(node: AugmentedNode) { - ts.forEachChild(node, visitAstNode); + const result = projectNode(ast); + result.$lineStarts = Array.from(ast.getLineStarts()); + result.$tokens = extractTokens(ast, parseDiagnostics.length === 0); + result.parseDiagnostics = parseDiagnostics.map(diagnostic => ({ + messageText: diagnostic.text, + $pos: diagnostic.pos, + })); + return result; +} - // fill in line/column info - if ("pos" in node) { - node.$pos = augmentPos(node.pos, true); - } - if ("end" in node) { - node.$end = augmentPos(node.end); - } +function extractTokens(ast: ts.SourceFile, rescan: boolean): JsonObject[] { + const scanner = ts.createScanner(false, ts.LanguageVariant.JSX, ast.text); + const rescans = new Map ts.SyntaxKind>(); + if (rescan) visit(ast); - if (ts.isVariableDeclarationList(node as any)) { - if (node.flags & ts.NodeFlags.Let) { - node.$declarationKind = "let"; - } else if (node.flags & ts.NodeFlags.Const) { - node.$declarationKind = "const"; - } else { - node.$declarationKind = "var"; + function visit(node: ts.Node): void { + node.forEachChild(visit); + if (ts.isRegularExpressionLiteral(node)) { + rescans.set(node.getStart(ast), () => scanner.reScanSlashToken()); + } else if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { + const template = node.parent.parent; + const tagged = ts.isTemplateExpression(template) && ts.isTaggedTemplateExpression(template.parent); + rescans.set(node.getStart(ast), () => scanner.reScanTemplateToken(tagged)); + } else if (ts.isBinaryExpression(node)) { + const operator = node.operatorToken; + switch (operator.kind) { + case ts.SyntaxKind.GreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: + case ts.SyntaxKind.GreaterThanGreaterThanToken: + rescans.set(operator.getStart(ast), () => scanner.reScanGreaterToken()); + break; } } } + + const tokens: JsonObject[] = []; + let kind: ts.SyntaxKind; + do { + kind = scanner.scan(); + const callback = rescans.get(scanner.getTokenStart()); + // Keep the pre-rescan kind, as in the original Java-facing token protocol. + if (callback != null) callback(); + tokens.push({ + kind, + tokenPos: scanner.getTokenStart(), + text: scanner.getTokenText(), + }); + } while (kind !== ts.SyntaxKind.EndOfFile); + return tokens; } diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index 7ca31c428123..0a41991bbd5e 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -8,7 +8,8 @@ * When run without arguments, the wrapper enters server mode, expecting JSON-encoded requests * on `stdin`, one per line. * - * Each request should have a `command` property, which should either be `parse` or `quit`. + * Each request should have a `command` property, which should be `parse`, `prepare-files`, + * `reset`, `get-metadata`, or `quit`. * * A `quit` request causes the wrapper to terminate. * @@ -16,25 +17,15 @@ * TypeScript file. * * In response to a `parse` request, the wrapper invokes the TypeScript parser to parse - * the given file, and dumps a JSON representation of the resulting AST to stdout, - * with the following adjustments: - * - * - the `kind` and `operator` properties have symbolic string values instead of numeric - * values (the latter change between versions of the TypeScript compiler, the former - * don't); - * - position information is given in terms of line/column pairs instead of offsets; - * - declarations have a `declarationKind` property indicating whether they are `var`, - * `let` or `const` declarations; - * - `parent` pointers are omitted (since they cannot be serialised to JSON); - * - tokens and comments are added to the AST root node in a `tokens` property. + * the given file, and dumps a JSON representation of the resulting AST to stdout. */ -"use strict"; - import * as pathlib from "path"; import * as readline from "readline"; -import * as ts from "typescript"; -import * as ast_extractor from "./ast_extractor"; +import * as ts from "typescript/ast"; +import { API, JsxEmit } from "typescript/sync"; +import { version } from "typescript"; +import { extractAst } from "./ast_extractor.js"; // Remove limit on stack trace depth. Error.stackTraceLimit = Infinity; @@ -63,16 +54,13 @@ class State { /** List of files that have been requested. */ public pendingFiles: string[] = []; public pendingFileIndex = 0; - /** Next response to be delivered. */ - public pendingResponse: string = null; + public pendingResponse: string | null = null; } let state = new State(); /** - * Debugging method for finding cycles in the TypeScript AST. Should not be used in production. - * - * If cycles are found, the whitelist in `astProperties` is too permissive. + * Debugging method for finding cycles in the projected AST. Should not be used in production. */ // tslint:disable-next-line:no-unused-variable function checkCycle(root: any) { @@ -85,8 +73,6 @@ function checkCycle(root: any) { obj.$cycle_visiting = true; for (let k in obj) { if (!obj.hasOwnProperty(k)) continue; - // Ignore numeric and whitelisted properties. - if (+k !== +k && !astPropertySet.has(k)) continue; if (k === "$cycle_visiting") continue; let cycle = visit(obj[k]); if (cycle) { @@ -104,293 +90,123 @@ function checkCycle(root: any) { } } -/** Property names to extract from the TypeScript AST. */ -const astProperties: string[] = [ - "$declarationKind", - "$end", - "$lineStarts", - "$overloadIndex", - "$pos", - "$tokens", - "argument", - "argumentExpression", - "arguments", - "assertsModifier", - "asteriskToken", - "attributes", - "block", - "body", - "caseBlock", - "catchClause", - "checkType", - "children", - "clauses", - "closingElement", - "closingFragment", - "condition", - "constraint", - "constructor", - "declarationList", - "declarations", - "default", - "delete", - "dotDotDotToken", - "elements", - "elementType", - "elementTypes", - "elseStatement", - "escapedText", - "exclamationToken", - "exportClause", - "expression", - "exprName", - "extendsType", - "falseType", - "finallyBlock", - "flags", - "head", - "heritageClauses", - "importClause", - "incrementor", - "indexType", - "init", - "initializer", - "isExportEquals", - "isTypeOf", - "isTypeOnly", - "keywordToken", - "kind", - "label", - "left", - "literal", - "members", - "messageText", - "modifiers", - "moduleReference", - "moduleSpecifier", - "name", - "namedBindings", - "objectType", - "openingElement", - "openingFragment", - "operand", - "operator", - "operatorToken", - "parameterName", - "parameters", - "parseDiagnostics", - "phaseModifier", - "properties", - "propertyName", - "qualifier", - "questionDotToken", - "questionToken", - "right", - "selfClosing", - "statement", - "statements", - "tag", - "tagName", - "template", - "templateSpans", - "text", - "thenStatement", - "token", - "tokenPos", - "trueType", - "tryBlock", - "type", - "typeArguments", - "typeName", - "typeParameter", - "typeParameters", - "types", - "variableDeclaration", - "whenFalse", - "whenTrue", -]; +// The API instance used to interact with the TypeScript compiler. Should be +// accessed through `getAPI` and `closeAPI`. +let api: API | undefined; -/** Property names used in a parse command response, in addition to the AST itself. */ -const astMetaProperties: string[] = [ - "ast", - "type", -]; - -/** Property names to extract in an AST response. */ -const astPropertySet = new Set([...astProperties, ...astMetaProperties]); +function getAPI(): API { + return api ??= new API(); +} -/** - * Converts (part of) an AST to a JSON string, ignoring properties we're not interested in. - */ -function stringifyAST(obj: any) { - return JSON.stringify(obj, (k, v) => { - // Filter out properties that aren't numeric, empty, or whitelisted. - // Note `k` is the empty string for the root object, which is also covered by +k === +k. - return (+k === +k || astPropertySet.has(k)) ? v : undefined; - }); +function closeAPI() { + const previous = api; + api = undefined; + previous?.close(); } function extractFile(filename: string): string { - let ast = getAstForFile(filename); - - return stringifyAST({ + const program = getAPI().createProgram([pathlib.resolve(filename)], { + experimentalDecorators: true, + jsx: JsxEmit.Preserve, + noResolve: true, + noLib: true, + }); + const ast = program.getSourceFile(pathlib.resolve(filename)); + if (ast == null) { + throw new Error("TypeScript did not return a source file for " + filename); + } + const json = JSON.stringify({ type: "ast", - ast, + ast: extractAst(ast, program.getSyntacticDiagnostics()), }); + program.dispose(); + return json; } function prepareNextFile() { if (state.pendingResponse != null) return; if (state.pendingFileIndex < state.pendingFiles.length) { - let nextFilename = state.pendingFiles[state.pendingFileIndex]; - state.pendingResponse = extractFile(nextFilename); + state.pendingResponse = extractFile(state.pendingFiles[state.pendingFileIndex]); } } -function handleParseCommand(command: ParseCommand, checkPending = true) { - let filename = command.filename; - let expectedFilename = state.pendingFiles[state.pendingFileIndex]; - if (expectedFilename !== filename && checkPending) { - // File was requested out of order. This happens in rare cases because the Java process decided against extracting it, - // for example because it was too large. Just recover and accept that some work was wasted. +// Start working on the next file as soon as the old one is flushed. +// Note that if we didn't wait for flushing, this would block the I/O +// loop and delay flushing. +function afterResponse(expectedState: State) { + return (error?: Error | null) => { + if (!error && state === expectedState) { + prepareNextFile(); + } + }; +} + +function handleParseCommand(command: ParseCommand) { + const filename = command.filename; + const expectedFilename = state.pendingFiles[state.pendingFileIndex]; + if (expectedFilename !== filename) { + // File was requested out of order. This happens in rare cases because the Java process decided + // against extracting it, for example because it was too large. Just recover and accept some wasted work. state.pendingResponse = null; state.pendingFileIndex = state.pendingFiles.indexOf(filename); } ++state.pendingFileIndex; - let response = state.pendingResponse || extractFile(command.filename); + const response = state.pendingResponse || extractFile(filename); state.pendingResponse = null; - process.stdout.write(response + "\n", () => { - // Start working on the next file as soon as the old one is flushed. - // Note that if we didn't wait for flushing, this would block the I/O - // loop and delay flushing. - prepareNextFile(); - }); -} - -/** - * Returns true if the given source file has an actual AST, as opposed to - * a SourceFile that is a "redirect" to another SourceFile. - * - * The TypeScript API should not expose such redirecting SourceFiles, - * but we sometimes get them anyway. - */ -function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolean { - return ast.redirectInfo == null; -} - -/** - * Gets the AST and source code by parsing the given file. - */ -function getAstForFile(filename: string): ts.SourceFile { - let { ast, code } = parseSingleFile(filename); - if (ast != null && isExtractableSourceFile(ast)) { - ast_extractor.augmentAst(ast, code); - } - return ast; -} - -function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } { - let code = ts.sys.readFile(filename); - - // create a compiler host that only allows access to `filename` - let compilerHost: ts.CompilerHost = { - fileExists() { return true; }, - getCanonicalFileName() { return filename; }, - getCurrentDirectory() { return ""; }, - getDefaultLibFileName() { return "lib.d.ts"; }, - getNewLine() { return "\n"; }, - getSourceFile() { - return ts.createSourceFile(filename, code, ts.ScriptTarget.Latest, true); - }, - readFile() { return null; }, - useCaseSensitiveFileNames() { return true; }, - writeFile() { return null; }, - getDirectories() { return []; }, - }; - - // parse `filename` with minimial checking - let compilerOptions: ts.CompilerOptions = { - experimentalDecorators: true, - jsx: ts.JsxEmit.Preserve, - noResolve: true, - }; - - let program = ts.createProgram([filename], compilerOptions, compilerHost); - - let ast = program.getSourceFile(filename); - - return { ast, code }; -} - -function handleResetCommand(command: ResetCommand) { - reset(); - console.log(JSON.stringify({ - type: "reset-done", - })); -} - -function handlePrepareFilesCommand(command: PrepareFilesCommand) { - state.pendingFiles = command.filenames; - state.pendingFileIndex = 0; - state.pendingResponse = null; - process.stdout.write('{"type":"ok"}\n', () => { - prepareNextFile(); - }); -} - -function handleGetMetadataCommand(command: GetMetadataCommand) { - console.log(JSON.stringify({ - type: "metadata", - syntaxKinds: ts.SyntaxKind, - nodeFlags: ts.NodeFlags, - })); -} - -function reset() { - state = new State(); + process.stdout.write(response + "\n", afterResponse(state)); } function runReadLineInterface() { - reset(); - let rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + rl.on("close", () => { + state = new State(); + process.stdin.destroy(); + closeAPI(); + }); rl.on("line", (line: string) => { - let req: Command = JSON.parse(line); + const req: Command = JSON.parse(line); switch (req.command) { case "parse": handleParseCommand(req); break; case "prepare-files": - handlePrepareFilesCommand(req); + state = new State(); + state.pendingFiles = req.filenames; + process.stdout.write('{"type":"ok"}\n', afterResponse(state)); break; case "reset": - handleResetCommand(req); + state = new State(); + closeAPI(); + console.log('{"type":"reset-done"}'); break; case "get-metadata": - handleGetMetadataCommand(req); + console.log(JSON.stringify({ + type: "metadata", + syntaxKinds: ts.SyntaxKind, + nodeFlags: ts.NodeFlags, + })); break; case "quit": rl.close(); break; default: - throw new Error("Unknown command " + (req as any).command + "."); + throw new Error("Unknown command " + (req as { command: string }).command + "."); } }); } // Parse command-line arguments. if (process.argv.length > 2) { - let argument = process.argv[2]; + const argument = process.argv[2]; if (argument === "--version") { - console.log("parser-wrapper with TypeScript " + ts.version); + console.log("parser-wrapper with TypeScript " + version); } else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") { - handleParseCommand({ - command: "parse", - filename: argument, - }, false); + process.stdout.write(extractFile(argument) + "\n"); + closeAPI(); } else { console.error("Unrecognized file or flag: " + argument); + process.exitCode = 1; } - process.exit(0); } else { runReadLineInterface(); } diff --git a/javascript/extractor/lib/typescript/test/lifecycle.test.mjs b/javascript/extractor/lib/typescript/test/lifecycle.test.mjs new file mode 100644 index 000000000000..dd8d1a4d2718 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/lifecycle.test.mjs @@ -0,0 +1,304 @@ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { after, test } from "node:test"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-lifecycle-")); +const entryPoint = join(directory, "main.js"); +for (const name of ["main.js", "ast_extractor.js"]) { + copyFileSync(join(root, "build", name), join(directory, name)); +} +copyFileSync(join(root, "package.json"), join(directory, "package.json")); +symlinkSync(join(root, "node_modules"), join(directory, "node_modules"), "dir"); +const filename = join(directory, "input.ts"); +writeFileSync(filename, "const value = 1;"); +const preload = join(directory, "track-native.cjs"); +writeFileSync(preload, ` +const cp = require("node:child_process"); +const fs = require("node:fs"); +const spawn = cp.spawn; +cp.spawn = function (...args) { + const child = spawn(...args); + fs.appendFileSync(process.env.NATIVE_LOG, JSON.stringify({ + pid: child.pid, + stdio: args[2].stdio, + }) + "\\n"); + if (process.env.KILL_NATIVE) child.kill("SIGKILL"); + return child; +}; +`); +after(() => rmSync(directory, { recursive: true, force: true })); + +function options(name, env = {}) { + return { + encoding: "utf8", + timeout: 10000, + env: { + ...process.env, + ...env, + NATIVE_LOG: join(directory, name + ".log"), + }, + }; +} + +function run(name, input, env = {}, args = []) { + const opts = options(name, env); + const result = spawnSync(process.execPath, ["--require", preload, entryPoint, ...args], { + ...opts, input, + }); + assert.ifError(result.error); + assert.equal(result.signal, null, result.stderr); + return result; +} + +function nativeProcesses(name) { + return readFileSync(join(directory, name + ".log"), "utf8").trim().split("\n").map(JSON.parse); +} + +async function assertTerminated(pid) { + for (let i = 0; i < 100; ++i) { + try { + process.kill(pid, 0); + } catch (error) { + assert.equal(error.code, "ESRCH"); + return; + } + await setTimeout(10); + } + assert.fail(`Native process ${pid} survived disposal`); +} + +test("compiled wrapper preserves heritage, assertions and tokens, and accepts source-phase imports", () => { + const input = join(directory, "compatibility.ts"); + writeFileSync(input, ` + interface I extends A, N.B {} + class C implements I {} + import data from "data" assert { type: "json" }; + import source wasm from "./module.wasm"; + const expression = /data/g; + const template = \`value:\${data}\`; + `); + const result = run("compatibility", [ + { command: "get-metadata" }, + { command: "parse", filename: input }, + { command: "quit" }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + const [metadata, { ast }] = result.stdout.trim().split("\n").map(JSON.parse); + const kinds = metadata.syntaxKinds; + assert.deepEqual(ast.parseDiagnostics, []); + const heritage = ast.statements[0].heritageClauses[0].types; + assert.equal(heritage[0].kind, kinds.TypeReference); + assert.equal(heritage[0].typeName.text, "A"); + assert.equal(heritage[1].typeName.right.text, "B"); + assert.equal(heritage[1].typeArguments[0].kind, kinds.StringKeyword); + assert.equal(ast.statements[2].attributes.attributes[0].name.text, "type"); + assert.equal(ast.statements[2].attributes.attributes[0].value.text, "json"); + assert.equal(ast.statements[3].importClause.phaseModifier, kinds.SourceKeyword); + assert.equal(ast.statements[3].importClause.name.text, "wasm"); + assert(ast.$tokens.some(token => token.text === "/data/g" && token.kind === kinds.SlashToken)); + assert(ast.$tokens.some(token => token.text === "}`" && token.kind === kinds.CloseBraceToken)); +}); + +test("compiled ESM wrapper reuses its native process until reset or quit", { timeout: 15000 }, async () => { + const name = "repeated"; + const opts = options(name); + const child = spawn(process.execPath, ["--require", preload, entryPoint], opts); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", code => accept(code)); + }); + async function request(command) { + child.stdin.write(JSON.stringify(command) + "\n"); + const line = await lines.next(); + assert.equal(line.done, false, stderr); + return JSON.parse(line.value); + } + try { + for (let session = 0; session < 2; ++session) { + for (let batch = 0; batch < 2; ++batch) { + assert.deepEqual(await request({ command: "prepare-files", filenames: Array(5).fill(filename) }), + { type: "ok" }); + for (let i = 0; i < 5; ++i) { + const response = await request({ command: "parse", filename }); + assert.equal(response.type, "ast"); + assert.deepEqual(response.ast.parseDiagnostics, []); + const processes = nativeProcesses(name); + assert.equal(processes.length, session + 1); + const native = processes[session]; + assert.deepEqual(native.stdio, process.platform === "win32" + ? ["ignore", "ignore", "inherit"] : ["pipe", "pipe", "inherit"]); + process.kill(native.pid, 0); + } + } + if (session === 0) { + assert.deepEqual(await request({ command: "reset" }), { type: "reset-done" }); + await assertTerminated(nativeProcesses(name)[session].pid); + } + } + // Quit must terminate even when the sender leaves stdin open. + child.stdin.write('{"command":"quit"}\n'); + assert.equal(await exited, 0, stderr); + assert.equal(stderr, ""); + await assertTerminated(nativeProcesses(name)[1].pid); + } finally { + child.stdin.destroy(); + child.kill(); + await exited; + } +}); + +test("metadata and reset do not start a native process", () => { + const result = run("lazy", [ + { command: "get-metadata" }, + { command: "reset" }, + { command: "quit" }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + assert.deepEqual(result.stdout.trim().split("\n").map(line => JSON.parse(line).type), ["metadata", "reset-done"]); + assert.equal(existsSync(options("lazy").env.NATIVE_LOG), false); +}); + +test("reset and quit invalidate queued prefetch callbacks", () => { + const missing = join(directory, "missing.ts"); + const result = run("cancelled", [ + { command: "prepare-files", filenames: [missing] }, + { command: "reset" }, + { command: "prepare-files", filenames: [missing] }, + { command: "quit" }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + assert.deepEqual(result.stdout.trim().split("\n").map(JSON.parse), [ + { type: "ok" }, { type: "reset-done" }, { type: "ok" }, + ]); +}); + +test("single-file mode and EOF flush responses and close native processes", async () => { + for (const [name, input, args] of [ + ["single", "", [filename]], + ["eof", JSON.stringify({ command: "parse", filename }) + "\n", []], + ]) { + const result = run(name, input, {}, args); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.equal(result.stderr, ""); + await assertTerminated(nativeProcesses(name)[0].pid); + } +}); + +test("native failures are fatal, close the process, and never become JSON diagnostics", async () => { + const result = run("killed", "", { KILL_NATIVE: "1" }, [filename]); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /Error|EOF|EPIPE|ECONNRESET/); + await assertTerminated(nativeProcesses("killed")[0].pid); +}); + +test("fatal extraction errors are not masked by failing cleanup", async () => { + const failureEntryPoint = join(directory, "fail-source-file.mjs"); + writeFileSync(failureEntryPoint, ` + import { API, Program } from "typescript/sync"; + Object.defineProperties(Program.prototype, { + getSourceFile: { value() { throw new Error("Source file failure"); } }, + dispose: { value() { throw new Error("Program cleanup failure"); } }, + }); + Object.defineProperty(API.prototype, "close", { + value() { throw new Error("API cleanup failure"); }, + }); + await import("./main.js"); + `); + for (const [name, args, input] of [ + ["failed-single", [filename], ""], + ["failed-server", [], JSON.stringify({ command: "parse", filename }) + "\n"], + ]) { + const result = spawnSync(process.execPath, ["--require", preload, failureEntryPoint, ...args], { + ...options(name), input, + }); + assert.ifError(result.error); + assert.equal(result.signal, null, result.stderr); + assert.equal(result.status, 1, result.stderr); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /Source file failure/); + assert.doesNotMatch(result.stderr, /cleanup failure/); + await assertTerminated(nativeProcesses(name)[0].pid); + } +}); + +test("prefetch and stdout failures are fatal and terminate the native process", async () => { + for (const [name, command, message] of [ + ["prefetch", { command: "prepare-files", filenames: [join(directory, "missing.ts")] }, + /did not return a source file/], + ["stdout", { command: "parse", filename }, /EPIPE/], + ]) { + const child = spawn(process.execPath, ["--require", preload, entryPoint], options(name)); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + if (name === "stdout") child.stdout.destroy(); + else child.stdout.resume(); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", code => accept(code)); + }); + try { + child.stdin.write(JSON.stringify(command) + "\n"); + assert.equal(await exited, 1, stderr); + assert.match(stderr, message); + await assertTerminated(nativeProcesses(name)[0].pid); + } finally { + child.stdin.destroy(); + child.kill(); + await exited; + } + } +}); + +test("protocol and extraction failures close an already running native process", async () => { + for (const [name, input, message] of [ + ["active-json", "{\n", /SyntaxError/], + ["active-command", '{"command":"invalid"}\n', /Unknown command/], + ["active-missing", JSON.stringify({ command: "parse", filename: join(directory, "missing.ts") }) + "\n", + /did not return a source file/], + ]) { + const result = run(name, JSON.stringify({ command: "parse", filename }) + "\n" + input); + assert.equal(result.status, 1, name); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.match(result.stderr, message); + const processes = nativeProcesses(name); + assert.equal(processes.length, 1); + await assertTerminated(processes[0].pid); + } +}); + +test("invalid commands and files fail on stderr", () => { + for (const [name, input, args, message] of [ + ["json", "{\n", [], /SyntaxError/], + ["command", '{"command":"invalid"}\n', [], /Unknown command/], + ["missing", "", [join(directory, "missing.ts")], /did not return a source file/], + ]) { + const result = run(name, input, {}, args); + assert.equal(result.status, 1, name); + assert.equal(result.stdout, "", name); + assert.match(result.stderr, message); + } +}); + +test("the removed SEMMLE_TYPESCRIPT_HOME variable is ignored", () => { + const result = run("override-ignored", "", { + SEMMLE_TYPESCRIPT_HOME: join(directory, "missing-typescript"), + }, [filename]); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.equal(result.stderr, ""); +}); diff --git a/javascript/extractor/lib/typescript/test/wrapper.test.mjs b/javascript/extractor/lib/typescript/test/wrapper.test.mjs new file mode 100644 index 000000000000..db1113d8cf80 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/wrapper.test.mjs @@ -0,0 +1,343 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { copyFileSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { after, test } from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { API, JsxEmit } from "typescript/sync"; +import { SyntaxKind, NodeFlags } from "typescript/ast"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-wrapper-")); +const entryPoint = join(directory, "main.js"); +copyFileSync(join(root, "package.json"), join(directory, "package.json")); +symlinkSync(join(root, "node_modules"), join(directory, "node_modules"), "dir"); +for (const name of ["main.js", "ast_extractor.js"]) { + copyFileSync(join(root, "build", name), join(directory, name)); +} +const { extractAst } = await import(pathToFileURL(join(directory, "ast_extractor.js"))); +const api = new API(); + +after(() => { + try { + api.close(); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +function extract(text, extension = ".ts") { + const filename = join(directory, "input" + extension); + writeFileSync(filename, text); + const program = api.createProgram([filename], { + noResolve: true, noLib: true, jsx: JsxEmit.Preserve, experimentalDecorators: true, + }); + try { + return JSON.parse(JSON.stringify(extractAst(program.getSourceFile(filename), program.getSyntacticDiagnostics()))); + } finally { + program.dispose(); + api.clearSourceFileCache(); + } +} + +function declaration(ast, index = 0) { + return ast.statements[index].declarationList.declarations[0]; +} + +test("AST getters are projected without native implementation details or unrelated booleans", () => { + const text = "// comment\r\nexport const \\u0061 = \"\u{1f600}\";\r\nlet b = 2; var c = 3;"; + const ast = extract(text); + const first = declaration(ast); + assert.equal(first.name.text, "a"); + assert.equal(first.name.$pos, text.indexOf("\\u0061")); + assert.equal(first.initializer.text, "\u{1f600}"); + assert.equal(first.initializer.$end, text.indexOf('";') + 1); + assert.equal(ast.statements[0].modifiers[0].kind, SyntaxKind.ExportKeyword); + assert.deepEqual(ast.$lineStarts, [0, text.indexOf("export"), text.indexOf("let")]); + assert.deepEqual(ast.statements.map(statement => statement.declarationList.flags & NodeFlags.BlockScoped), + [NodeFlags.Const, NodeFlags.Let, NodeFlags.None]); + assert.deepEqual(ast.parseDiagnostics, []); + const json = JSON.stringify(ast); + for (const field of ["parent", "view", "sourceFile", "isTypeOnly", "isTypeOf", "isExportEquals"]) { + assert(!json.includes(`"${field}":`), field); + } +}); + +test("per-kind property lists retain optional fields absent from earlier nodes", () => { + const ast = extract(` + let empty; + let definite!: number; + let initialized: number = 1; + function plain() {} + export async function* rich(...args: T[]): Iterable { yield args[0]; } + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert(!("type" in declaration(ast))); + assert(!("initializer" in declaration(ast))); + assert.equal(declaration(ast, 1).exclamationToken.kind, SyntaxKind.ExclamationToken); + assert.equal(declaration(ast, 1).type.kind, SyntaxKind.NumberKeyword); + assert.equal(declaration(ast, 2).initializer.text, "1"); + const [plain, rich] = ast.statements.slice(3); + assert(!("modifiers" in plain)); + assert(!("typeParameters" in plain)); + assert.deepEqual(rich.modifiers.map(modifier => modifier.kind), [SyntaxKind.ExportKeyword, SyntaxKind.AsyncKeyword]); + assert.equal(rich.asteriskToken.kind, SyntaxKind.AsteriskToken); + assert.equal(rich.typeParameters[0].constraint.kind, SyntaxKind.ObjectKeyword); + assert.equal(rich.typeParameters[0].defaultType.kind, SyntaxKind.TypeLiteral); + assert.equal(rich.parameters[0].dotDotDotToken.kind, SyntaxKind.DotDotDotToken); + assert.equal(rich.type.typeName.text, "Iterable"); + assert.equal(rich.body.statements[0].expression.kind, SyntaxKind.YieldExpression); +}); + +test("scalar getters are retained alongside per-kind child properties", () => { + const ast = extract(` + import ordinary from "ordinary"; + import type { T } from "types"; + import source wasm from "./module.wasm"; + const positive = +value; + const negative = -value; + value++; + value--; + type Keys = keyof T; + function construct() { return new.target; } + const meta = import.meta; + interface I extends Base {} + class C extends Base implements I {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert(!("phaseModifier" in ast.statements[0].importClause)); + assert.equal(ast.statements[1].importClause.phaseModifier, SyntaxKind.TypeKeyword); + assert.equal(ast.statements[2].importClause.phaseModifier, SyntaxKind.SourceKeyword); + assert.equal(declaration(ast, 3).initializer.operator, SyntaxKind.PlusToken); + assert.equal(declaration(ast, 4).initializer.operator, SyntaxKind.MinusToken); + assert.equal(ast.statements[5].expression.operator, SyntaxKind.PlusPlusToken); + assert.equal(ast.statements[6].expression.operator, SyntaxKind.MinusMinusToken); + assert.equal(ast.statements[7].type.operator, SyntaxKind.KeyOfKeyword); + assert.equal(ast.statements[8].body.statements[0].expression.keywordToken, SyntaxKind.NewKeyword); + assert.equal(declaration(ast, 9).initializer.keywordToken, SyntaxKind.ImportKeyword); + assert.equal(ast.statements[10].heritageClauses[0].token, SyntaxKind.ExtendsKeyword); + assert.deepEqual(ast.statements[11].heritageClauses.map(clause => clause.token), + [SyntaxKind.ExtendsKeyword, SyntaxKind.ImplementsKeyword]); +}); + +test("known syntax kinds do not probe unrelated child getters", () => { + const ast = extractAst({ + kind: SyntaxKind.SourceFile, flags: 0, pos: 0, end: 0, text: "", statements: [], + forEachChild() {}, + getLineStarts() { return [0]; }, + get argument() { throw new Error("Unrelated child property was read"); }, + }, []); + assert.deepEqual(ast.statements, []); + assert.equal(ast.text, ""); +}); + +test("syntax error messages and UTF-16 offsets retain the Java-facing response", () => { + const text = 'const emoji = "\u{1f600}";\r\nconst broken = ;\r\n'; + const ast = extract(text); + assert.deepEqual(ast.parseDiagnostics, [{ + messageText: "Expression expected.", $pos: text.lastIndexOf(";"), + }]); + assert.deepEqual(ast.$lineStarts, [0, text.indexOf("const broken"), text.length]); +}); + +test("optional members, definite assignment, type defaults, and decorators use native fields", () => { + const ast = extract(` + type T = U; + interface I { p?: string; m?(): void } + @sealed export class C { + @field p!: string; + @method m(@parameter publicArg?: number) {} + } + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert.equal(ast.statements[0].typeParameters[0].defaultType.kind, SyntaxKind.NumberKeyword); + const [property, method] = ast.statements[1].members; + assert.equal(property.postfixToken.kind, SyntaxKind.QuestionToken); + assert.equal(method.postfixToken.kind, SyntaxKind.QuestionToken); + const cls = ast.statements[2]; + assert.deepEqual(cls.modifiers.map(modifier => modifier.kind), [SyntaxKind.Decorator, SyntaxKind.ExportKeyword]); + assert.equal(cls.members[0].postfixToken.kind, SyntaxKind.ExclamationToken); + assert.equal(cls.members[0].modifiers[0].kind, SyntaxKind.Decorator); + assert.equal(cls.members[1].parameters[0].modifiers[0].kind, SyntaxKind.Decorator); + assert.equal(cls.members[1].parameters[0].questionToken.kind, SyntaxKind.QuestionToken); +}); + +test("global augmentations and dotted namespaces need no synthetic flags", () => { + const ast = extract(` + declare /* a */ global { interface Window {} } + namespace global {} + module global {} + namespace A.B.C {} + namespace A { namespace B {} } + declare module "external" {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + const [global, namedGlobal, moduleGlobal, dotted, nested, external] = ast.statements; + assert.equal(global.name.text, "global"); + assert.equal(namedGlobal.name.text, "global"); + assert.equal(moduleGlobal.name.text, "global"); + assert.equal(dotted.body.kind, SyntaxKind.ModuleDeclaration); + assert.equal(dotted.body.body.kind, SyntaxKind.ModuleDeclaration); + assert.equal(dotted.body.flags, NodeFlags.None); + assert.equal(dotted.body.body.flags, NodeFlags.None); + assert.equal(nested.body.statements[0].flags, NodeFlags.None); + assert.equal(external.name.text, "external"); +}); + +test("import attributes and type-only phases use native property names", () => { + const ast = extract(` + import type { X } from "types"; + import { type Y, Z } from "types"; + import data from "data" with { type: "json" }; + export { data } from "data" with { type: "json" }; + type T = typeof import("data", { with: { type: "json" } }); + export type { X }; + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert.equal(ast.statements[0].importClause.phaseModifier, SyntaxKind.TypeKeyword); + assert.equal(ast.statements[1].importClause.namedBindings.elements[0].isTypeOnly, true); + assert.equal(ast.statements[1].importClause.namedBindings.elements[1].isTypeOnly, false); + for (const node of [ast.statements[2], ast.statements[3], ast.statements[4].type]) { + assert.equal(node.attributes.attributes[0].name.text, "type"); + assert.equal(node.attributes.attributes[0].value.text, "json"); + } + assert.equal(ast.statements[4].type.isTypeOf, true); + assert.equal(ast.statements[5].isTypeOnly, true); +}); + +test("regex, nested templates, tagged templates, type templates and operators are rescanned", () => { + const text = 'const x = `a${/a+/gi.test(s) ? tag`b${v}c` : `d${w}e`}f`; ' + + 'type T = `a${string}b`; const y = a >= b >> c >>> d; a >>= b; a >>>= b;'; + const ast = extract(text); + assert.deepEqual(ast.parseDiagnostics, []); + const rescannedTexts = ["/a+/gi", "}c`", "}e`", "}f`", "}b`", ">=", ">>", ">>>", ">>=", ">>>="]; + const tokens = ast.$tokens.filter(token => rescannedTexts.includes(token.text)); + assert.deepEqual(tokens.map(token => token.text), ["/a+/gi", "}c`", "}e`", "}f`", "}b`", ">=", ">>", ">>>", ">>=", ">>>="]); + assert.deepEqual(tokens.map(token => token.kind), [ + SyntaxKind.SlashToken, + ...Array(4).fill(SyntaxKind.CloseBraceToken), + ...Array(5).fill(SyntaxKind.GreaterThanToken), + ]); + for (const token of tokens) { + assert.equal(text.slice(token.tokenPos, token.tokenPos + token.text.length), token.text); + } + assert.equal(ast.$tokens.at(-1).kind, SyntaxKind.EndOfFile); + assert.equal(ast.$tokens.at(-1).tokenPos, text.length); +}); + +test("interface heritage preserves native type references", () => { + const ast = extract(` + interface I extends A, N.B {} + class C extends Base implements A, N.B {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + const [iface, cls] = ast.statements; + for (const heritage of [iface.heritageClauses[0], cls.heritageClauses[1]]) { + assert.equal(heritage.types[0].kind, SyntaxKind.TypeReference); + assert.equal(heritage.types[0].typeName.text, "A"); + assert.equal(heritage.types[1].typeName.right.text, "B"); + assert.equal(heritage.types[1].typeArguments[0].kind, SyntaxKind.StringKeyword); + assert(!("expression" in heritage.types[1])); + } + assert.equal(cls.heritageClauses[0].types[0].expression.text, "Base"); +}); + +test("legacy import assertions retain attributes without hiding genuine syntax errors", () => { + const source = ` + import data from "data" assert { type: "json" }; + export { data } from "data" assert { type: "json" }; + const pattern = /data/g; + `; + const ast = extract(source); + assert.deepEqual(ast.parseDiagnostics, []); + for (const node of ast.statements.slice(0, 2)) { + assert.equal(node.attributes.attributes[0].name.text, "type"); + assert.equal(node.attributes.attributes[0].value.text, "json"); + } + assert(ast.$tokens.some(token => token.kind === SyntaxKind.SlashToken && token.text === "/data/g")); + const invalid = extract(source + "const broken = ;"); + assert.equal(invalid.parseDiagnostics.length, 1); + assert.equal(invalid.parseDiagnostics[0].messageText, "Expression expected."); + assert.equal(invalid.parseDiagnostics[0].$pos, (source + "const broken = ;").lastIndexOf(";")); +}); + +test("TSX children, attributes, and positions survive projection", () => { + const ast = extract('const view = <>
{value}
;', ".tsx"); + assert.deepEqual(ast.parseDiagnostics, []); + const fragment = declaration(ast).initializer; + assert.equal(fragment.kind, SyntaxKind.JsxFragment); + assert.equal(fragment.children[0].openingElement.attributes.properties[0].name.text, "title"); + assert.equal(fragment.children[0].children[0].expression.text, "value"); + assert.equal(fragment.children[1].kind, SyntaxKind.JsxSelfClosingElement); + assert.equal(fragment.children[1].attributes.properties[0].expression.text, "props"); +}); + +test("the actual wrapper handles metadata, prepared files, out-of-order parsing, reset and quit", async () => { + const first = join(directory, "first.ts"); + const second = join(directory, "second.ts"); + writeFileSync(first, "const first = 1;"); + writeFileSync(second, "let second = 2;"); + const child = spawn(process.execPath, [entryPoint], { stdio: ["pipe", "pipe", "pipe"] }); + let stderr = ""; + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", (code, signal) => { + try { + assert.equal(signal, null); + assert.equal(code, 0, stderr); + accept(); + } catch (error) { + reject(error); + } + }); + }); + async function request(command) { + child.stdin.write(JSON.stringify(command) + "\n"); + const line = await lines.next(); + assert.equal(line.done, false, stderr); + return JSON.parse(line.value); + } + try { + const metadata = await request({ command: "get-metadata" }); + assert.equal(metadata.type, "metadata"); + assert.deepEqual(metadata.syntaxKinds, SyntaxKind); + assert.deepEqual(metadata.nodeFlags, NodeFlags); + assert(!("EndOfFileToken" in metadata.syntaxKinds)); + assert(!("GlobalAugmentation" in metadata.nodeFlags)); + assert.deepEqual(await request({ command: "prepare-files", filenames: [first, second] }), { type: "ok" }); + assert.equal(declaration((await request({ command: "parse", filename: first })).ast).name.text, "first"); + assert.equal(declaration((await request({ command: "parse", filename: second })).ast).name.text, "second"); + await request({ command: "prepare-files", filenames: [first, second] }); + assert.equal(declaration((await request({ command: "parse", filename: second })).ast).name.text, "second"); + assert.deepEqual(await request({ command: "reset" }), { type: "reset-done" }); + assert.equal(declaration((await request({ command: "parse", filename: first })).ast).name.text, "first"); + child.stdin.end('{"command":"quit"}\n'); + await exited; + assert.equal(stderr, ""); + } finally { + child.stdin.destroy(); + child.kill(); + await exited; + } +}); + +test("single-file invocation flushes JSON and reports missing files", () => { + const filename = join(directory, "command-line.ts"); + writeFileSync(filename, "const value = 1;"); + const options = { + encoding: "utf8", + }; + const response = JSON.parse(execFileSync(process.execPath, [entryPoint, filename], options)); + assert.equal(declaration(response.ast).name.text, "value"); + assert.throws(() => execFileSync(process.execPath, [entryPoint, resolve(directory, "missing.ts")], { + ...options, stdio: ["ignore", "pipe", "pipe"], + }), error => { + assert.equal(error.stdout, ""); + assert.match(error.stderr, /TypeScript did not return a source file/); + return true; + }); +}); diff --git a/javascript/extractor/lib/typescript/tsconfig.json b/javascript/extractor/lib/typescript/tsconfig.json index 81038684f405..b106ced15989 100644 --- a/javascript/extractor/lib/typescript/tsconfig.json +++ b/javascript/extractor/lib/typescript/tsconfig.json @@ -1,14 +1,14 @@ { "compilerOptions": { - "module": "commonjs", + "module": "nodenext", "noImplicitAny": true, "removeComments": true, "sourceMap": false, - "baseUrl": "./", - "lib": ["es2016"], - "target": "es5", + "lib": ["es2022", "esnext.disposable"], + "types": ["node"], + "target": "es2022", + "rootDir": "src", "outDir": "build", - "allowJs": true, }, "include": [ "src" diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java index 0c013f906b2b..898e2e1eac02 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java @@ -261,7 +261,7 @@ private void extractTokensAndComments( SourceLocation loc = new SourceLocation(text, start, end); String kind = getKind(token); switch (kind) { - case "EndOfFileToken": + case "EndOfFile": tokens.add(new Token(loc, Token.Type.EOF)); break; case "SingleLineCommentTrivia": @@ -927,22 +927,6 @@ private Node convertCatchClause(JsonObject node, SourceLocation loc) throws Pars return new CatchClause(loc, pattern, null, convertChild(node, "block")); } - private List convertSuperInterfaceClause(JsonArray supers) throws ParseError { - List result = new ArrayList<>(); - for (JsonElement elt : supers) { - JsonObject superType = elt.getAsJsonObject(); - ITypeExpression objectType = convertChildAsType(superType, "expression"); - if (objectType == null) continue; - List typeArguments = convertChildrenAsTypes(superType, "typeArguments"); - if (typeArguments.isEmpty()) { - result.add(objectType); - } else { - result.add(new GenericTypeExpr(getSourceLocation(superType), objectType, typeArguments)); - } - } - return result; - } - private Node convertClass(JsonObject node, String kind, SourceLocation loc) throws ParseError { Identifier id = convertChild(node, "name"); List typeParameters = convertChildrenNotNull(node, "typeParameters"); @@ -957,7 +941,7 @@ private Node convertClass(JsonObject node, String kind, SourceLocation loc) thro superClass = (Expression) convertNode(supers.get(0).getAsJsonObject()); } } else { - superInterfaces = convertSuperInterfaceClause(supers); + superInterfaces = convertChildrenAsTypes(heritageClause, "types"); } afterHead = heritageClause.get("$end").getAsInt(); } @@ -1562,7 +1546,7 @@ private Node convertInterfaceDeclaration(JsonObject node, SourceLocation loc) th for (JsonElement elt : getChildIterable(node, "heritageClauses")) { JsonObject heritageClause = elt.getAsJsonObject(); if (heritageClause.get("token").getAsInt() == syntaxKindExtends) { - superInterfaces = convertSuperInterfaceClause(heritageClause.get("types").getAsJsonArray()); + superInterfaces = convertChildrenAsTypes(heritageClause, "types"); break; } } @@ -1762,6 +1746,10 @@ private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) th Node nameNode = convertChild(node, "name"); List body; Statement b = convertChild(node, "body"); + if (hasKind(node.get("body"), "ModuleDeclaration")) { + // The inner declarations in `namespace A.B` are implicitly exported. + b = new ExportNamedDeclaration(b.getLoc(), b, new ArrayList<>(), null, null); + } if (b instanceof BlockStatement) { body = ((BlockStatement) b).getBody(); } else { @@ -1774,11 +1762,11 @@ private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) th attachSymbolInformation(decl, node); return decl; } - if (hasFlag(node, "GlobalAugmentation")) { + Identifier name = (Identifier) nameNode; + if (isGlobalAugmentation(node, loc, name)) { // Declaration of form: declare global {...} return new GlobalAugmentationDeclaration(loc, body); } - Identifier name = (Identifier) nameNode; boolean isInstantiated = false; for (Statement stmt : body) { isInstantiated = isInstantiated || isInstantiatingNamespaceMember(stmt); @@ -1787,13 +1775,18 @@ private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) th NamespaceDeclaration decl = new NamespaceDeclaration(loc, name, body, isInstantiated, hasDeclareKeyword); attachSymbolInformation(decl, node); - if (hasFlag(node, "NestedNamespace")) { - // In a nested namespace declaration `namespace A.B`, the nested namespace `B` - // is implicitly exported. - return new ExportNamedDeclaration(loc, decl, new ArrayList<>(), null, null); - } else { - return fixExports(loc, decl); + return fixExports(loc, decl); + } + + private boolean isGlobalAugmentation(JsonObject node, SourceLocation loc, Identifier name) { + if (!name.getName().equals("global")) return false; + // There is no `module` or `namespace` keyword between the modifiers and `global`. + int start = loc.getStart().getOffset(); + for (JsonElement modifier : getModifiers(node)) { + start = modifier.getAsJsonObject().get("$end").getAsInt(); } + return source.substring(start, name.getLoc().getStart().getOffset()) + .matches("(?U)(?:\\uFEFF|" + WHITESPACE_CHAR + ")*"); } private boolean isInstantiatingNamespaceMember(Statement node) { @@ -2123,10 +2116,10 @@ private Node convertPropertyDeclaration(JsonObject node, String kind, SourceLoca if (kind.equals("PropertySignature")) { flags |= DeclarationFlags.abstract_; } - if (node.get("questionToken") != null) { + if (hasKind(node.get("postfixToken"), "QuestionToken")) { flags |= DeclarationFlags.optional; } - if (node.get("exclamationToken") != null) { + if (hasKind(node.get("postfixToken"), "ExclamationToken")) { flags |= DeclarationFlags.definiteAssignmentAssertion; } if (hasModifier(node, "DeclareKeyword")) { @@ -2323,7 +2316,7 @@ private Node convertTypeAssertionExpression(JsonObject node, SourceLocation loc) private Node convertImportAttributes(JsonObject node, SourceLocation loc) throws ParseError { List properties = new ArrayList<>(); - for (INode child : convertChildren(node, "elements")) { + for (INode child : convertChildren(node, "attributes")) { properties.add((Property) child); } // Adjust location to skip over the `with` or `assert` keyword. @@ -2336,7 +2329,7 @@ private Node convertImportAttributes(JsonObject node, SourceLocation loc) throws private Node convertImportAttribute(JsonObject node, SourceLocation loc) throws ParseError { return new Property( - loc, convertChild(node, "key"), convertChild(node, "value"), "init", false, false); + loc, convertChild(node, "name"), convertChild(node, "value"), "init", false, false); } private Node convertSatisfiesExpression(JsonObject node, SourceLocation loc) throws ParseError { @@ -2371,7 +2364,7 @@ private Node convertTypeParameter(JsonObject node, SourceLocation loc) throws Pa loc, convertChild(node, "name"), convertChildAsType(node, "constraint"), - convertChildAsType(node, "default")); + convertChildAsType(node, "defaultType")); } private Node convertTypePredicate(JsonObject node, SourceLocation loc) throws ParseError { @@ -2720,6 +2713,9 @@ private String getDeclarationKind(JsonObject declarationList) { if (hasFlag(declarationList, "Using")) { return "using"; } - return declarationList.get("$declarationKind").getAsString(); + if (hasFlag(declarationList, "Let")) { + return "let"; + } + return hasFlag(declarationList, "Const") ? "const" : "var"; } } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java index 4061f1a8651b..b179eb2d1134 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java @@ -2,13 +2,14 @@ import com.google.devtools.build.runfiles.Runfiles; import com.semmle.util.process.Env; -import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -20,6 +21,7 @@ NodeJSDetectorTests.class, ES2015DetectorTests.class, TrapTests.class, + TypeScriptTests.class, ObjectRestSpreadTests.class, ClassPropertiesTests.class, FunctionSentTests.class, @@ -36,20 +38,10 @@ public static void setUp() throws Exception { Runfiles.Preloaded runfiles = Runfiles.preload(); String nodePath = runfiles.unmapped().rlocation(System.getenv("NODE_BIN")); String tsWrapperZip = runfiles.unmapped().rlocation(System.getenv("TS_WRAPPER_ZIP")); + String tsNativeZip = runfiles.unmapped().rlocation(System.getenv("TS_NATIVE_ZIP")); Path tempDir = Files.createTempDirectory("ts-wrapper"); - // extract the ts-wrapper.zip to tempDir: - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tsWrapperZip))) { - ZipEntry entry = zis.getNextEntry(); - while (entry != null) { - Path entryPath = tempDir.resolve(entry.getName()); - if (entry.isDirectory()) { - Files.createDirectories(entryPath); - } else { - Files.copy(zis, entryPath); - } - entry = zis.getNextEntry(); - } - } + extractZip(tsWrapperZip, tempDir); + extractZip(tsNativeZip, tempDir.resolve("typescript-parser-wrapper/node_modules/@typescript")); Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js"); if (!Files.exists(tsWrapper)) { throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper); @@ -60,4 +52,31 @@ public static void setUp() throws Exception { Env.systemEnv().pushEnvironmentContext(envUpdate); } + + private static void extractZip(String filename, Path destination) throws IOException { + // Unix modes are in the central directory, which ZipInputStream does not expose. + try (ZipFile zip = new ZipFile(Path.of(filename).toFile())) { + var entries = zip.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + Path entryPath = destination.resolve(entry.getName()).normalize(); + if (!entryPath.startsWith(destination)) { + throw new IOException("ZIP entry is outside its destination: " + entry.getName()); + } + if (entry.isDirectory()) { + Files.createDirectories(entryPath); + } else { + Files.createDirectories(entryPath.getParent()); + try (InputStream input = zip.getInputStream(entry)) { + Files.copy(input, entryPath); + } + if ((entry.getUnixMode() & 0111) != 0 + && entryPath.getFileSystem().supportedFileAttributeViews().contains("posix") + && !entryPath.toFile().setExecutable(true, false)) { + throw new IOException("Could not make ZIP entry executable: " + entryPath); + } + } + } + } + } } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel index 2d6a5d01c660..faecec7cb010 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel +++ b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel @@ -5,15 +5,18 @@ java_test( srcs = glob(["**/*.java"]), data = [ "//javascript/extractor/lib/typescript", + "//javascript/extractor/lib/typescript:native", "//javascript/extractor/parser-tests", "//javascript/extractor/tests", "@nodejs//:node_bin", ], env = { "NODE_BIN": "$(rlocationpath @nodejs//:node_bin)", + "TS_NATIVE_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript:native)", "TS_WRAPPER_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript)", }, test_class = "com.semmle.js.extractor.test.AllTests", + runtime_deps = ["@semmle_code//resources/lib/java:commons-io"], # To use `replaceExpectedOutput` you need to uncomment the following line # (to be allowed to override the .trap files on disk) # tags = ["no-sandbox"], diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java new file mode 100644 index 000000000000..1bc36f2fcc61 --- /dev/null +++ b/javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java @@ -0,0 +1,99 @@ +package com.semmle.js.extractor.test; + +import com.semmle.js.ast.ClassDeclaration; +import com.semmle.js.ast.DeclarationFlags; +import com.semmle.js.ast.ExportNamedDeclaration; +import com.semmle.js.ast.Program; +import com.semmle.js.ast.Token; +import com.semmle.js.ast.VariableDeclaration; +import com.semmle.js.extractor.ExtractionMetrics; +import com.semmle.js.parser.JSParser.Result; +import com.semmle.ts.ast.ExternalModuleDeclaration; +import com.semmle.ts.ast.GlobalAugmentationDeclaration; +import com.semmle.ts.ast.InterfaceDeclaration; +import com.semmle.ts.ast.NamespaceDeclaration; +import com.semmle.ts.extractor.TypeScriptParser; +import java.io.File; +import java.nio.file.Files; +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class TypeScriptTests { + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private final TypeScriptParser parser = new TypeScriptParser(); + + @After + public void tearDown() { + parser.killProcess(); + } + + private Program parse(String source) throws Exception { + File file = folder.newFile("input.ts"); + Files.writeString(file.toPath(), source); + Result result = parser.parse(file, source, new ExtractionMetrics()); + Assert.assertTrue(result.getErrors().toString(), result.getErrors().isEmpty()); + Assert.assertEquals(Token.Type.EOF, result.getTokens().get(result.getTokens().size() - 1).getType()); + return (Program) result.getAST(); + } + + @Test + public void namespaceClassifications() throws Exception { + Program program = + parse( + "declare /* a */ global { interface Window {} }\n" + + "declare\u00a0/* b */\ufeffglobal {}\n" + + "namespace global {}\n" + + "module global {}\n" + + "namespace A.B.C {}\n" + + "namespace X { namespace Y {} }\n" + + "declare module \"external\" {}"); + Assert.assertTrue(program.getBody().get(0) instanceof GlobalAugmentationDeclaration); + Assert.assertTrue(program.getBody().get(1) instanceof GlobalAugmentationDeclaration); + Assert.assertTrue(program.getBody().get(2) instanceof NamespaceDeclaration); + Assert.assertTrue(program.getBody().get(3) instanceof NamespaceDeclaration); + NamespaceDeclaration a = (NamespaceDeclaration) program.getBody().get(4); + ExportNamedDeclaration exportedB = (ExportNamedDeclaration) a.getBody().get(0); + NamespaceDeclaration b = (NamespaceDeclaration) exportedB.getDeclaration(); + ExportNamedDeclaration exportedC = (ExportNamedDeclaration) b.getBody().get(0); + Assert.assertEquals("B", b.getName().getName()); + Assert.assertEquals( + "C", ((NamespaceDeclaration) exportedC.getDeclaration()).getName().getName()); + NamespaceDeclaration x = (NamespaceDeclaration) program.getBody().get(5); + Assert.assertTrue(x.getBody().get(0) instanceof NamespaceDeclaration); + Assert.assertTrue(program.getBody().get(6) instanceof ExternalModuleDeclaration); + } + + @Test + public void memberPostfixTokensAndTypeDefaults() throws Exception { + Program program = + parse( + "interface I { p?: string; m?(): void; required: T }\n" + + "class C { p!: string; optional?: number; required: number }"); + InterfaceDeclaration iface = (InterfaceDeclaration) program.getBody().get(0); + Assert.assertEquals("number", iface.getTypeParameters().get(0).getDefault().getLoc().getSource()); + Assert.assertTrue(DeclarationFlags.isOptional(iface.getBody().get(0).getFlags())); + Assert.assertTrue(DeclarationFlags.isAbstract(iface.getBody().get(1).getFlags())); + Assert.assertFalse(DeclarationFlags.isOptional(iface.getBody().get(2).getFlags())); + ClassDeclaration cls = (ClassDeclaration) program.getBody().get(1); + var members = cls.getClassDef().getBody().getBody(); + Assert.assertTrue(DeclarationFlags.hasDefiniteAssignmentAssertion(members.get(0).getFlags())); + Assert.assertFalse(DeclarationFlags.isOptional(members.get(0).getFlags())); + Assert.assertTrue(DeclarationFlags.isOptional(members.get(1).getFlags())); + Assert.assertFalse(DeclarationFlags.hasDefiniteAssignmentAssertion(members.get(1).getFlags())); + Assert.assertFalse(DeclarationFlags.isOptional(members.get(2).getFlags())); + } + + @Test + public void variableDeclarationKinds() throws Exception { + Program program = + parse("var a; let b; const c = 1; using d = resource(); await using e = resource();"); + String[] kinds = {"var", "let", "const", "using", "using"}; + for (int i = 0; i < kinds.length; ++i) { + Assert.assertEquals(kinds[i], ((VariableDeclaration) program.getBody().get(i)).getKind()); + } + } +} diff --git a/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap b/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap index 193a2afeb185..91161d974584 100644 --- a/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap +++ b/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap @@ -1311,837 +1311,1089 @@ properties(#20503,#20501,0,0,"type: ""json""") locations_default(#20504,#10000,1,24,1,35) hasLocation(#20503,#20504) #20505=* -stmts(#20505,27,#20001,1,"import ... son"" };") -hasLocation(#20505,#20009) -stmt_containers(#20505,#20001) +exprs(#20505,0,#20503,0,"type") +hasLocation(#20505,#20063) +enclosing_stmt(#20505,#20497) +expr_containers(#20505,#20001) +literals("type","type",#20505) #20506=* -exprs(#20506,4,#20505,-1,"""module""") -hasLocation(#20506,#20083) -enclosing_stmt(#20506,#20505) +exprs(#20506,4,#20503,1,"""json""") +hasLocation(#20506,#20067) +enclosing_stmt(#20506,#20497) expr_containers(#20506,#20001) -literals("module","""module""",#20506) +literals("json","""json""",#20506) #20507=* -regexpterm(#20507,14,#20506,0,"module") -#20508=@"loc,{#10000},2,22,2,27" -locations_default(#20508,#10000,2,22,2,27) +regexpterm(#20507,14,#20506,0,"json") +#20508=@"loc,{#10000},1,31,1,34" +locations_default(#20508,#10000,1,31,1,34) hasLocation(#20507,#20508) -regexp_const_value(#20507,"module") +regexp_const_value(#20507,"json") #20509=* -exprs(#20509,8,#20505,-10,"{ type: ""json"" }") -#20510=@"loc,{#10000},2,35,2,50" -locations_default(#20510,#10000,2,35,2,50) -hasLocation(#20509,#20510) -enclosing_stmt(#20509,#20505) -expr_containers(#20509,#20001) +stmts(#20509,27,#20001,1,"import ... son"" };") +hasLocation(#20509,#20009) +stmt_containers(#20509,#20001) +#20510=* +exprs(#20510,4,#20509,-1,"""module""") +hasLocation(#20510,#20083) +enclosing_stmt(#20510,#20509) +expr_containers(#20510,#20001) +literals("module","""module""",#20510) #20511=* -properties(#20511,#20509,0,0,"type: ""json""") -#20512=@"loc,{#10000},2,37,2,48" -locations_default(#20512,#10000,2,37,2,48) +regexpterm(#20511,14,#20510,0,"module") +#20512=@"loc,{#10000},2,22,2,27" +locations_default(#20512,#10000,2,22,2,27) hasLocation(#20511,#20512) +regexp_const_value(#20511,"module") #20513=* -exprs(#20513,85,#20505,0,"* as v1") -#20514=@"loc,{#10000},2,8,2,14" -locations_default(#20514,#10000,2,8,2,14) +exprs(#20513,8,#20509,-10,"{ type: ""json"" }") +#20514=@"loc,{#10000},2,35,2,50" +locations_default(#20514,#10000,2,35,2,50) hasLocation(#20513,#20514) -enclosing_stmt(#20513,#20505) +enclosing_stmt(#20513,#20509) expr_containers(#20513,#20001) #20515=* -exprs(#20515,78,#20513,1,"v1") -hasLocation(#20515,#20079) -enclosing_stmt(#20515,#20505) -expr_containers(#20515,#20001) -literals("v1","v1",#20515) -decl(#20515,#20486) -typedecl(#20515,#20489) -namespacedecl(#20515,#20492) -#20516=* -stmts(#20516,27,#20001,2,"import ... son"" };") -hasLocation(#20516,#20011) -stmt_containers(#20516,#20001) +properties(#20515,#20513,0,0,"type: ""json""") +#20516=@"loc,{#10000},2,37,2,48" +locations_default(#20516,#10000,2,37,2,48) +hasLocation(#20515,#20516) #20517=* -exprs(#20517,4,#20516,-1,"""module""") -hasLocation(#20517,#20109) -enclosing_stmt(#20517,#20516) +exprs(#20517,0,#20515,0,"type") +hasLocation(#20517,#20089) +enclosing_stmt(#20517,#20509) expr_containers(#20517,#20001) -literals("module","""module""",#20517) +literals("type","type",#20517) #20518=* -regexpterm(#20518,14,#20517,0,"module") -#20519=@"loc,{#10000},3,21,3,26" -locations_default(#20519,#10000,3,21,3,26) -hasLocation(#20518,#20519) -regexp_const_value(#20518,"module") -#20520=* -exprs(#20520,8,#20516,-10,"{ type: ""json"" }") -#20521=@"loc,{#10000},3,34,3,49" -locations_default(#20521,#10000,3,34,3,49) -hasLocation(#20520,#20521) -enclosing_stmt(#20520,#20516) -expr_containers(#20520,#20001) -#20522=* -properties(#20522,#20520,0,0,"type: ""json""") -#20523=@"loc,{#10000},3,36,3,47" -locations_default(#20523,#10000,3,36,3,47) -hasLocation(#20522,#20523) +exprs(#20518,4,#20515,1,"""json""") +hasLocation(#20518,#20093) +enclosing_stmt(#20518,#20509) +expr_containers(#20518,#20001) +literals("json","""json""",#20518) +#20519=* +regexpterm(#20519,14,#20518,0,"json") +#20520=@"loc,{#10000},2,44,2,47" +locations_default(#20520,#10000,2,44,2,47) +hasLocation(#20519,#20520) +regexp_const_value(#20519,"json") +#20521=* +exprs(#20521,85,#20509,0,"* as v1") +#20522=@"loc,{#10000},2,8,2,14" +locations_default(#20522,#10000,2,8,2,14) +hasLocation(#20521,#20522) +enclosing_stmt(#20521,#20509) +expr_containers(#20521,#20001) +#20523=* +exprs(#20523,78,#20521,1,"v1") +hasLocation(#20523,#20079) +enclosing_stmt(#20523,#20509) +expr_containers(#20523,#20001) +literals("v1","v1",#20523) +decl(#20523,#20486) +typedecl(#20523,#20489) +namespacedecl(#20523,#20492) #20524=* -exprs(#20524,83,#20516,0,"v2") -hasLocation(#20524,#20103) -enclosing_stmt(#20524,#20516) -expr_containers(#20524,#20001) +stmts(#20524,27,#20001,2,"import ... son"" };") +hasLocation(#20524,#20011) +stmt_containers(#20524,#20001) #20525=* -exprs(#20525,0,#20524,0,"v2") -hasLocation(#20525,#20103) -enclosing_stmt(#20525,#20516) +exprs(#20525,4,#20524,-1,"""module""") +hasLocation(#20525,#20109) +enclosing_stmt(#20525,#20524) expr_containers(#20525,#20001) -literals("v2","v2",#20525) +literals("module","""module""",#20525) #20526=* -exprs(#20526,78,#20524,1,"v2") -hasLocation(#20526,#20103) -enclosing_stmt(#20526,#20516) -expr_containers(#20526,#20001) -literals("v2","v2",#20526) -decl(#20526,#20487) -typedecl(#20526,#20490) -namespacedecl(#20526,#20493) -#20527=* -stmts(#20527,27,#20001,3,"import ... son"" };") -hasLocation(#20527,#20013) -stmt_containers(#20527,#20001) +regexpterm(#20526,14,#20525,0,"module") +#20527=@"loc,{#10000},3,21,3,26" +locations_default(#20527,#10000,3,21,3,26) +hasLocation(#20526,#20527) +regexp_const_value(#20526,"module") #20528=* -exprs(#20528,4,#20527,-1,"""module""") -hasLocation(#20528,#20131) -enclosing_stmt(#20528,#20527) +exprs(#20528,8,#20524,-10,"{ type: ""json"" }") +#20529=@"loc,{#10000},3,34,3,49" +locations_default(#20529,#10000,3,34,3,49) +hasLocation(#20528,#20529) +enclosing_stmt(#20528,#20524) expr_containers(#20528,#20001) -literals("module","""module""",#20528) -#20529=* -regexpterm(#20529,14,#20528,0,"module") -#20530=@"loc,{#10000},4,17,4,22" -locations_default(#20530,#10000,4,17,4,22) -hasLocation(#20529,#20530) -regexp_const_value(#20529,"module") -#20531=* -exprs(#20531,8,#20527,-10,"{ type: ""json"" }") -#20532=@"loc,{#10000},4,30,4,45" -locations_default(#20532,#10000,4,30,4,45) -hasLocation(#20531,#20532) -enclosing_stmt(#20531,#20527) -expr_containers(#20531,#20001) +#20530=* +properties(#20530,#20528,0,0,"type: ""json""") +#20531=@"loc,{#10000},3,36,3,47" +locations_default(#20531,#10000,3,36,3,47) +hasLocation(#20530,#20531) +#20532=* +exprs(#20532,0,#20530,0,"type") +hasLocation(#20532,#20115) +enclosing_stmt(#20532,#20524) +expr_containers(#20532,#20001) +literals("type","type",#20532) #20533=* -properties(#20533,#20531,0,0,"type: ""json""") -#20534=@"loc,{#10000},4,32,4,43" -locations_default(#20534,#10000,4,32,4,43) -hasLocation(#20533,#20534) -#20535=* -exprs(#20535,84,#20527,0,"v3") -hasLocation(#20535,#20127) -enclosing_stmt(#20535,#20527) -expr_containers(#20535,#20001) +exprs(#20533,4,#20530,1,"""json""") +hasLocation(#20533,#20119) +enclosing_stmt(#20533,#20524) +expr_containers(#20533,#20001) +literals("json","""json""",#20533) +#20534=* +regexpterm(#20534,14,#20533,0,"json") +#20535=@"loc,{#10000},3,43,3,46" +locations_default(#20535,#10000,3,43,3,46) +hasLocation(#20534,#20535) +regexp_const_value(#20534,"json") #20536=* -exprs(#20536,78,#20535,1,"v3") -hasLocation(#20536,#20127) -enclosing_stmt(#20536,#20527) +exprs(#20536,83,#20524,0,"v2") +hasLocation(#20536,#20103) +enclosing_stmt(#20536,#20524) expr_containers(#20536,#20001) -literals("v3","v3",#20536) -decl(#20536,#20488) -typedecl(#20536,#20491) -namespacedecl(#20536,#20494) #20537=* -stmts(#20537,30,#20001,4,"export ... son"" };") -hasLocation(#20537,#20017) -stmt_containers(#20537,#20001) +exprs(#20537,0,#20536,0,"v2") +hasLocation(#20537,#20103) +enclosing_stmt(#20537,#20524) +expr_containers(#20537,#20001) +literals("v2","v2",#20537) #20538=* -exprs(#20538,4,#20537,-2,"""module""") -hasLocation(#20538,#20157) -enclosing_stmt(#20538,#20537) +exprs(#20538,78,#20536,1,"v2") +hasLocation(#20538,#20103) +enclosing_stmt(#20538,#20524) expr_containers(#20538,#20001) -literals("module","""module""",#20538) +literals("v2","v2",#20538) +decl(#20538,#20487) +typedecl(#20538,#20490) +namespacedecl(#20538,#20493) #20539=* -regexpterm(#20539,14,#20538,0,"module") -#20540=@"loc,{#10000},6,21,6,26" -locations_default(#20540,#10000,6,21,6,26) -hasLocation(#20539,#20540) -regexp_const_value(#20539,"module") +stmts(#20539,27,#20001,3,"import ... son"" };") +hasLocation(#20539,#20013) +stmt_containers(#20539,#20001) +#20540=* +exprs(#20540,4,#20539,-1,"""module""") +hasLocation(#20540,#20131) +enclosing_stmt(#20540,#20539) +expr_containers(#20540,#20001) +literals("module","""module""",#20540) #20541=* -exprs(#20541,8,#20537,-10,"{ type: ""json"" }") -#20542=@"loc,{#10000},6,34,6,49" -locations_default(#20542,#10000,6,34,6,49) +regexpterm(#20541,14,#20540,0,"module") +#20542=@"loc,{#10000},4,17,4,22" +locations_default(#20542,#10000,4,17,4,22) hasLocation(#20541,#20542) -enclosing_stmt(#20541,#20537) -expr_containers(#20541,#20001) +regexp_const_value(#20541,"module") #20543=* -properties(#20543,#20541,0,0,"type: ""json""") -#20544=@"loc,{#10000},6,36,6,47" -locations_default(#20544,#10000,6,36,6,47) +exprs(#20543,8,#20539,-10,"{ type: ""json"" }") +#20544=@"loc,{#10000},4,30,4,45" +locations_default(#20544,#10000,4,30,4,45) hasLocation(#20543,#20544) +enclosing_stmt(#20543,#20539) +expr_containers(#20543,#20001) #20545=* -exprs(#20545,86,#20537,0,"v4") -hasLocation(#20545,#20151) -enclosing_stmt(#20545,#20537) -expr_containers(#20545,#20001) -#20546=* -exprs(#20546,0,#20545,0,"v4") -hasLocation(#20546,#20151) -enclosing_stmt(#20546,#20537) -expr_containers(#20546,#20001) -literals("v4","v4",#20546) +properties(#20545,#20543,0,0,"type: ""json""") +#20546=@"loc,{#10000},4,32,4,43" +locations_default(#20546,#10000,4,32,4,43) +hasLocation(#20545,#20546) #20547=* -exprs(#20547,0,#20545,1,"v4") -hasLocation(#20547,#20151) -enclosing_stmt(#20547,#20537) +exprs(#20547,0,#20545,0,"type") +hasLocation(#20547,#20137) +enclosing_stmt(#20547,#20539) expr_containers(#20547,#20001) -literals("v4","v4",#20547) +literals("type","type",#20547) #20548=* -stmts(#20548,28,#20001,5,"export ... son"" };") -hasLocation(#20548,#20019) -stmt_containers(#20548,#20001) +exprs(#20548,4,#20545,1,"""json""") +hasLocation(#20548,#20141) +enclosing_stmt(#20548,#20539) +expr_containers(#20548,#20001) +literals("json","""json""",#20548) #20549=* -exprs(#20549,4,#20548,0,"""module""") -hasLocation(#20549,#20179) -enclosing_stmt(#20549,#20548) -expr_containers(#20549,#20001) -literals("module","""module""",#20549) -#20550=* -regexpterm(#20550,14,#20549,0,"module") -#20551=@"loc,{#10000},7,16,7,21" -locations_default(#20551,#10000,7,16,7,21) -hasLocation(#20550,#20551) -regexp_const_value(#20550,"module") +regexpterm(#20549,14,#20548,0,"json") +#20550=@"loc,{#10000},4,39,4,42" +locations_default(#20550,#10000,4,39,4,42) +hasLocation(#20549,#20550) +regexp_const_value(#20549,"json") +#20551=* +exprs(#20551,84,#20539,0,"v3") +hasLocation(#20551,#20127) +enclosing_stmt(#20551,#20539) +expr_containers(#20551,#20001) #20552=* -exprs(#20552,8,#20548,-10,"{ type: ""json"" }") -#20553=@"loc,{#10000},7,29,7,44" -locations_default(#20553,#10000,7,29,7,44) -hasLocation(#20552,#20553) -enclosing_stmt(#20552,#20548) +exprs(#20552,78,#20551,1,"v3") +hasLocation(#20552,#20127) +enclosing_stmt(#20552,#20539) expr_containers(#20552,#20001) +literals("v3","v3",#20552) +decl(#20552,#20488) +typedecl(#20552,#20491) +namespacedecl(#20552,#20494) +#20553=* +stmts(#20553,30,#20001,4,"export ... son"" };") +hasLocation(#20553,#20017) +stmt_containers(#20553,#20001) #20554=* -properties(#20554,#20552,0,0,"type: ""json""") -#20555=@"loc,{#10000},7,31,7,42" -locations_default(#20555,#10000,7,31,7,42) -hasLocation(#20554,#20555) -#20556=* -stmts(#20556,30,#20001,6,"export ... son"" };") -hasLocation(#20556,#20021) -stmt_containers(#20556,#20001) +exprs(#20554,4,#20553,-2,"""module""") +hasLocation(#20554,#20157) +enclosing_stmt(#20554,#20553) +expr_containers(#20554,#20001) +literals("module","""module""",#20554) +#20555=* +regexpterm(#20555,14,#20554,0,"module") +#20556=@"loc,{#10000},6,21,6,26" +locations_default(#20556,#10000,6,21,6,26) +hasLocation(#20555,#20556) +regexp_const_value(#20555,"module") #20557=* -exprs(#20557,4,#20556,-2,"""module""") -hasLocation(#20557,#20205) -enclosing_stmt(#20557,#20556) +exprs(#20557,8,#20553,-10,"{ type: ""json"" }") +#20558=@"loc,{#10000},6,34,6,49" +locations_default(#20558,#10000,6,34,6,49) +hasLocation(#20557,#20558) +enclosing_stmt(#20557,#20553) expr_containers(#20557,#20001) -literals("module","""module""",#20557) -#20558=* -regexpterm(#20558,14,#20557,0,"module") -#20559=@"loc,{#10000},8,22,8,27" -locations_default(#20559,#10000,8,22,8,27) -hasLocation(#20558,#20559) -regexp_const_value(#20558,"module") -#20560=* -exprs(#20560,8,#20556,-10,"{ type: ""json"" }") -#20561=@"loc,{#10000},8,35,8,50" -locations_default(#20561,#10000,8,35,8,50) -hasLocation(#20560,#20561) -enclosing_stmt(#20560,#20556) -expr_containers(#20560,#20001) +#20559=* +properties(#20559,#20557,0,0,"type: ""json""") +#20560=@"loc,{#10000},6,36,6,47" +locations_default(#20560,#10000,6,36,6,47) +hasLocation(#20559,#20560) +#20561=* +exprs(#20561,0,#20559,0,"type") +hasLocation(#20561,#20163) +enclosing_stmt(#20561,#20553) +expr_containers(#20561,#20001) +literals("type","type",#20561) #20562=* -properties(#20562,#20560,0,0,"type: ""json""") -#20563=@"loc,{#10000},8,37,8,48" -locations_default(#20563,#10000,8,37,8,48) -hasLocation(#20562,#20563) -#20564=* -exprs(#20564,96,#20556,0,"* as v5") -#20565=@"loc,{#10000},8,8,8,14" -locations_default(#20565,#10000,8,8,8,14) -hasLocation(#20564,#20565) -enclosing_stmt(#20564,#20556) -expr_containers(#20564,#20001) +exprs(#20562,4,#20559,1,"""json""") +hasLocation(#20562,#20167) +enclosing_stmt(#20562,#20553) +expr_containers(#20562,#20001) +literals("json","""json""",#20562) +#20563=* +regexpterm(#20563,14,#20562,0,"json") +#20564=@"loc,{#10000},6,43,6,46" +locations_default(#20564,#10000,6,43,6,46) +hasLocation(#20563,#20564) +regexp_const_value(#20563,"json") +#20565=* +exprs(#20565,86,#20553,0,"v4") +hasLocation(#20565,#20151) +enclosing_stmt(#20565,#20553) +expr_containers(#20565,#20001) #20566=* -exprs(#20566,0,#20564,1,"v5") -hasLocation(#20566,#20201) -enclosing_stmt(#20566,#20556) +exprs(#20566,0,#20565,0,"v4") +hasLocation(#20566,#20151) +enclosing_stmt(#20566,#20553) expr_containers(#20566,#20001) -literals("v5","v5",#20566) +literals("v4","v4",#20566) #20567=* -stmts(#20567,22,#20001,7,"const v ... "" } });") -hasLocation(#20567,#20025) -stmt_containers(#20567,#20001) +exprs(#20567,0,#20565,1,"v4") +hasLocation(#20567,#20151) +enclosing_stmt(#20567,#20553) +expr_containers(#20567,#20001) +literals("v4","v4",#20567) #20568=* -exprs(#20568,64,#20567,0,"v6 = im ... n"" } })") -#20569=@"loc,{#10000},10,7,10,57" -locations_default(#20569,#10000,10,7,10,57) -hasLocation(#20568,#20569) -enclosing_stmt(#20568,#20567) -expr_containers(#20568,#20001) +stmts(#20568,28,#20001,5,"export ... son"" };") +hasLocation(#20568,#20019) +stmt_containers(#20568,#20001) +#20569=* +exprs(#20569,4,#20568,0,"""module""") +hasLocation(#20569,#20179) +enclosing_stmt(#20569,#20568) +expr_containers(#20569,#20001) +literals("module","""module""",#20569) #20570=* -exprs(#20570,78,#20568,0,"v6") -hasLocation(#20570,#20223) -enclosing_stmt(#20570,#20567) -expr_containers(#20570,#20001) -literals("v6","v6",#20570) -decl(#20570,#20495) -#20571=* -exprs(#20571,99,#20568,1,"import( ... n"" } })") -#20572=@"loc,{#10000},10,12,10,57" -locations_default(#20572,#10000,10,12,10,57) -hasLocation(#20571,#20572) -enclosing_stmt(#20571,#20567) -expr_containers(#20571,#20001) -#20573=* -exprs(#20573,4,#20571,0,"""module""") -hasLocation(#20573,#20231) -enclosing_stmt(#20573,#20567) -expr_containers(#20573,#20001) -literals("module","""module""",#20573) +regexpterm(#20570,14,#20569,0,"module") +#20571=@"loc,{#10000},7,16,7,21" +locations_default(#20571,#10000,7,16,7,21) +hasLocation(#20570,#20571) +regexp_const_value(#20570,"module") +#20572=* +exprs(#20572,8,#20568,-10,"{ type: ""json"" }") +#20573=@"loc,{#10000},7,29,7,44" +locations_default(#20573,#10000,7,29,7,44) +hasLocation(#20572,#20573) +enclosing_stmt(#20572,#20568) +expr_containers(#20572,#20001) #20574=* -regexpterm(#20574,14,#20573,0,"module") -#20575=@"loc,{#10000},10,20,10,25" -locations_default(#20575,#10000,10,20,10,25) +properties(#20574,#20572,0,0,"type: ""json""") +#20575=@"loc,{#10000},7,31,7,42" +locations_default(#20575,#10000,7,31,7,42) hasLocation(#20574,#20575) -regexp_const_value(#20574,"module") #20576=* -exprs(#20576,8,#20571,1,"{ ""with ... on"" } }") -#20577=@"loc,{#10000},10,29,10,56" -locations_default(#20577,#10000,10,29,10,56) -hasLocation(#20576,#20577) -enclosing_stmt(#20576,#20567) +exprs(#20576,0,#20574,0,"type") +hasLocation(#20576,#20185) +enclosing_stmt(#20576,#20568) expr_containers(#20576,#20001) +literals("type","type",#20576) +#20577=* +exprs(#20577,4,#20574,1,"""json""") +hasLocation(#20577,#20189) +enclosing_stmt(#20577,#20568) +expr_containers(#20577,#20001) +literals("json","""json""",#20577) #20578=* -properties(#20578,#20576,0,0,"""with"": ... json"" }") -#20579=@"loc,{#10000},10,31,10,54" -locations_default(#20579,#10000,10,31,10,54) +regexpterm(#20578,14,#20577,0,"json") +#20579=@"loc,{#10000},7,38,7,41" +locations_default(#20579,#10000,7,38,7,41) hasLocation(#20578,#20579) +regexp_const_value(#20578,"json") #20580=* -exprs(#20580,4,#20578,0,"""with""") -hasLocation(#20580,#20237) -enclosing_stmt(#20580,#20567) -expr_containers(#20580,#20001) -literals("with","""with""",#20580) +stmts(#20580,30,#20001,6,"export ... son"" };") +hasLocation(#20580,#20021) +stmt_containers(#20580,#20001) #20581=* -regexpterm(#20581,14,#20580,0,"with") -#20582=@"loc,{#10000},10,32,10,35" -locations_default(#20582,#10000,10,32,10,35) -hasLocation(#20581,#20582) -regexp_const_value(#20581,"with") -#20583=* -exprs(#20583,8,#20578,1,"{ type: ""json"" }") -#20584=@"loc,{#10000},10,39,10,54" -locations_default(#20584,#10000,10,39,10,54) -hasLocation(#20583,#20584) -enclosing_stmt(#20583,#20567) -expr_containers(#20583,#20001) -#20585=* -properties(#20585,#20583,0,0,"type: ""json""") -#20586=@"loc,{#10000},10,41,10,52" -locations_default(#20586,#10000,10,41,10,52) -hasLocation(#20585,#20586) -#20587=* -exprs(#20587,0,#20585,0,"type") -hasLocation(#20587,#20243) -enclosing_stmt(#20587,#20567) -expr_containers(#20587,#20001) -literals("type","type",#20587) +exprs(#20581,4,#20580,-2,"""module""") +hasLocation(#20581,#20205) +enclosing_stmt(#20581,#20580) +expr_containers(#20581,#20001) +literals("module","""module""",#20581) +#20582=* +regexpterm(#20582,14,#20581,0,"module") +#20583=@"loc,{#10000},8,22,8,27" +locations_default(#20583,#10000,8,22,8,27) +hasLocation(#20582,#20583) +regexp_const_value(#20582,"module") +#20584=* +exprs(#20584,8,#20580,-10,"{ type: ""json"" }") +#20585=@"loc,{#10000},8,35,8,50" +locations_default(#20585,#10000,8,35,8,50) +hasLocation(#20584,#20585) +enclosing_stmt(#20584,#20580) +expr_containers(#20584,#20001) +#20586=* +properties(#20586,#20584,0,0,"type: ""json""") +#20587=@"loc,{#10000},8,37,8,48" +locations_default(#20587,#10000,8,37,8,48) +hasLocation(#20586,#20587) #20588=* -exprs(#20588,4,#20585,1,"""json""") -hasLocation(#20588,#20247) -enclosing_stmt(#20588,#20567) +exprs(#20588,0,#20586,0,"type") +hasLocation(#20588,#20211) +enclosing_stmt(#20588,#20580) expr_containers(#20588,#20001) -literals("json","""json""",#20588) +literals("type","type",#20588) #20589=* -regexpterm(#20589,14,#20588,0,"json") -#20590=@"loc,{#10000},10,48,10,51" -locations_default(#20590,#10000,10,48,10,51) -hasLocation(#20589,#20590) -regexp_const_value(#20589,"json") -#20591=* -stmts(#20591,27,#20001,8,"import ""module"";") -#20592=@"loc,{#10000},12,1,12,16" -locations_default(#20592,#10000,12,1,12,16) -hasLocation(#20591,#20592) -stmt_containers(#20591,#20001) -#20593=* -exprs(#20593,4,#20591,-1,"""module""") -hasLocation(#20593,#20259) -enclosing_stmt(#20593,#20591) -expr_containers(#20593,#20001) -literals("module","""module""",#20593) +exprs(#20589,4,#20586,1,"""json""") +hasLocation(#20589,#20215) +enclosing_stmt(#20589,#20580) +expr_containers(#20589,#20001) +literals("json","""json""",#20589) +#20590=* +regexpterm(#20590,14,#20589,0,"json") +#20591=@"loc,{#10000},8,44,8,47" +locations_default(#20591,#10000,8,44,8,47) +hasLocation(#20590,#20591) +regexp_const_value(#20590,"json") +#20592=* +exprs(#20592,96,#20580,0,"* as v5") +#20593=@"loc,{#10000},8,8,8,14" +locations_default(#20593,#10000,8,8,8,14) +hasLocation(#20592,#20593) +enclosing_stmt(#20592,#20580) +expr_containers(#20592,#20001) #20594=* -regexpterm(#20594,14,#20593,0,"module") -#20595=@"loc,{#10000},12,9,12,14" -locations_default(#20595,#10000,12,9,12,14) -hasLocation(#20594,#20595) -regexp_const_value(#20594,"module") +exprs(#20594,0,#20592,1,"v5") +hasLocation(#20594,#20201) +enclosing_stmt(#20594,#20580) +expr_containers(#20594,#20001) +literals("v5","v5",#20594) +#20595=* +stmts(#20595,22,#20001,7,"const v ... "" } });") +hasLocation(#20595,#20025) +stmt_containers(#20595,#20001) #20596=* -stmts(#20596,2,#20001,9,"assert( ... on"" });") -#20597=@"loc,{#10000},13,1,13,25" -locations_default(#20597,#10000,13,1,13,25) +exprs(#20596,64,#20595,0,"v6 = im ... n"" } })") +#20597=@"loc,{#10000},10,7,10,57" +locations_default(#20597,#10000,10,7,10,57) hasLocation(#20596,#20597) -stmt_containers(#20596,#20001) +enclosing_stmt(#20596,#20595) +expr_containers(#20596,#20001) #20598=* -exprs(#20598,13,#20596,0,"assert( ... son"" })") -#20599=@"loc,{#10000},13,1,13,24" -locations_default(#20599,#10000,13,1,13,24) -hasLocation(#20598,#20599) -enclosing_stmt(#20598,#20596) +exprs(#20598,78,#20596,0,"v6") +hasLocation(#20598,#20223) +enclosing_stmt(#20598,#20595) expr_containers(#20598,#20001) -#20600=* -exprs(#20600,79,#20598,-1,"assert") -hasLocation(#20600,#20263) -enclosing_stmt(#20600,#20596) -expr_containers(#20600,#20001) -literals("assert","assert",#20600) -#20601=@"var;{assert};{#20000}" -variables(#20601,"assert",#20000) -bind(#20600,#20601) +literals("v6","v6",#20598) +decl(#20598,#20495) +#20599=* +exprs(#20599,99,#20596,1,"import( ... n"" } })") +#20600=@"loc,{#10000},10,12,10,57" +locations_default(#20600,#10000,10,12,10,57) +hasLocation(#20599,#20600) +enclosing_stmt(#20599,#20595) +expr_containers(#20599,#20001) +#20601=* +exprs(#20601,4,#20599,0,"""module""") +hasLocation(#20601,#20231) +enclosing_stmt(#20601,#20595) +expr_containers(#20601,#20001) +literals("module","""module""",#20601) #20602=* -exprs(#20602,8,#20598,0,"{ type: ""json"" }") -#20603=@"loc,{#10000},13,8,13,23" -locations_default(#20603,#10000,13,8,13,23) +regexpterm(#20602,14,#20601,0,"module") +#20603=@"loc,{#10000},10,20,10,25" +locations_default(#20603,#10000,10,20,10,25) hasLocation(#20602,#20603) -enclosing_stmt(#20602,#20596) -expr_containers(#20602,#20001) +regexp_const_value(#20602,"module") #20604=* -properties(#20604,#20602,0,0,"type: ""json""") -#20605=@"loc,{#10000},13,10,13,21" -locations_default(#20605,#10000,13,10,13,21) +exprs(#20604,8,#20599,1,"{ ""with ... on"" } }") +#20605=@"loc,{#10000},10,29,10,56" +locations_default(#20605,#10000,10,29,10,56) hasLocation(#20604,#20605) +enclosing_stmt(#20604,#20595) +expr_containers(#20604,#20001) #20606=* -exprs(#20606,0,#20604,0,"type") -hasLocation(#20606,#20269) -enclosing_stmt(#20606,#20596) -expr_containers(#20606,#20001) -literals("type","type",#20606) -#20607=* -exprs(#20607,4,#20604,1,"""json""") -hasLocation(#20607,#20273) -enclosing_stmt(#20607,#20596) -expr_containers(#20607,#20001) -literals("json","""json""",#20607) +properties(#20606,#20604,0,0,"""with"": ... json"" }") +#20607=@"loc,{#10000},10,31,10,54" +locations_default(#20607,#10000,10,31,10,54) +hasLocation(#20606,#20607) #20608=* -regexpterm(#20608,14,#20607,0,"json") -#20609=@"loc,{#10000},13,17,13,20" -locations_default(#20609,#10000,13,17,13,20) -hasLocation(#20608,#20609) -regexp_const_value(#20608,"json") -#20610=* -stmts(#20610,27,#20001,10,"import ... son"" };") -hasLocation(#20610,#20035) -stmt_containers(#20610,#20001) +exprs(#20608,4,#20606,0,"""with""") +hasLocation(#20608,#20237) +enclosing_stmt(#20608,#20595) +expr_containers(#20608,#20001) +literals("with","""with""",#20608) +#20609=* +regexpterm(#20609,14,#20608,0,"with") +#20610=@"loc,{#10000},10,32,10,35" +locations_default(#20610,#10000,10,32,10,35) +hasLocation(#20609,#20610) +regexp_const_value(#20609,"with") #20611=* -exprs(#20611,4,#20610,-1,"""module""") -hasLocation(#20611,#20283) -enclosing_stmt(#20611,#20610) +exprs(#20611,8,#20606,1,"{ type: ""json"" }") +#20612=@"loc,{#10000},10,39,10,54" +locations_default(#20612,#10000,10,39,10,54) +hasLocation(#20611,#20612) +enclosing_stmt(#20611,#20595) expr_containers(#20611,#20001) -literals("module","""module""",#20611) -#20612=* -regexpterm(#20612,14,#20611,0,"module") -#20613=@"loc,{#10000},15,9,15,14" -locations_default(#20613,#10000,15,9,15,14) -hasLocation(#20612,#20613) -regexp_const_value(#20612,"module") -#20614=* -exprs(#20614,8,#20610,-10,"{ type: ""json"" }") -#20615=@"loc,{#10000},15,24,15,39" -locations_default(#20615,#10000,15,24,15,39) -hasLocation(#20614,#20615) -enclosing_stmt(#20614,#20610) -expr_containers(#20614,#20001) +#20613=* +properties(#20613,#20611,0,0,"type: ""json""") +#20614=@"loc,{#10000},10,41,10,52" +locations_default(#20614,#10000,10,41,10,52) +hasLocation(#20613,#20614) +#20615=* +exprs(#20615,0,#20613,0,"type") +hasLocation(#20615,#20243) +enclosing_stmt(#20615,#20595) +expr_containers(#20615,#20001) +literals("type","type",#20615) #20616=* -properties(#20616,#20614,0,0,"type: ""json""") -#20617=@"loc,{#10000},15,26,15,37" -locations_default(#20617,#10000,15,26,15,37) -hasLocation(#20616,#20617) -#20618=* -stmts(#20618,27,#20001,11,"import ... son"" };") -hasLocation(#20618,#20037) -stmt_containers(#20618,#20001) +exprs(#20616,4,#20613,1,"""json""") +hasLocation(#20616,#20247) +enclosing_stmt(#20616,#20595) +expr_containers(#20616,#20001) +literals("json","""json""",#20616) +#20617=* +regexpterm(#20617,14,#20616,0,"json") +#20618=@"loc,{#10000},10,48,10,51" +locations_default(#20618,#10000,10,48,10,51) +hasLocation(#20617,#20618) +regexp_const_value(#20617,"json") #20619=* -exprs(#20619,4,#20618,-1,"""module""") -hasLocation(#20619,#20309) -enclosing_stmt(#20619,#20618) -expr_containers(#20619,#20001) -literals("module","""module""",#20619) -#20620=* -regexpterm(#20620,14,#20619,0,"module") -#20621=@"loc,{#10000},16,22,16,27" -locations_default(#20621,#10000,16,22,16,27) -hasLocation(#20620,#20621) -regexp_const_value(#20620,"module") +stmts(#20619,27,#20001,8,"import ""module"";") +#20620=@"loc,{#10000},12,1,12,16" +locations_default(#20620,#10000,12,1,12,16) +hasLocation(#20619,#20620) +stmt_containers(#20619,#20001) +#20621=* +exprs(#20621,4,#20619,-1,"""module""") +hasLocation(#20621,#20259) +enclosing_stmt(#20621,#20619) +expr_containers(#20621,#20001) +literals("module","""module""",#20621) #20622=* -exprs(#20622,8,#20618,-10,"{ type: ""json"" }") -#20623=@"loc,{#10000},16,37,16,52" -locations_default(#20623,#10000,16,37,16,52) +regexpterm(#20622,14,#20621,0,"module") +#20623=@"loc,{#10000},12,9,12,14" +locations_default(#20623,#10000,12,9,12,14) hasLocation(#20622,#20623) -enclosing_stmt(#20622,#20618) -expr_containers(#20622,#20001) +regexp_const_value(#20622,"module") #20624=* -properties(#20624,#20622,0,0,"type: ""json""") -#20625=@"loc,{#10000},16,39,16,50" -locations_default(#20625,#10000,16,39,16,50) +stmts(#20624,2,#20001,9,"assert( ... on"" });") +#20625=@"loc,{#10000},13,1,13,25" +locations_default(#20625,#10000,13,1,13,25) hasLocation(#20624,#20625) +stmt_containers(#20624,#20001) #20626=* -exprs(#20626,85,#20618,0,"* as v1") -#20627=@"loc,{#10000},16,8,16,14" -locations_default(#20627,#10000,16,8,16,14) +exprs(#20626,13,#20624,0,"assert( ... son"" })") +#20627=@"loc,{#10000},13,1,13,24" +locations_default(#20627,#10000,13,1,13,24) hasLocation(#20626,#20627) -enclosing_stmt(#20626,#20618) +enclosing_stmt(#20626,#20624) expr_containers(#20626,#20001) #20628=* -exprs(#20628,78,#20626,1,"v1") -hasLocation(#20628,#20305) -enclosing_stmt(#20628,#20618) +exprs(#20628,79,#20626,-1,"assert") +hasLocation(#20628,#20263) +enclosing_stmt(#20628,#20624) expr_containers(#20628,#20001) -literals("v1","v1",#20628) -decl(#20628,#20486) -typedecl(#20628,#20489) -namespacedecl(#20628,#20492) -#20629=* -stmts(#20629,27,#20001,12,"import ... son"" };") -hasLocation(#20629,#20039) -stmt_containers(#20629,#20001) +literals("assert","assert",#20628) +#20629=@"var;{assert};{#20000}" +variables(#20629,"assert",#20000) +bind(#20628,#20629) #20630=* -exprs(#20630,4,#20629,-1,"""module""") -hasLocation(#20630,#20335) -enclosing_stmt(#20630,#20629) +exprs(#20630,8,#20626,0,"{ type: ""json"" }") +#20631=@"loc,{#10000},13,8,13,23" +locations_default(#20631,#10000,13,8,13,23) +hasLocation(#20630,#20631) +enclosing_stmt(#20630,#20624) expr_containers(#20630,#20001) -literals("module","""module""",#20630) -#20631=* -regexpterm(#20631,14,#20630,0,"module") -#20632=@"loc,{#10000},17,21,17,26" -locations_default(#20632,#10000,17,21,17,26) -hasLocation(#20631,#20632) -regexp_const_value(#20631,"module") -#20633=* -exprs(#20633,8,#20629,-10,"{ type: ""json"" }") -#20634=@"loc,{#10000},17,36,17,51" -locations_default(#20634,#10000,17,36,17,51) -hasLocation(#20633,#20634) -enclosing_stmt(#20633,#20629) -expr_containers(#20633,#20001) +#20632=* +properties(#20632,#20630,0,0,"type: ""json""") +#20633=@"loc,{#10000},13,10,13,21" +locations_default(#20633,#10000,13,10,13,21) +hasLocation(#20632,#20633) +#20634=* +exprs(#20634,0,#20632,0,"type") +hasLocation(#20634,#20269) +enclosing_stmt(#20634,#20624) +expr_containers(#20634,#20001) +literals("type","type",#20634) #20635=* -properties(#20635,#20633,0,0,"type: ""json""") -#20636=@"loc,{#10000},17,38,17,49" -locations_default(#20636,#10000,17,38,17,49) -hasLocation(#20635,#20636) -#20637=* -exprs(#20637,83,#20629,0,"v2") -hasLocation(#20637,#20329) -enclosing_stmt(#20637,#20629) -expr_containers(#20637,#20001) +exprs(#20635,4,#20632,1,"""json""") +hasLocation(#20635,#20273) +enclosing_stmt(#20635,#20624) +expr_containers(#20635,#20001) +literals("json","""json""",#20635) +#20636=* +regexpterm(#20636,14,#20635,0,"json") +#20637=@"loc,{#10000},13,17,13,20" +locations_default(#20637,#10000,13,17,13,20) +hasLocation(#20636,#20637) +regexp_const_value(#20636,"json") #20638=* -exprs(#20638,0,#20637,0,"v2") -hasLocation(#20638,#20329) -enclosing_stmt(#20638,#20629) -expr_containers(#20638,#20001) -literals("v2","v2",#20638) +stmts(#20638,27,#20001,10,"import ... son"" };") +hasLocation(#20638,#20035) +stmt_containers(#20638,#20001) #20639=* -exprs(#20639,78,#20637,1,"v2") -hasLocation(#20639,#20329) -enclosing_stmt(#20639,#20629) +exprs(#20639,4,#20638,-1,"""module""") +hasLocation(#20639,#20283) +enclosing_stmt(#20639,#20638) expr_containers(#20639,#20001) -literals("v2","v2",#20639) -decl(#20639,#20487) -typedecl(#20639,#20490) -namespacedecl(#20639,#20493) +literals("module","""module""",#20639) #20640=* -stmts(#20640,27,#20001,13,"import ... son"" };") -hasLocation(#20640,#20041) -stmt_containers(#20640,#20001) -#20641=* -exprs(#20641,4,#20640,-1,"""module""") -hasLocation(#20641,#20357) -enclosing_stmt(#20641,#20640) -expr_containers(#20641,#20001) -literals("module","""module""",#20641) +regexpterm(#20640,14,#20639,0,"module") +#20641=@"loc,{#10000},15,9,15,14" +locations_default(#20641,#10000,15,9,15,14) +hasLocation(#20640,#20641) +regexp_const_value(#20640,"module") #20642=* -regexpterm(#20642,14,#20641,0,"module") -#20643=@"loc,{#10000},18,17,18,22" -locations_default(#20643,#10000,18,17,18,22) +exprs(#20642,8,#20638,-10,"{ type: ""json"" }") +#20643=@"loc,{#10000},15,24,15,39" +locations_default(#20643,#10000,15,24,15,39) hasLocation(#20642,#20643) -regexp_const_value(#20642,"module") +enclosing_stmt(#20642,#20638) +expr_containers(#20642,#20001) #20644=* -exprs(#20644,8,#20640,-10,"{ type: ""json"" }") -#20645=@"loc,{#10000},18,32,18,47" -locations_default(#20645,#10000,18,32,18,47) +properties(#20644,#20642,0,0,"type: ""json""") +#20645=@"loc,{#10000},15,26,15,37" +locations_default(#20645,#10000,15,26,15,37) hasLocation(#20644,#20645) -enclosing_stmt(#20644,#20640) -expr_containers(#20644,#20001) #20646=* -properties(#20646,#20644,0,0,"type: ""json""") -#20647=@"loc,{#10000},18,34,18,45" -locations_default(#20647,#10000,18,34,18,45) -hasLocation(#20646,#20647) +exprs(#20646,0,#20644,0,"type") +hasLocation(#20646,#20289) +enclosing_stmt(#20646,#20638) +expr_containers(#20646,#20001) +literals("type","type",#20646) +#20647=* +exprs(#20647,4,#20644,1,"""json""") +hasLocation(#20647,#20293) +enclosing_stmt(#20647,#20638) +expr_containers(#20647,#20001) +literals("json","""json""",#20647) #20648=* -exprs(#20648,84,#20640,0,"v3") -hasLocation(#20648,#20353) -enclosing_stmt(#20648,#20640) -expr_containers(#20648,#20001) -#20649=* -exprs(#20649,78,#20648,1,"v3") -hasLocation(#20649,#20353) -enclosing_stmt(#20649,#20640) -expr_containers(#20649,#20001) -literals("v3","v3",#20649) -decl(#20649,#20488) -typedecl(#20649,#20491) -namespacedecl(#20649,#20494) +regexpterm(#20648,14,#20647,0,"json") +#20649=@"loc,{#10000},15,33,15,36" +locations_default(#20649,#10000,15,33,15,36) +hasLocation(#20648,#20649) +regexp_const_value(#20648,"json") #20650=* -stmts(#20650,30,#20001,14,"export ... son"" };") -hasLocation(#20650,#20045) +stmts(#20650,27,#20001,11,"import ... son"" };") +hasLocation(#20650,#20037) stmt_containers(#20650,#20001) #20651=* -exprs(#20651,4,#20650,-2,"""module""") -hasLocation(#20651,#20383) +exprs(#20651,4,#20650,-1,"""module""") +hasLocation(#20651,#20309) enclosing_stmt(#20651,#20650) expr_containers(#20651,#20001) literals("module","""module""",#20651) #20652=* regexpterm(#20652,14,#20651,0,"module") -#20653=@"loc,{#10000},20,21,20,26" -locations_default(#20653,#10000,20,21,20,26) +#20653=@"loc,{#10000},16,22,16,27" +locations_default(#20653,#10000,16,22,16,27) hasLocation(#20652,#20653) regexp_const_value(#20652,"module") #20654=* exprs(#20654,8,#20650,-10,"{ type: ""json"" }") -#20655=@"loc,{#10000},20,36,20,51" -locations_default(#20655,#10000,20,36,20,51) +#20655=@"loc,{#10000},16,37,16,52" +locations_default(#20655,#10000,16,37,16,52) hasLocation(#20654,#20655) enclosing_stmt(#20654,#20650) expr_containers(#20654,#20001) #20656=* properties(#20656,#20654,0,0,"type: ""json""") -#20657=@"loc,{#10000},20,38,20,49" -locations_default(#20657,#10000,20,38,20,49) +#20657=@"loc,{#10000},16,39,16,50" +locations_default(#20657,#10000,16,39,16,50) hasLocation(#20656,#20657) #20658=* -exprs(#20658,86,#20650,0,"v4") -hasLocation(#20658,#20377) +exprs(#20658,0,#20656,0,"type") +hasLocation(#20658,#20315) enclosing_stmt(#20658,#20650) expr_containers(#20658,#20001) +literals("type","type",#20658) #20659=* -exprs(#20659,0,#20658,0,"v4") -hasLocation(#20659,#20377) +exprs(#20659,4,#20656,1,"""json""") +hasLocation(#20659,#20319) enclosing_stmt(#20659,#20650) expr_containers(#20659,#20001) -literals("v4","v4",#20659) +literals("json","""json""",#20659) #20660=* -exprs(#20660,0,#20658,1,"v4") -hasLocation(#20660,#20377) -enclosing_stmt(#20660,#20650) -expr_containers(#20660,#20001) -literals("v4","v4",#20660) -#20661=* -stmts(#20661,28,#20001,15,"export ... son"" };") -hasLocation(#20661,#20047) -stmt_containers(#20661,#20001) +regexpterm(#20660,14,#20659,0,"json") +#20661=@"loc,{#10000},16,46,16,49" +locations_default(#20661,#10000,16,46,16,49) +hasLocation(#20660,#20661) +regexp_const_value(#20660,"json") #20662=* -exprs(#20662,4,#20661,0,"""module""") -hasLocation(#20662,#20405) -enclosing_stmt(#20662,#20661) +exprs(#20662,85,#20650,0,"* as v1") +#20663=@"loc,{#10000},16,8,16,14" +locations_default(#20663,#10000,16,8,16,14) +hasLocation(#20662,#20663) +enclosing_stmt(#20662,#20650) expr_containers(#20662,#20001) -literals("module","""module""",#20662) -#20663=* -regexpterm(#20663,14,#20662,0,"module") -#20664=@"loc,{#10000},21,16,21,21" -locations_default(#20664,#10000,21,16,21,21) -hasLocation(#20663,#20664) -regexp_const_value(#20663,"module") +#20664=* +exprs(#20664,78,#20662,1,"v1") +hasLocation(#20664,#20305) +enclosing_stmt(#20664,#20650) +expr_containers(#20664,#20001) +literals("v1","v1",#20664) +decl(#20664,#20486) +typedecl(#20664,#20489) +namespacedecl(#20664,#20492) #20665=* -exprs(#20665,8,#20661,-10,"{ type: ""json"" }") -#20666=@"loc,{#10000},21,31,21,46" -locations_default(#20666,#10000,21,31,21,46) -hasLocation(#20665,#20666) -enclosing_stmt(#20665,#20661) -expr_containers(#20665,#20001) +stmts(#20665,27,#20001,12,"import ... son"" };") +hasLocation(#20665,#20039) +stmt_containers(#20665,#20001) +#20666=* +exprs(#20666,4,#20665,-1,"""module""") +hasLocation(#20666,#20335) +enclosing_stmt(#20666,#20665) +expr_containers(#20666,#20001) +literals("module","""module""",#20666) #20667=* -properties(#20667,#20665,0,0,"type: ""json""") -#20668=@"loc,{#10000},21,33,21,44" -locations_default(#20668,#10000,21,33,21,44) +regexpterm(#20667,14,#20666,0,"module") +#20668=@"loc,{#10000},17,21,17,26" +locations_default(#20668,#10000,17,21,17,26) hasLocation(#20667,#20668) +regexp_const_value(#20667,"module") #20669=* -stmts(#20669,30,#20001,16,"export ... son"" };") -hasLocation(#20669,#20049) -stmt_containers(#20669,#20001) -#20670=* -exprs(#20670,4,#20669,-2,"""module""") -hasLocation(#20670,#20431) -enclosing_stmt(#20670,#20669) -expr_containers(#20670,#20001) -literals("module","""module""",#20670) +exprs(#20669,8,#20665,-10,"{ type: ""json"" }") +#20670=@"loc,{#10000},17,36,17,51" +locations_default(#20670,#10000,17,36,17,51) +hasLocation(#20669,#20670) +enclosing_stmt(#20669,#20665) +expr_containers(#20669,#20001) #20671=* -regexpterm(#20671,14,#20670,0,"module") -#20672=@"loc,{#10000},22,22,22,27" -locations_default(#20672,#10000,22,22,22,27) +properties(#20671,#20669,0,0,"type: ""json""") +#20672=@"loc,{#10000},17,38,17,49" +locations_default(#20672,#10000,17,38,17,49) hasLocation(#20671,#20672) -regexp_const_value(#20671,"module") #20673=* -exprs(#20673,8,#20669,-10,"{ type: ""json"" }") -#20674=@"loc,{#10000},22,37,22,52" -locations_default(#20674,#10000,22,37,22,52) -hasLocation(#20673,#20674) -enclosing_stmt(#20673,#20669) +exprs(#20673,0,#20671,0,"type") +hasLocation(#20673,#20341) +enclosing_stmt(#20673,#20665) expr_containers(#20673,#20001) +literals("type","type",#20673) +#20674=* +exprs(#20674,4,#20671,1,"""json""") +hasLocation(#20674,#20345) +enclosing_stmt(#20674,#20665) +expr_containers(#20674,#20001) +literals("json","""json""",#20674) #20675=* -properties(#20675,#20673,0,0,"type: ""json""") -#20676=@"loc,{#10000},22,39,22,50" -locations_default(#20676,#10000,22,39,22,50) +regexpterm(#20675,14,#20674,0,"json") +#20676=@"loc,{#10000},17,45,17,48" +locations_default(#20676,#10000,17,45,17,48) hasLocation(#20675,#20676) +regexp_const_value(#20675,"json") #20677=* -exprs(#20677,96,#20669,0,"* as v5") -#20678=@"loc,{#10000},22,8,22,14" -locations_default(#20678,#10000,22,8,22,14) -hasLocation(#20677,#20678) -enclosing_stmt(#20677,#20669) +exprs(#20677,83,#20665,0,"v2") +hasLocation(#20677,#20329) +enclosing_stmt(#20677,#20665) expr_containers(#20677,#20001) +#20678=* +exprs(#20678,0,#20677,0,"v2") +hasLocation(#20678,#20329) +enclosing_stmt(#20678,#20665) +expr_containers(#20678,#20001) +literals("v2","v2",#20678) #20679=* -exprs(#20679,0,#20677,1,"v5") -hasLocation(#20679,#20427) -enclosing_stmt(#20679,#20669) +exprs(#20679,78,#20677,1,"v2") +hasLocation(#20679,#20329) +enclosing_stmt(#20679,#20665) expr_containers(#20679,#20001) -literals("v5","v5",#20679) +literals("v2","v2",#20679) +decl(#20679,#20487) +typedecl(#20679,#20490) +namespacedecl(#20679,#20493) #20680=* -stmts(#20680,22,#20001,17,"const v ... "" } });") -hasLocation(#20680,#20053) +stmts(#20680,27,#20001,13,"import ... son"" };") +hasLocation(#20680,#20041) stmt_containers(#20680,#20001) #20681=* -exprs(#20681,64,#20680,0,"v6 = im ... n"" } })") -#20682=@"loc,{#10000},24,7,24,57" -locations_default(#20682,#10000,24,7,24,57) -hasLocation(#20681,#20682) +exprs(#20681,4,#20680,-1,"""module""") +hasLocation(#20681,#20357) enclosing_stmt(#20681,#20680) expr_containers(#20681,#20001) -#20683=* -exprs(#20683,78,#20681,0,"v6") -hasLocation(#20683,#20449) -enclosing_stmt(#20683,#20680) -expr_containers(#20683,#20001) -literals("v6","v6",#20683) -decl(#20683,#20495) +literals("module","""module""",#20681) +#20682=* +regexpterm(#20682,14,#20681,0,"module") +#20683=@"loc,{#10000},18,17,18,22" +locations_default(#20683,#10000,18,17,18,22) +hasLocation(#20682,#20683) +regexp_const_value(#20682,"module") #20684=* -exprs(#20684,99,#20681,1,"import( ... n"" } })") -#20685=@"loc,{#10000},24,12,24,57" -locations_default(#20685,#10000,24,12,24,57) +exprs(#20684,8,#20680,-10,"{ type: ""json"" }") +#20685=@"loc,{#10000},18,32,18,47" +locations_default(#20685,#10000,18,32,18,47) hasLocation(#20684,#20685) enclosing_stmt(#20684,#20680) expr_containers(#20684,#20001) #20686=* -exprs(#20686,4,#20684,0,"""module""") -hasLocation(#20686,#20457) -enclosing_stmt(#20686,#20680) -expr_containers(#20686,#20001) -literals("module","""module""",#20686) -#20687=* -regexpterm(#20687,14,#20686,0,"module") -#20688=@"loc,{#10000},24,20,24,25" -locations_default(#20688,#10000,24,20,24,25) -hasLocation(#20687,#20688) -regexp_const_value(#20687,"module") +properties(#20686,#20684,0,0,"type: ""json""") +#20687=@"loc,{#10000},18,34,18,45" +locations_default(#20687,#10000,18,34,18,45) +hasLocation(#20686,#20687) +#20688=* +exprs(#20688,0,#20686,0,"type") +hasLocation(#20688,#20363) +enclosing_stmt(#20688,#20680) +expr_containers(#20688,#20001) +literals("type","type",#20688) #20689=* -exprs(#20689,8,#20684,1,"{ asser ... on"" } }") -#20690=@"loc,{#10000},24,29,24,56" -locations_default(#20690,#10000,24,29,24,56) -hasLocation(#20689,#20690) +exprs(#20689,4,#20686,1,"""json""") +hasLocation(#20689,#20367) enclosing_stmt(#20689,#20680) expr_containers(#20689,#20001) -#20691=* -properties(#20691,#20689,0,0,"assert: ... json"" }") -#20692=@"loc,{#10000},24,31,24,54" -locations_default(#20692,#10000,24,31,24,54) -hasLocation(#20691,#20692) +literals("json","""json""",#20689) +#20690=* +regexpterm(#20690,14,#20689,0,"json") +#20691=@"loc,{#10000},18,41,18,44" +locations_default(#20691,#10000,18,41,18,44) +hasLocation(#20690,#20691) +regexp_const_value(#20690,"json") +#20692=* +exprs(#20692,84,#20680,0,"v3") +hasLocation(#20692,#20353) +enclosing_stmt(#20692,#20680) +expr_containers(#20692,#20001) #20693=* -exprs(#20693,0,#20691,0,"assert") -hasLocation(#20693,#20463) +exprs(#20693,78,#20692,1,"v3") +hasLocation(#20693,#20353) enclosing_stmt(#20693,#20680) expr_containers(#20693,#20001) -literals("assert","assert",#20693) +literals("v3","v3",#20693) +decl(#20693,#20488) +typedecl(#20693,#20491) +namespacedecl(#20693,#20494) #20694=* -exprs(#20694,8,#20691,1,"{ type: ""json"" }") -#20695=@"loc,{#10000},24,39,24,54" -locations_default(#20695,#10000,24,39,24,54) -hasLocation(#20694,#20695) -enclosing_stmt(#20694,#20680) -expr_containers(#20694,#20001) +stmts(#20694,30,#20001,14,"export ... son"" };") +hasLocation(#20694,#20045) +stmt_containers(#20694,#20001) +#20695=* +exprs(#20695,4,#20694,-2,"""module""") +hasLocation(#20695,#20383) +enclosing_stmt(#20695,#20694) +expr_containers(#20695,#20001) +literals("module","""module""",#20695) #20696=* -properties(#20696,#20694,0,0,"type: ""json""") -#20697=@"loc,{#10000},24,41,24,52" -locations_default(#20697,#10000,24,41,24,52) +regexpterm(#20696,14,#20695,0,"module") +#20697=@"loc,{#10000},20,21,20,26" +locations_default(#20697,#10000,20,21,20,26) hasLocation(#20696,#20697) +regexp_const_value(#20696,"module") #20698=* -exprs(#20698,0,#20696,0,"type") -hasLocation(#20698,#20469) -enclosing_stmt(#20698,#20680) +exprs(#20698,8,#20694,-10,"{ type: ""json"" }") +#20699=@"loc,{#10000},20,36,20,51" +locations_default(#20699,#10000,20,36,20,51) +hasLocation(#20698,#20699) +enclosing_stmt(#20698,#20694) expr_containers(#20698,#20001) -literals("type","type",#20698) -#20699=* -exprs(#20699,4,#20696,1,"""json""") -hasLocation(#20699,#20473) -enclosing_stmt(#20699,#20680) -expr_containers(#20699,#20001) -literals("json","""json""",#20699) #20700=* -regexpterm(#20700,14,#20699,0,"json") -#20701=@"loc,{#10000},24,48,24,51" -locations_default(#20701,#10000,24,48,24,51) +properties(#20700,#20698,0,0,"type: ""json""") +#20701=@"loc,{#10000},20,38,20,49" +locations_default(#20701,#10000,20,38,20,49) hasLocation(#20700,#20701) -regexp_const_value(#20700,"json") #20702=* -entry_cfg_node(#20702,#20001) -#20703=@"loc,{#10000},1,1,1,0" -locations_default(#20703,#10000,1,1,1,0) -hasLocation(#20702,#20703) +exprs(#20702,0,#20700,0,"type") +hasLocation(#20702,#20389) +enclosing_stmt(#20702,#20694) +expr_containers(#20702,#20001) +literals("type","type",#20702) +#20703=* +exprs(#20703,4,#20700,1,"""json""") +hasLocation(#20703,#20393) +enclosing_stmt(#20703,#20694) +expr_containers(#20703,#20001) +literals("json","""json""",#20703) #20704=* -exit_cfg_node(#20704,#20001) -hasLocation(#20704,#20483) -successor(#20680,#20683) -successor(#20686,#20684) -successor(#20684,#20681) -successor(#20683,#20686) -successor(#20681,#20704) -successor(#20669,#20670) -successor(#20677,#20679) -successor(#20679,#20680) -successor(#20670,#20677) -successor(#20661,#20662) -successor(#20662,#20669) -successor(#20650,#20651) -successor(#20658,#20659) -successor(#20660,#20661) -successor(#20659,#20660) -successor(#20651,#20658) -successor(#20640,#20650) -successor(#20629,#20640) -successor(#20618,#20629) -successor(#20610,#20618) -successor(#20596,#20600) -successor(#20602,#20606) -successor(#20607,#20604) -successor(#20606,#20607) -successor(#20604,#20598) -successor(#20600,#20602) -successor(#20598,#20610) -successor(#20591,#20596) -successor(#20567,#20570) -successor(#20573,#20571) -successor(#20571,#20568) -successor(#20570,#20573) -successor(#20568,#20591) -successor(#20556,#20557) -successor(#20564,#20566) +regexpterm(#20704,14,#20703,0,"json") +#20705=@"loc,{#10000},20,45,20,48" +locations_default(#20705,#10000,20,45,20,48) +hasLocation(#20704,#20705) +regexp_const_value(#20704,"json") +#20706=* +exprs(#20706,86,#20694,0,"v4") +hasLocation(#20706,#20377) +enclosing_stmt(#20706,#20694) +expr_containers(#20706,#20001) +#20707=* +exprs(#20707,0,#20706,0,"v4") +hasLocation(#20707,#20377) +enclosing_stmt(#20707,#20694) +expr_containers(#20707,#20001) +literals("v4","v4",#20707) +#20708=* +exprs(#20708,0,#20706,1,"v4") +hasLocation(#20708,#20377) +enclosing_stmt(#20708,#20694) +expr_containers(#20708,#20001) +literals("v4","v4",#20708) +#20709=* +stmts(#20709,28,#20001,15,"export ... son"" };") +hasLocation(#20709,#20047) +stmt_containers(#20709,#20001) +#20710=* +exprs(#20710,4,#20709,0,"""module""") +hasLocation(#20710,#20405) +enclosing_stmt(#20710,#20709) +expr_containers(#20710,#20001) +literals("module","""module""",#20710) +#20711=* +regexpterm(#20711,14,#20710,0,"module") +#20712=@"loc,{#10000},21,16,21,21" +locations_default(#20712,#10000,21,16,21,21) +hasLocation(#20711,#20712) +regexp_const_value(#20711,"module") +#20713=* +exprs(#20713,8,#20709,-10,"{ type: ""json"" }") +#20714=@"loc,{#10000},21,31,21,46" +locations_default(#20714,#10000,21,31,21,46) +hasLocation(#20713,#20714) +enclosing_stmt(#20713,#20709) +expr_containers(#20713,#20001) +#20715=* +properties(#20715,#20713,0,0,"type: ""json""") +#20716=@"loc,{#10000},21,33,21,44" +locations_default(#20716,#10000,21,33,21,44) +hasLocation(#20715,#20716) +#20717=* +exprs(#20717,0,#20715,0,"type") +hasLocation(#20717,#20411) +enclosing_stmt(#20717,#20709) +expr_containers(#20717,#20001) +literals("type","type",#20717) +#20718=* +exprs(#20718,4,#20715,1,"""json""") +hasLocation(#20718,#20415) +enclosing_stmt(#20718,#20709) +expr_containers(#20718,#20001) +literals("json","""json""",#20718) +#20719=* +regexpterm(#20719,14,#20718,0,"json") +#20720=@"loc,{#10000},21,40,21,43" +locations_default(#20720,#10000,21,40,21,43) +hasLocation(#20719,#20720) +regexp_const_value(#20719,"json") +#20721=* +stmts(#20721,30,#20001,16,"export ... son"" };") +hasLocation(#20721,#20049) +stmt_containers(#20721,#20001) +#20722=* +exprs(#20722,4,#20721,-2,"""module""") +hasLocation(#20722,#20431) +enclosing_stmt(#20722,#20721) +expr_containers(#20722,#20001) +literals("module","""module""",#20722) +#20723=* +regexpterm(#20723,14,#20722,0,"module") +#20724=@"loc,{#10000},22,22,22,27" +locations_default(#20724,#10000,22,22,22,27) +hasLocation(#20723,#20724) +regexp_const_value(#20723,"module") +#20725=* +exprs(#20725,8,#20721,-10,"{ type: ""json"" }") +#20726=@"loc,{#10000},22,37,22,52" +locations_default(#20726,#10000,22,37,22,52) +hasLocation(#20725,#20726) +enclosing_stmt(#20725,#20721) +expr_containers(#20725,#20001) +#20727=* +properties(#20727,#20725,0,0,"type: ""json""") +#20728=@"loc,{#10000},22,39,22,50" +locations_default(#20728,#10000,22,39,22,50) +hasLocation(#20727,#20728) +#20729=* +exprs(#20729,0,#20727,0,"type") +hasLocation(#20729,#20437) +enclosing_stmt(#20729,#20721) +expr_containers(#20729,#20001) +literals("type","type",#20729) +#20730=* +exprs(#20730,4,#20727,1,"""json""") +hasLocation(#20730,#20441) +enclosing_stmt(#20730,#20721) +expr_containers(#20730,#20001) +literals("json","""json""",#20730) +#20731=* +regexpterm(#20731,14,#20730,0,"json") +#20732=@"loc,{#10000},22,46,22,49" +locations_default(#20732,#10000,22,46,22,49) +hasLocation(#20731,#20732) +regexp_const_value(#20731,"json") +#20733=* +exprs(#20733,96,#20721,0,"* as v5") +#20734=@"loc,{#10000},22,8,22,14" +locations_default(#20734,#10000,22,8,22,14) +hasLocation(#20733,#20734) +enclosing_stmt(#20733,#20721) +expr_containers(#20733,#20001) +#20735=* +exprs(#20735,0,#20733,1,"v5") +hasLocation(#20735,#20427) +enclosing_stmt(#20735,#20721) +expr_containers(#20735,#20001) +literals("v5","v5",#20735) +#20736=* +stmts(#20736,22,#20001,17,"const v ... "" } });") +hasLocation(#20736,#20053) +stmt_containers(#20736,#20001) +#20737=* +exprs(#20737,64,#20736,0,"v6 = im ... n"" } })") +#20738=@"loc,{#10000},24,7,24,57" +locations_default(#20738,#10000,24,7,24,57) +hasLocation(#20737,#20738) +enclosing_stmt(#20737,#20736) +expr_containers(#20737,#20001) +#20739=* +exprs(#20739,78,#20737,0,"v6") +hasLocation(#20739,#20449) +enclosing_stmt(#20739,#20736) +expr_containers(#20739,#20001) +literals("v6","v6",#20739) +decl(#20739,#20495) +#20740=* +exprs(#20740,99,#20737,1,"import( ... n"" } })") +#20741=@"loc,{#10000},24,12,24,57" +locations_default(#20741,#10000,24,12,24,57) +hasLocation(#20740,#20741) +enclosing_stmt(#20740,#20736) +expr_containers(#20740,#20001) +#20742=* +exprs(#20742,4,#20740,0,"""module""") +hasLocation(#20742,#20457) +enclosing_stmt(#20742,#20736) +expr_containers(#20742,#20001) +literals("module","""module""",#20742) +#20743=* +regexpterm(#20743,14,#20742,0,"module") +#20744=@"loc,{#10000},24,20,24,25" +locations_default(#20744,#10000,24,20,24,25) +hasLocation(#20743,#20744) +regexp_const_value(#20743,"module") +#20745=* +exprs(#20745,8,#20740,1,"{ asser ... on"" } }") +#20746=@"loc,{#10000},24,29,24,56" +locations_default(#20746,#10000,24,29,24,56) +hasLocation(#20745,#20746) +enclosing_stmt(#20745,#20736) +expr_containers(#20745,#20001) +#20747=* +properties(#20747,#20745,0,0,"assert: ... json"" }") +#20748=@"loc,{#10000},24,31,24,54" +locations_default(#20748,#10000,24,31,24,54) +hasLocation(#20747,#20748) +#20749=* +exprs(#20749,0,#20747,0,"assert") +hasLocation(#20749,#20463) +enclosing_stmt(#20749,#20736) +expr_containers(#20749,#20001) +literals("assert","assert",#20749) +#20750=* +exprs(#20750,8,#20747,1,"{ type: ""json"" }") +#20751=@"loc,{#10000},24,39,24,54" +locations_default(#20751,#10000,24,39,24,54) +hasLocation(#20750,#20751) +enclosing_stmt(#20750,#20736) +expr_containers(#20750,#20001) +#20752=* +properties(#20752,#20750,0,0,"type: ""json""") +#20753=@"loc,{#10000},24,41,24,52" +locations_default(#20753,#10000,24,41,24,52) +hasLocation(#20752,#20753) +#20754=* +exprs(#20754,0,#20752,0,"type") +hasLocation(#20754,#20469) +enclosing_stmt(#20754,#20736) +expr_containers(#20754,#20001) +literals("type","type",#20754) +#20755=* +exprs(#20755,4,#20752,1,"""json""") +hasLocation(#20755,#20473) +enclosing_stmt(#20755,#20736) +expr_containers(#20755,#20001) +literals("json","""json""",#20755) +#20756=* +regexpterm(#20756,14,#20755,0,"json") +#20757=@"loc,{#10000},24,48,24,51" +locations_default(#20757,#10000,24,48,24,51) +hasLocation(#20756,#20757) +regexp_const_value(#20756,"json") +#20758=* +entry_cfg_node(#20758,#20001) +#20759=@"loc,{#10000},1,1,1,0" +locations_default(#20759,#10000,1,1,1,0) +hasLocation(#20758,#20759) +#20760=* +exit_cfg_node(#20760,#20001) +hasLocation(#20760,#20483) +successor(#20736,#20739) +successor(#20742,#20740) +successor(#20740,#20737) +successor(#20739,#20742) +successor(#20737,#20760) +successor(#20721,#20722) +successor(#20733,#20735) +successor(#20735,#20736) +successor(#20722,#20733) +successor(#20709,#20710) +successor(#20710,#20721) +successor(#20694,#20695) +successor(#20706,#20707) +successor(#20708,#20709) +successor(#20707,#20708) +successor(#20695,#20706) +successor(#20680,#20694) +successor(#20665,#20680) +successor(#20650,#20665) +successor(#20638,#20650) +successor(#20624,#20628) +successor(#20630,#20634) +successor(#20635,#20632) +successor(#20634,#20635) +successor(#20632,#20626) +successor(#20628,#20630) +successor(#20626,#20638) +successor(#20619,#20624) +successor(#20595,#20598) +successor(#20601,#20599) +successor(#20599,#20596) +successor(#20598,#20601) +successor(#20596,#20619) +successor(#20580,#20581) +successor(#20592,#20594) +successor(#20594,#20595) +successor(#20581,#20592) +successor(#20568,#20569) +successor(#20569,#20580) +successor(#20553,#20554) +successor(#20565,#20566) +successor(#20567,#20568) successor(#20566,#20567) -successor(#20557,#20564) -successor(#20548,#20549) -successor(#20549,#20556) -successor(#20537,#20538) -successor(#20545,#20546) -successor(#20547,#20548) -successor(#20546,#20547) -successor(#20538,#20545) -successor(#20527,#20537) -successor(#20516,#20527) -successor(#20505,#20516) -successor(#20497,#20505) -successor(#20648,#20497) -successor(#20637,#20648) -successor(#20626,#20637) -successor(#20535,#20626) -successor(#20524,#20535) -successor(#20513,#20524) -successor(#20702,#20513) +successor(#20554,#20565) +successor(#20539,#20553) +successor(#20524,#20539) +successor(#20509,#20524) +successor(#20497,#20509) +successor(#20692,#20497) +successor(#20677,#20692) +successor(#20662,#20677) +successor(#20551,#20662) +successor(#20536,#20551) +successor(#20521,#20536) +successor(#20758,#20521) numlines(#10000,24,18,2) filetype(#10000,"typescript") diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected new file mode 100644 index 000000000000..769e9dfee998 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected @@ -0,0 +1,5 @@ +imports +| tst.ts:1:1:1:40 | import ... .wasm"; | +references +| tst.ts:1:15:1:18 | wasm | +| tst.ts:3:1:3:4 | wasm | diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql new file mode 100644 index 000000000000..0997989d37ec --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql @@ -0,0 +1,5 @@ +import javascript + +query predicate imports(ImportDeclaration decl) { any() } + +query predicate references(VarRef ref) { any() } diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts b/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts new file mode 100644 index 000000000000..d1c1fa88139a --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts @@ -0,0 +1,3 @@ +import source wasm from "./module.wasm"; + +wasm; From e648d08d12b3a0708409ee9a72fbfd8d90449f7c Mon Sep 17 00:00:00 2001 From: Simon Friis Vindum Date: Mon, 19 Oct 2026 12:41:52 +0200 Subject: [PATCH 4/5] JS: Classify native parser crashes consistently after pipe errors Use the wrapper exit-code classification after IPC failures so a native prefetch crash is reported consistently with malformed responses. Preserve the communication-error fallback when the wrapper is still running. Make the internal-error integration test terminate the native child instead of relying on the legacy parser's stack overflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9f12f7cb-26aa-405b-a1f0-9dcc239778cb --- .../com/semmle/ts/extractor/TypeScriptParser.java | 15 +++++++++------ .../diagnostics/internal-error/fail-native.cjs | 7 +++++++ .../diagnostics/internal-error/test.py | 8 ++++++-- 3 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java index 17989d859a9c..4d65e6ed1594 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java @@ -365,12 +365,7 @@ private JsonObject talkToParserWrapper(JsonObject request) { throw getExceptionFromMalformedResponse(response, e); } } catch (IOException e) { - throw new CatastrophicError( - "Could not communicate with TypeScript parser wrapper " - + "(command: ." - + parserWrapperCommand - + ").", - e); + throw getExceptionFromMalformedResponse(response, e); } } @@ -397,6 +392,14 @@ private RuntimeException getExceptionFromMalformedResponse(String response, Exce } catch (InterruptedException e1) { Exceptions.ignore(e, "This is for diagnostic purposes only."); } + if (e instanceof IOException) { + return new CatastrophicError( + "Could not communicate with TypeScript parser wrapper " + + "(command: ." + + parserWrapperCommand + + ").", + e); + } if (response == null) { return new CatastrophicError("No response from TypeScript parser wrapper", e); } diff --git a/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs b/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs new file mode 100644 index 000000000000..6dd5449b1acb --- /dev/null +++ b/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs @@ -0,0 +1,7 @@ +const childProcess = require("node:child_process"); +const spawn = childProcess.spawn; +childProcess.spawn = function (...args) { + const child = spawn.apply(this, args); + child.kill(); + return child; +}; diff --git a/javascript/ql/integration-tests/diagnostics/internal-error/test.py b/javascript/ql/integration-tests/diagnostics/internal-error/test.py index 25a95e763ee8..c9e36473f627 100644 --- a/javascript/ql/integration-tests/diagnostics/internal-error/test.py +++ b/javascript/ql/integration-tests/diagnostics/internal-error/test.py @@ -1,2 +1,6 @@ -def test(codeql, javascript): - codeql.database.create(_assert_failure=True) +import os + + +def test(codeql, javascript, cwd): + os.environ["NODE_OPTIONS"] = f'--require "{cwd / "fail-native.cjs"}"' + codeql.database.create(source_root="src", _assert_failure=True) From 6493379a39353a6795290dcab3d9d4c3c92007a9 Mon Sep 17 00:00:00 2001 From: Simon Friis Vindum Date: Mon, 19 Oct 2026 11:59:10 +0200 Subject: [PATCH 5/5] JS: Document the native TypeScript parser and runtime requirements Describe the pinned compiler, build and packaging workflow, AST projection, shared-API lifecycle, compatibility coverage, and performance comparisons. Raise the documented minimum Node.js version to 16.20.0. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9f12f7cb-26aa-405b-a1f0-9dcc239778cb --- .../codeql-overview/system-requirements.rst | 2 +- javascript/extractor/lib/typescript/README.md | 224 ++++++++++++++++++ 2 files changed, 225 insertions(+), 1 deletion(-) diff --git a/docs/codeql/codeql-overview/system-requirements.rst b/docs/codeql/codeql-overview/system-requirements.rst index 9f477a9b39bf..1fb75319bb40 100644 --- a/docs/codeql/codeql-overview/system-requirements.rst +++ b/docs/codeql/codeql-overview/system-requirements.rst @@ -30,7 +30,7 @@ For extraction of compiled languages on Windows: For TypeScript extraction on all platforms: -- Node.js 14 or higher must be installed and available on the ``PATH`` as ``node``. +- Node.js 16.20.0 or higher must be installed and available on the ``PATH`` as ``node``. For Python extraction: diff --git a/javascript/extractor/lib/typescript/README.md b/javascript/extractor/lib/typescript/README.md index 998cec250aaa..5fd153588f05 100644 --- a/javascript/extractor/lib/typescript/README.md +++ b/javascript/extractor/lib/typescript/README.md @@ -16,3 +16,227 @@ To debug the parser script: 2. Open VSCode and choose "Debug: Attach to Node process" from the command palette, and choose the process that looks like the parser-wrapper. + +Building and running +-------------------- + +The wrapper uses the exact `typescript@7.1.0-beta` package for both +compilation/type checking and the native runtime. Its integrity and those of all +platform-specific optional dependencies are recorded in `package-lock.json`. + +```sh +npm ci +npm run build +npm test +node build/main.js --version +``` + +Node.js 16.20.0 is the minimum supported runtime, matching the pinned TypeScript +package's engine requirement. The existing Bazel toolchain uses Node.js 18.15.0 +and does not need an upgrade. Build scripts invoke TypeScript's +`lib/tsc.js` launcher explicitly because these Node.js versions cannot load its +extensionless ESM `bin/tsc` entry point. The installed wrapper must include its +`package.json` to retain the ESM package boundary. + +The existing TSLint rules require the legacy JavaScript compiler API, not the +native TypeScript 7 API. If compatible TSLint tooling is not already installed, +run it separately without replacing the wrapper's build/runtime dependency: + +```sh +npm exec --yes --package=tslint@6.1.3 --package=typescript@5.9.2 -- tslint --project . +``` + +The wrapper always uses the bundled TypeScript dependency. +`SEMMLE_TYPESCRIPT_HOME` is no longer supported and is ignored. + +Distribution packaging +---------------------- + +The Bazel build produces two archives: `:typescript` (`javascript.zip`) contains +the compiled ESM wrapper and JavaScript API; `:native` (`native.zip`) contains the +matching native packages for the target CodeQL distribution: + +| Distribution | Native packages | +| ------------- | ----------------------------------------------------------------------------- | +| `linux64` | `@typescript/typescript-linux-x64` | +| `linux-arm64` | `@typescript/typescript-linux-arm64` | +| `osx64` | `@typescript/typescript-darwin-arm64` and `@typescript/typescript-darwin-x64` | +| `win64` | `@typescript/typescript-win32-x64` | + +The build installs the selected packages explicitly with `--force --no-save`, +using the pinned versions and integrities in the lockfile. Merely passing +`--force` to `npm ci` still filters optional dependencies by the build host. +`package-runtime.mjs` verifies the native package versions and marks executables +as `0755` before ZIP creation. The common CodeQL archive contains the wrapper; +the architecture archive installs native packages into +`tools/typescript-parser-wrapper/node_modules/@typescript`. Both archives must +be installed, with executable permissions preserved. No separate TypeScript +installation is needed. + +Run the installed-package smoke test with: + +```sh +bazel test //javascript/extractor/lib/typescript:package-test +``` + +It checks all distribution selections, mismatched runtime rejection, common/arch +ZIP separation and executable metadata, then installs the merged archives with +the production `ripunzip` and parses TS, TSX, and malformed source through the +installed JSON protocol. It runs on the host platform and uses Bazel's Node.js +runtime. The Java extractor test suite also installs both ZIPs and restores +executable permissions using their central-directory metadata. + +All four distribution runtime ZIPs have been cross-assembled and inspected. +Installed parsing has been verified with Node.js 18.15.0 on macOS arm64 and on +macOS x64 through Rosetta. CI integration tests have also passed on Linux x64, +Linux arm64, and Windows x64, including Windows 2022. + +Native lifecycle +---------------- + +A single native API/process is created lazily and reused across files and +prepare-files batches. Each file still owns a separate program: AST projection +and JSON serialization finish before the program is disposed, including on +failure. Program disposal releases its snapshot and reference-counted source-file +caches without restarting the compiler. Reset closes the API, and the next parse +creates a new one. Quit, EOF, failure, and completion of single-file mode also +close the API. + +Native stderr is inherited separately from the JSON stdout protocol. Syntax +diagnostics remain successful AST responses; native, I/O, and protocol failures +are reported on stderr and terminate the wrapper with a nonzero exit status. +Reset, quit, EOF, and failure invalidate queued prefetch work; quit also closes +stdin without requiring the sender to close its pipe. + +AST projection and regression tests +----------------------------------- + +The wrapper targets the native `typescript/sync` and `typescript/ast` entry +points, using the same pinned dependency in the production ESM build. +`npm test` builds the production wrapper with the pinned compiler and runs +`test/wrapper.test.mjs` and `test/lifecycle.test.mjs` against its compiled output, +copied with its ESM package boundary into temporary installations. No separate +probe installation or legacy compiler is needed for these tests. + +AST nodes are explicitly projected into plain objects before their native +program is disposed. Syntax-kind and node-flag metadata are sent unchanged. +Projection checks only the whitelisted properties applicable to each syntax kind, +including optional properties, and falls back to the full whitelist for unknown +kinds. `src/ast_extractor.ts` loads the private +`dist/api/node/protocol.generated.js` metadata once from the locally installed, +pinned TypeScript dependency. +Updating TypeScript may require adapting this internal path or metadata format. +Scalar getters and kind-specific booleans are handled separately. +The Java converter consumes native fields directly, including `postfixToken`, +`defaultType`, and import attributes, and recognizes the native `EndOfFile` +spelling. Namespace classification and variable declaration kinds are derived +in Java without synthetic flags or properties. + +The tests cover AST projection, metadata, UTF-16 source positions, syntax +diagnostics, TSX, repeated parses, native process reuse and disposal, the child memory +budget, reset/quit/EOF, cancelled prefetch, the ignored legacy override, and fatal native +failures. Run them with Node.js 16.20.0 to verify the minimum supported runtime. + +Compatibility and regression coverage +------------------------------------ + +Native interface `extends` and class `implements` elements are type references. +The Java converter handles these using its existing type conversion, preserving +generic arguments and qualified names without rewriting their node kinds. + +The native parser still produces ASTs for legacy `assert { ... }` import +assertions, but adds diagnostic 2880. The wrapper excludes only that diagnostic +to preserve previously accepted syntax; genuine syntax diagnostics are unchanged. +Import-attribute names and values are explicitly serialized. Their absence +from the old wrapper's JSON was a bug: the old Java converter expected `key`/`value`, +but the old filter exposed neither correctly. Java now reads the native `name`/`value` +fields. The corresponding TRAP expectation was regenerated; existing QL +expectations did not need changes. + +Rescanning preserves token text and offsets while retaining the original +pre-rescan token kinds on the wire. In particular, regex and template-tail tokens +keep the classifications expected by existing TRAP tests. + +Source-phase imports (`import source wasm from "./module.wasm"`) are accepted. +Their declaration, imported path, and variable bindings use the existing import +representation; the schema does not yet distinguish the source phase. The new +`TypeScript/SourceImports` QL test covers these bindings. + +Validation on October 19, 2026 passed the pinned-compiler build/type check, +existing TSLint rules, parser compatibility and lifecycle tests, +500 Java tests, 68 TypeScript QL tests, the syntax-error query test, both extraction +diagnostic integration tests, and the installed-package smoke test. The +internal-error integration test deliberately terminates the native child rather +than relying on the old parser's stack overflow. + +All 19 production-output regression tests also pass on Node.js 16.20.0, the +minimum supported runtime. + +For QL tests, install the newly built extractor and explicitly select it; otherwise +the CLI may silently exercise an older installed extractor: + +```sh +# From the repository root, with the Java extractor build dependencies available: +bazel run //javascript:install +codeql resolve languages --search-path=javascript/extractor-pack:. --format=json +codeql test run -j4 --search-path=javascript/extractor-pack:. \ + javascript/ql/test/library-tests/TypeScript \ + javascript/ql/test/query-tests/LanguageFeatures/SyntaxError +``` + +Performance comparison +---------------------- + +The current shared-API implementation was compared with refreshed `main` at +`fb0ba8c8a8a` on October 21, 2026. Both ran on macOS arm64 with Node.js 18.15.0; +`main` was built and run with TypeScript 5.9.2, and this branch used the pinned +7.1 native compiler. These are medians of five fresh-process runs after one +warm-up per variant and workload, alternating variant order: + +| Workload | Main time | Branch time | Main combined RSS | Branch combined RSS | +| --- | --- | --- | --- | --- | +| 104 TypeScript fixtures (33,779 bytes) | 0.081 s | 0.120 s | 105.5 MiB | 110.4 MiB | +| Five parses of TypeScript's 588,128-byte declaration file | 0.886 s | 0.992 s | 263.7 MiB | 247.2 MiB | + +The branch takes 47.3% more time on the fixtures and 11.9% more on the declaration +workload. Sampled combined RSS is 4.7% higher and 6.3% lower, respectively. +Both variants parse identical inputs. Timing starts after the metadata handshake +and includes prepare-files, AST extraction, serialization, and protocol responses, +including the branch's first native-process startup. It excludes Java conversion +and database import. RSS is the maximum sampled sum of Node and its native +descendants, sampled at a nominal 5 ms interval; short-lived peaks may be missed. +Current `main` includes parser cleanup absent from the older baseline below. + +### Historical and API-lifetime comparisons + +The initial implementation created a native process per file. On macOS arm64 +with Node.js 18.15.0, a comparison against the pre-migration wrapper from +`1971c368e1d` compiled with TypeScript 5.9.2 gave these medians over three runs: + +| Workload | 5.9.2 time | 7.1 time | 5.9.2 combined RSS | 7.1 combined RSS | +| --------------------------------------------------------- | ---------- | -------- | ------------------ | ---------------- | +| 103 TypeScript fixtures (33,589 bytes) | 0.237 s | 1.349 s | 124.1 MiB | 113.9 MiB | +| Five parses of TypeScript's 588,128-byte declaration file | 0.991 s | 1.116 s | 269.3 MiB | 249.1 MiB | + +Timing covers prepare-files, extraction, serialization, and protocol responses +after the metadata handshake, not Java conversion or database import. Combined +RSS is the maximum sampled sum of Node and its native descendants, sampled every +5 ms; short-lived peaks may be missed. The per-file native process costs about +5.7x on tiny files and 1.13x on the declaration workload, while measured memory is +slightly lower. + +A follow-up comparison on October 21, 2026, using the same platform and Node.js +version, measured the per-file and shared-API lifetimes directly (medians of three +runs): + +| Workload | Per-file API time | Shared API time | Per-file combined RSS | Shared combined RSS | +| --- | --- | --- | --- | --- | +| 104 TypeScript fixtures (33,779 bytes) | 1.017 s | 0.118 s | 115.8 MiB | 109.4 MiB | +| Five parses of the same declaration file | 1.044 s | 0.985 s | 250.2 MiB | 260.2 MiB | + +AST responses were byte-identical between lifetimes. A longer shared-API run on +Node.js 26.7.0 processed 50,000 distinct generated TypeScript files: native RSS +was 27.9 MiB after 1,000 files and 30.4 MiB after 50,000, while combined RSS +stabilized around 176 MiB. Another run completed 500 parses of the declaration +file with peak native RSS of 56.5 MiB. These workloads showed no unbounded +per-file retention, so the wrapper reuses the API without a fixed batch limit.