TS: Support tsconfig.json extending from ./node_modules

Author: asgerf

javascript/extractor/lib/typescript/src/common.ts

@@ -1,6 +1,7 @@
 import * as ts from "./typescript";
 import { TypeTable } from "./type_table";
 import * as pathlib from "path";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 /**
  * Extracts the package name from the prefix of an import string.
@@ -12,21 +13,16 @@ export class Project {
   public program: ts.Program = null;
   private host: ts.CompilerHost;
   private resolutionCache: ts.ModuleResolutionCache;
-  private sourceRoot: string;
-  /** Directory whose folder structure mirrors the real source root, but with `node_modules` installed. */
-  private virtualSourceRoot: string;
 
-  constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable, public packageLocations: PackageLocationMap) {
+  constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable, public packageLocations: PackageLocationMap,
+        public virtualSourceRoot: VirtualSourceRoot) {
     this.resolveModuleNames = this.resolveModuleNames.bind(this);
 
     this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options);
     let host = ts.createCompilerHost(config.options, true);
     host.resolveModuleNames = this.resolveModuleNames;
     host.trace = undefined; // Disable tracing which would otherwise go to standard out
     this.host = host;
-
-    this.sourceRoot = process.cwd();
-    this.virtualSourceRoot = process.env["CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR"];
   }
 
   public unload(): void {
@@ -83,7 +79,7 @@ export class Project {
     if (packageEntryPoint == null) {
       // The package is not overridden, but we have established that it begins with a valid package name.
       // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
-      let virtualContainingFile = this.toVirtualPath(containingFile);
+      let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile);
       if (virtualContainingFile != null) {
         return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
       }
@@ -119,15 +115,6 @@ export class Project {
 
     return null;
   }
-
-  /**
-   * Maps a path under the real source root to the corresonding path in the virtual source root.
-   */
-  private toVirtualPath(path: string) {
-    let relative = pathlib.relative(this.sourceRoot, path);
-    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
-    return pathlib.join(this.virtualSourceRoot, relative);
-  }
 }
 
 export type PackageLocationMap = Map<string, string>;

javascript/extractor/lib/typescript/src/main.ts

@@ -37,8 +37,9 @@ import * as readline from "readline";
 import * as ts from "./typescript";
 import * as ast_extractor from "./ast_extractor";
 
-import { Project } from "./common";
+import { Project, PackageLocationMap } from "./common";
 import { TypeTable } from "./type_table";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 interface ParseCommand {
     command: "parse";
@@ -47,7 +48,8 @@ interface ParseCommand {
 interface OpenProjectCommand {
     command: "open-project";
     tsConfig: string;
-    packageLocations: [string, string][];
+    packageEntryPoints: [string, string][];
+    packageJsonFiles: [string, string][];
 }
 interface CloseProjectCommand {
     command: "close-project";
@@ -243,20 +245,62 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} {
     return {ast, code};
 }
 
+const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/;
+
 function handleOpenProjectCommand(command: OpenProjectCommand) {
     Error.stackTraceLimit = Infinity;
     let tsConfigFilename = String(command.tsConfig);
     let tsConfig = ts.readConfigFile(tsConfigFilename, ts.sys.readFile);
     let basePath = pathlib.dirname(tsConfigFilename);
 
+    let packageEntryPoints = new Map(command.packageEntryPoints);
+    let packageJsonFiles = new Map(command.packageJsonFiles);
+    let virtualSourceRoot = new VirtualSourceRoot(process.cwd(), process.env["CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR"]);
+
+    /**
+     * Rewrites path segments of form `node_modules/PACK/suffix` to be relative to
+     * the location of package PACK in the source tree, if it exists.
+     */
+    function redirectNodeModulesPath(path: string) {
+        let nodeModulesMatch = nodeModulesRex.exec(path);
+        if (nodeModulesMatch == null) return null;
+        let packageName = nodeModulesMatch[1];
+        let packageJsonFile = packageJsonFiles.get(packageName);
+        if (packageJsonFile == null) return null;
+        let packageDir = pathlib.dirname(packageJsonFile);
+        let suffix = nodeModulesMatch[2];
+        let finalPath = pathlib.join(packageDir, suffix);
+        if (!ts.sys.fileExists(finalPath)) return null;
+        return finalPath;
+    }
+
+    /**
+     * Create the host passed to the tsconfig.json parser.
+     *
+     * We override its file system access in case there is an "extends"
+     * clause pointing into "./node_modules", which must be redirected to
+     * the location of an installed package or a checked-in package.
+     */
     let parseConfigHost: ts.ParseConfigHost = {
         useCaseSensitiveFileNames: true,
-        readDirectory: ts.sys.readDirectory,
-        fileExists: (path: string) => fs.existsSync(path),
-        readFile: ts.sys.readFile,
+        readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching
+        fileExists: (path: string) => {
+            return ts.sys.fileExists(path)
+                || virtualSourceRoot.toVirtualPathIfFileExists(path) != null
+                || redirectNodeModulesPath(path) != null;
+        },
+        readFile: (path: string) => {
+            if (!fs.existsSync(path)) {
+                let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path);
+                if (virtualPath != null) return ts.sys.readFile(virtualPath);
+                virtualPath = redirectNodeModulesPath(path);
+                if (virtualPath != null) return ts.sys.readFile(virtualPath);
+            }
+            return ts.sys.readFile(path);
+        }
     };
     let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
-    let project = new Project(tsConfigFilename, config, state.typeTable, new Map(command.packageLocations));
+    let project = new Project(tsConfigFilename, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
     project.load();
 
     state.project = project;
@@ -530,7 +574,8 @@ if (process.argv.length > 2) {
         handleOpenProjectCommand({
             command: "open-project",
             tsConfig: argument,
-            packageLocations: [],
+            packageEntryPoints: [],
+            packageJsonFiles: [],
         });
         for (let sf of state.project.program.getSourceFiles()) {
             if (pathlib.basename(sf.fileName) === "lib.d.ts") continue;

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -0,0 +1,34 @@
+import * as pathlib from "path";
+import * as ts from "./typescript";
+
+/**
+ * Mapping from the source root to the virtual source root.
+ */
+export class VirtualSourceRoot {
+  constructor(
+    private sourceRoot: string,
+
+    /** Directory whose folder structure mirrors the real source root, but with `node_modules` installed. */
+    private virtualSourceRoot: string,
+  ) {}
+
+  /**
+   * Maps a path under the real source root to the corresonding path in the virtual source root.
+   */
+  public toVirtualPath(path: string) {
+    let relative = pathlib.relative(this.sourceRoot, path);
+    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+    return pathlib.join(this.virtualSourceRoot, relative);
+  }
+
+  /**
+   * Maps a path under the real source root to the corresonding path in the virtual source root.
+   */
+  public toVirtualPathIfFileExists(path: string) {
+    let virtualPath = this.toVirtualPath(path);
+    if (virtualPath != null && ts.sys.fileExists(virtualPath)) {
+        return virtualPath;
+    }
+    return null;
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -782,7 +782,7 @@ protected DependencyInstallationResult installDependencies(Set<Path> filesToExtr
       }
     }
 
-    return new DependencyInstallationResult(packageMainFile);
+    return new DependencyInstallationResult(packageMainFile, packagesInRepo);
   }
 
   /**

javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java

@@ -6,20 +6,31 @@
 
 /** Contains the results of installing dependencies. */
 public class DependencyInstallationResult {
-  private Map<String, Path> packageLocations;
+  private Map<String, Path> packageEntryPoints;
+  private Map<String, Path> packageJsonFiles;
 
   public static final DependencyInstallationResult empty =
-      new DependencyInstallationResult(Collections.emptyMap());
+      new DependencyInstallationResult(Collections.emptyMap(), Collections.emptyMap());
 
-  public DependencyInstallationResult(Map<String, Path> localPackages) {
-    this.packageLocations = localPackages;
+  public DependencyInstallationResult(
+      Map<String, Path> packageEntryPoints,
+      Map<String, Path> packageJsonFiles) {
+    this.packageEntryPoints = packageEntryPoints;
+    this.packageJsonFiles = packageJsonFiles;
   }
 
   /**
    * Returns the mapping from package names to the TypeScript file that should
    * act as its main entry point.
    */
-  public Map<String, Path> getPackageLocations() {
-    return packageLocations;
+  public Map<String, Path> getPackageEntryPoints() {
+    return packageEntryPoints;
+  }
+
+  /**
+   * Returns the mapping from package name to corresponding package.json.
+   */
+  public Map<String, Path> getPackageJsonFiles() {
+    return packageJsonFiles;
   }
 }

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -11,10 +11,12 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.lang.ProcessBuilder.Redirect;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
@@ -404,6 +406,21 @@ public void prepareFiles(List<File> files) {
     checkResponseType(response, "ok");
   }
 
+  /**
+   * Converts a map to an array of [key, value] pairs.
+   */
+  private JsonArray mapToArray(Map<String, Path> map) {
+    JsonArray result = new JsonArray();
+    map.forEach(
+        (key, path) -> {
+          JsonArray entry = new JsonArray();
+          entry.add(key);
+          entry.add(path.toString());
+          result.add(entry);
+        });
+    return result;
+  }
+
   /**
    * Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the
    * project.
@@ -416,16 +433,8 @@ public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult
     JsonObject request = new JsonObject();
     request.add("command", new JsonPrimitive("open-project"));
     request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
-    JsonArray packageLocations = new JsonArray();
-    deps.getPackageLocations()
-        .forEach(
-            (packageName, packageDir) -> {
-              JsonArray entry = new JsonArray();
-              entry.add(packageName);
-              entry.add(packageDir.toString());
-              packageLocations.add(entry);
-            });
-    request.add("packageLocations", packageLocations);
+    request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
+    request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
     JsonObject response = talkToParserWrapper(request);
     try {
       checkResponseType(response, "project-opened");