Remove gen index in favor of main (#1571)

* Remove gen index

* Fix semver

* Simplify error message

cli/api/commands/compile.ts

@@ -42,27 +42,10 @@ export async function compile(
     );
   }
 
-  if (compileConfig.useMain === null || compileConfig.useMain === undefined) {
-    try {
-      const packageJson = JSON.parse(
-        fs.readFileSync(`${compileConfig.projectDir}/package.json`, "utf8")
-      );
-      const dataformCoreVersion = packageJson.dependencies["@dataform/core"];
-      compileConfig.useMain = semver.subset(dataformCoreVersion, ">=2.0.4");
-    } catch (e) {
-      // Silently catch any thrown Error. Do not attempt to use `main` compilation.
-    }
-  }
-
   const result = await CompileChildProcess.forkProcess().compile(compileConfig);
 
-  let compileResult: dataform.CompiledGraph;
-  if (compileConfig.useMain) {
-    const decodedResult = decode64(dataform.CoreExecutionResponse, result);
-    compileResult = dataform.CompiledGraph.create(decodedResult.compile.compiledGraph);
-  } else {
-    compileResult = decode64(dataform.CompiledGraph, result);
-  }
+  const decodedResult = decode64(dataform.CoreExecutionResponse, result);
+  const compileResult = dataform.CompiledGraph.create(decodedResult.compile.compiledGraph);
 
   compileResult.tables.forEach(setOrValidateTableEnumType);
   return compileResult;

cli/vm/BUILD

@@ -14,7 +14,9 @@ ts_library(
         "//protos:ts",
         "@npm//@types/glob",
         "@npm//@types/node",
+        "@npm//@types/semver",
         "@npm//glob",
+        "@npm//semver",
         "@npm//vm2",
     ],
 )

cli/vm/compile.ts

@@ -1,17 +1,15 @@
+import * as glob from "glob";
 import * as path from "path";
+import * as semver from "semver";
 import { CompilerFunction, NodeVM } from "vm2";
 
-import { createCoreExecutionRequest, createGenIndexConfig } from "df/cli/vm/create_config";
+import { encode64 } from "df/common/protos";
 import { dataform } from "df/protos/ts";
 
-function missingValidCorePackageError() {
-  return new Error(
-    `Could not find a recent installed version of @dataform/core in the project. Ensure packages are installed and upgrade to a recent version.`
-  );
-}
 export function compile(compileConfig: dataform.ICompileConfig) {
   const vmIndexFileName = path.resolve(path.join(compileConfig.projectDir, "index.js"));
 
+  // First retrieve a compiler function for vm2 to process files.
   const indexGeneratorVm = new NodeVM({
     wrapper: "none",
     require: {
@@ -21,19 +19,13 @@ export function compile(compileConfig: dataform.ICompileConfig) {
       builtin: ["path"]
     }
   });
+  const compiler: CompilerFunction = runDataformCoreVmScript(
+    indexGeneratorVm,
+    vmIndexFileName,
+    'return require("@dataform/core").compiler'
+  );
 
-  const findCompiler = (): CompilerFunction => {
-    try {
-      return indexGeneratorVm.run('return require("@dataform/core").compiler', vmIndexFileName);
-    } catch (e) {
-      throw missingValidCorePackageError();
-    }
-  };
-  const compiler = findCompiler();
-  if (!compiler) {
-    throw missingValidCorePackageError();
-  }
-
+  // Then use vm2's native compiler integration to apply the compiler to files.
   const userCodeVm = new NodeVM({
     wrapper: "none",
     require: {
@@ -48,31 +40,20 @@ export function compile(compileConfig: dataform.ICompileConfig) {
     compiler
   });
 
-  if (compileConfig.useMain) {
-    try {
-      return userCodeVm.run(
-        `return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")`,
-        vmIndexFileName
-      );
-    } catch (e) {
-      throw missingValidCorePackageError();
-    }
+  const dataformCoreVersion: string = runDataformCoreVmScript(
+    userCodeVm,
+    vmIndexFileName,
+    'return require("@dataform/core").version || "0.0.0"'
+  );
+  if (semver.lt(dataformCoreVersion, "3.0.0")) {
+    throw new Error("@dataform/core ^3.0.0 required.");
   }
 
-  // Generate an index file and run it.
-  const findGenIndex = (): ((base64EncodedConfig: string) => string) => {
-    try {
-      return indexGeneratorVm.run(
-        'return require("@dataform/core").indexFileGenerator',
-        vmIndexFileName
-      );
-    } catch (e) {
-      throw missingValidCorePackageError();
-    }
-  };
-  const genIndex = findGenIndex();
-
-  return userCodeVm.run(genIndex(createGenIndexConfig(compileConfig)), vmIndexFileName);
+  return runDataformCoreVmScript(
+    userCodeVm,
+    vmIndexFileName,
+    `return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")`
+  );
 }
 
 export function listenForCompileRequest() {
@@ -90,6 +71,44 @@ export function listenForCompileRequest() {
   });
 }
 
+function missingValidCorePackageError() {
+  return new Error(
+    "Could not find a recent installed version of @dataform/core in the project. Ensure packages " +
+      "are installed and upgrade to a recent version."
+  );
+}
+
+function runDataformCoreVmScript(nodeVM: NodeVM, vmIndexFileName: string, script: string): any {
+  // Missing valid core package errors are thrown because if @dataform/core isn't installed,
+  // the properties of it can't be found.
+  const getResult = (): any => {
+    try {
+      return nodeVM.run(script, vmIndexFileName);
+    } catch (e) {
+      throw missingValidCorePackageError();
+    }
+  };
+  const result = getResult();
+  if (!result) {
+    throw missingValidCorePackageError();
+  }
+  return result as any;
+}
+
 if (require.main === module) {
   listenForCompileRequest();
 }
+
+/**
+ * @returns a base64 encoded {@see dataform.CoreExecutionRequest} proto.
+ */
+function createCoreExecutionRequest(compileConfig: dataform.ICompileConfig): string {
+  const filePaths = Array.from(
+    new Set<string>(glob.sync("!(node_modules)/**/*.*", { cwd: compileConfig.projectDir }))
+  );
+
+  return encode64(dataform.CoreExecutionRequest, {
+    // Add the list of file paths to the compile config if not already set.
+    compile: { compileConfig: { filePaths, ...compileConfig } }
+  });
+}

core/index.ts

@@ -1,5 +1,4 @@
 import { compile as compiler } from "df/core/compilers";
-import { genIndex as indexFileGenerator } from "df/core/gen_index";
 import { main } from "df/core/main";
 import { Session } from "df/core/session";
 import { version } from "df/core/version";
@@ -20,4 +19,4 @@ const supportedFeatures = [dataform.SupportedFeatures.ARRAY_BUFFER_IPC];
 
 // These exports constitute the public API of @dataform/core.
 // Changes to these will break @dataform/api, so take care!
-export { compiler, indexFileGenerator, main, session, supportedFeatures, version };
+export { compiler, main, session, supportedFeatures, version };

core/main.ts

@@ -79,6 +79,7 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st
   compileRequest.compileConfig.filePaths
     .filter(path => path.startsWith(`definitions${utils.pathSeperator}`))
     .filter(path => path.endsWith(".js") || path.endsWith(".sqlx"))
+    .sort()
     .forEach(definitionPath => {
       try {
         // tslint:disable-next-line: tsr-detect-non-literal-require

protos/core.proto

@@ -56,10 +56,7 @@ message CompileConfig {
   // Override compilation timeout settings.
   int32 timeout_millis = 6;
 
-  // Whether to use the new main function instead of gen index.
-  bool use_main = 9;
-
-  reserved 2, 4, 5, 7;
+  reserved 2, 4, 5, 7, 9;
 }
 
 message Target {

tests/api/BUILD

@@ -4,8 +4,6 @@ ts_test_suite(
     name = "tests",
     srcs = glob(["**/*.ts"]),
     data = [
-        "//tests/api/projects/backwards_compatibility:files",
-        "//tests/api/projects/backwards_compatibility:node_modules",
         "//tests/api/projects/common_v2:files",
         "//tests/api/projects/common_v2:node_modules",
         "//tests/api/projects/invalid_dataform_json:files",