diff --git a/cli/api/BUILD b/cli/api/BUILD index 7291e2e5f..0ec7241a8 100644 --- a/cli/api/BUILD +++ b/cli/api/BUILD @@ -16,6 +16,7 @@ ts_library( "//common/protos", "//common/strings", "//core", + "//core/compilation_sql", "//protos:ts", "//cli/vm:compile_loader", "//sqlx:lexer", diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 6faa564ed..caff6d25f 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -36,7 +36,7 @@ export function compile(compileConfig: dataform.ICompileConfig) { resolve: (moduleName, parentDirName) => path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName) }, - sourceExtensions: ["js", "sql", "sqlx"], + sourceExtensions: ["js", "sql", "sqlx", "yaml"], compiler }); diff --git a/core/BUILD b/core/BUILD index a0853e94c..4b3a60458 100644 --- a/core/BUILD +++ b/core/BUILD @@ -1,6 +1,8 @@ load("//tools:ts_library.bzl", "ts_library") load("//tools:expand_template.bzl", "expand_template") load("//:version.bzl", "DF_VERSION") +load("//testing:index.bzl", "ts_test_suite") +load("//tools:node_modules.bzl", "node_modules") package(default_visibility = ["//visibility:public"]) @@ -13,24 +15,71 @@ expand_template( template = "version.ts.tmpl", ) -filegroup( - name = "files", - srcs = glob(["**/*.*"]) + [":version.ts"], -) - ts_library( name = "core", - srcs = glob(["**/*.ts"]) + [":version.ts"], + srcs = [ + "assertion.ts", + "column_descriptors.ts", + "common.ts", + "compilers.ts", + "declaration.ts", + "index.ts", + "main.ts", + "operation.ts", + "session.ts", + "table.ts", + "targets.ts", + "tasks.ts", + "test.ts", + "utils.ts", + "workflow_settings.ts", + ":version.ts", + ], deps = [ "//common/errors", "//common/protos", "//common/strings", + "//core/compilation_sql", "//protos:ts", "//sqlx:lexer", + "@npm//@types/fs-extra", + "@npm//@types/js-yaml", "@npm//@types/node", "@npm//@types/semver", + "@npm//fs-extra", + "@npm//js-yaml", "@npm//protobufjs", "@npm//semver", "@npm//tarjan-graph", ], ) + +ts_test_suite( + name = "tests", + srcs = [ + "main_test.ts", + ], + data = [ + ":node_modules", + ], + deps = [ + ":core", + "//common/protos", + "//protos:ts", + "//testing", + "//tests/utils", + "@npm//@types/chai", + "@npm//@types/fs-extra", + "@npm//@types/node", + "@npm//chai", + "@npm//fs-extra", + "@npm//vm2", + ], +) + +node_modules( + name = "node_modules", + deps = [ + "//packages/@dataform/core:package_tar", + ], +) diff --git a/core/compilation_sql/BUILD b/core/compilation_sql/BUILD new file mode 100644 index 000000000..8562e708a --- /dev/null +++ b/core/compilation_sql/BUILD @@ -0,0 +1,13 @@ +load("//tools:ts_library.bzl", "ts_library") + +package(default_visibility = ["//visibility:public"]) + +ts_library( + name = "compilation_sql", + srcs = [ + "index.ts", + ], + deps = [ + "//protos:ts", + ] +) diff --git a/core/compilers.ts b/core/compilers.ts index bc7ae3c96..1313aa153 100644 --- a/core/compilers.ts +++ b/core/compilers.ts @@ -1,10 +1,23 @@ +import { load as loadYaml, YAMLException } from "js-yaml"; + import * as utils from "df/core/utils"; import { SyntaxTreeNode, SyntaxTreeNodeType } from "df/sqlx/lexer"; -export function compile(code: string, path: string) { +export function compile(code: string, path: string): string { if (path.endsWith(".sqlx")) { return compileSqlx(SyntaxTreeNode.create(code), path); } + if (path.endsWith(".yaml")) { + try { + const yamlAsJson = JSON.stringify(loadYaml(code)); + return `exports.asJson = () => (${yamlAsJson})`; + } catch (e) { + if (e instanceof YAMLException) { + throw Error(`${path} is not a valid YAML file: ${e}`); + } + throw e; + } + } return code; } @@ -31,7 +44,7 @@ export function extractJsBlocks(code: string): { sql: string; js: string } { }; } -function compileSqlx(rootNode: SyntaxTreeNode, path: string) { +function compileSqlx(rootNode: SyntaxTreeNode, path: string): string { const { config, js, sql, incremental, preOperations, postOperations, inputs } = extractSqlxParts( rootNode ); diff --git a/core/main.ts b/core/main.ts index 1fc1d08b7..91076c188 100644 --- a/core/main.ts +++ b/core/main.ts @@ -1,6 +1,7 @@ import { decode64, encode64 } from "df/common/protos"; import { Session } from "df/core/session"; import * as utils from "df/core/utils"; +import { readWorkflowSettings } from "df/core/workflow_settings"; import { dataform } from "df/protos/ts"; /** @@ -23,7 +24,7 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st const compileRequest = request.compile; // Read the project config from the root of the project. - const originalProjectConfig = require("dataform.json"); + const originalProjectConfig = readWorkflowSettings(); const projectConfigOverride = compileRequest.compileConfig.projectConfigOverride ?? {}; diff --git a/core/main_test.ts b/core/main_test.ts new file mode 100644 index 000000000..fef3cd97b --- /dev/null +++ b/core/main_test.ts @@ -0,0 +1,159 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { CompilerFunction, NodeVM } from "vm2"; + +import { decode64, encode64 } from "df/common/protos"; +import { compile } from "df/core/compilers"; +import { dataform } from "df/protos/ts"; +import { suite, test } from "df/testing"; +import { TmpDirFixture } from "df/testing/fixtures"; +import { asPlainObject } from "df/tests/utils"; + +const VALID_WORKFLOW_SETTINGS_YAML = ` +warehouse: bigquery +defaultDatabase: dataform +`; + +const VALID_DATAFORM_JSON = ` +{ + "warehouse": "bigquery", + "defaultDatabase": "dataform" +} +`; + +suite("@dataform/core", ({ afterEach }) => { + const tmpDirFixture = new TmpDirFixture(afterEach); + + suite("workflow settings", () => { + test(`main succeeds when a valid workflow_settings.yaml is present`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // tslint:disable-next-line: tsr-detect-non-literal-fs-filename + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + const coreExecutionRequest = dataform.CoreExecutionRequest.create({ + compile: { compileConfig: { projectDir } } + }); + + const result = runMainInVm(coreExecutionRequest); + + expect(asPlainObject(result.compile.compiledGraph.projectConfig)).deep.equals( + asPlainObject({ + warehouse: "bigquery", + defaultDatabase: "dataform" + }) + ); + }); + + // dataform.json for workflow settings is deprecated, but still currently supported. + test(`main succeeds when a valid dataform.json is present`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // tslint:disable-next-line: tsr-detect-non-literal-fs-filename + fs.writeFileSync(path.join(projectDir, "dataform.json"), VALID_DATAFORM_JSON); + const coreExecutionRequest = dataform.CoreExecutionRequest.create({ + compile: { compileConfig: { projectDir } } + }); + + const result = runMainInVm(coreExecutionRequest); + + expect(asPlainObject(result.compile.compiledGraph.projectConfig)).deep.equals( + asPlainObject({ + warehouse: "bigquery", + defaultDatabase: "dataform" + }) + ); + }); + + test(`main fails when no workflow settings file is present`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + const coreExecutionRequest = dataform.CoreExecutionRequest.create({ + compile: { compileConfig: { projectDir } } + }); + + expect(() => runMainInVm(coreExecutionRequest)).to.throw( + "Failed to resolve workflow_settings.yaml" + ); + }); + + test(`main fails when both workflow settings and dataform.json files are present`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // tslint:disable-next-line: tsr-detect-non-literal-fs-filename + fs.writeFileSync(path.join(projectDir, "dataform.json"), VALID_DATAFORM_JSON); + // tslint:disable-next-line: tsr-detect-non-literal-fs-filename + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + const coreExecutionRequest = dataform.CoreExecutionRequest.create({ + compile: { compileConfig: { projectDir } } + }); + + expect(() => runMainInVm(coreExecutionRequest)).to.throw( + "dataform.json has been deprecated and cannot be defined alongside workflow_settings.yaml" + ); + }); + + test(`main fails when workflow_settings.yaml is an invalid yaml file`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // tslint:disable-next-line: tsr-detect-non-literal-fs-filename + fs.writeFileSync(path.join(projectDir, "workflow_settings.yaml"), "&*19132sdS:asd:"); + const coreExecutionRequest = dataform.CoreExecutionRequest.create({ + compile: { compileConfig: { projectDir } } + }); + + expect(() => runMainInVm(coreExecutionRequest)).to.throw( + "workflow_settings.yaml contains invalid fields" + ); + }); + + test(`main fails when dataform.json is an invalid json file`, () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + // tslint:disable-next-line: tsr-detect-non-literal-fs-filename + fs.writeFileSync(path.join(projectDir, "dataform.json"), '{keyWithNoQuotes: "validValue"}'); + const coreExecutionRequest = dataform.CoreExecutionRequest.create({ + compile: { compileConfig: { projectDir } } + }); + + expect(() => runMainInVm(coreExecutionRequest)).to.throw( + "Unexpected token k in JSON at position 1" + ); + }); + }); +}); + +// A VM is needed when running main because Node functions like `require` are overridden. +function runMainInVm(coreExecutionRequest: dataform.CoreExecutionRequest) { + const projectDir = coreExecutionRequest.compile.compileConfig.projectDir; + + // Copy over the build Dataform Core that is set up as a node_modules directory. + fs.copySync(`${process.cwd()}/core/node_modules`, `${projectDir}/node_modules`); + + const compiler = compile as CompilerFunction; + // Then use vm2's native compiler integration to apply the compiler to files. + const nodeVm = new NodeVM({ + // Inheriting the console makes console.logs show when tests are running, which is useful for + // debugging. + console: "inherit", + wrapper: "none", + require: { + builtin: ["path"], + context: "sandbox", + external: true, + root: projectDir, + resolve: (moduleName, parentDirName) => + path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName) + }, + sourceExtensions: ["js", "sql", "sqlx", "yaml"], + compiler + }); + + const encodedCoreExecutionRequest = encode64(dataform.CoreExecutionRequest, coreExecutionRequest); + const vmIndexFileName = path.resolve(path.join(projectDir, "index.js")); + const encodedCoreExecutionResponse = nodeVm.run( + `return require("@dataform/core").main("${encodedCoreExecutionRequest}")`, + vmIndexFileName + ); + return decode64(dataform.CoreExecutionResponse, encodedCoreExecutionResponse); +} diff --git a/core/workflow_settings.ts b/core/workflow_settings.ts new file mode 100644 index 000000000..134bfdde2 --- /dev/null +++ b/core/workflow_settings.ts @@ -0,0 +1,46 @@ +import { dataform } from "df/protos/ts"; + +export function readWorkflowSettings(): dataform.ProjectConfig { + const workflowSettingsYaml = maybeRequire("workflow_settings.yaml"); + // `dataform.json` is deprecated; new versions of Dataform Core prefer `workflow_settings.yaml`. + const dataformJson = maybeRequire("dataform.json"); + + if (workflowSettingsYaml && dataformJson) { + throw Error( + "dataform.json has been deprecated and cannot be defined alongside workflow_settings.yaml" + ); + } + + if (workflowSettingsYaml) { + const workflowSettingsAsJson = workflowSettingsYaml.asJson(); + verifyWorkflowSettingsAsJson(workflowSettingsAsJson); + return dataform.ProjectConfig.create(workflowSettingsAsJson); + } + + if (dataformJson) { + verifyWorkflowSettingsAsJson(dataformJson); + return dataform.ProjectConfig.create(dataformJson); + } + + throw Error("Failed to resolve workflow_settings.yaml"); +} + +function verifyWorkflowSettingsAsJson(workflowSettingsAsJson?: object) { + // TODO(ekrekr): Implement a protobuf field validator. Protobufjs's verify method is not fit for + // purpose. + if (!workflowSettingsAsJson) { + throw Error("workflow_settings.yaml contains invalid fields"); + } +} + +function maybeRequire(file: string): any { + try { + // tslint:disable-next-line: tsr-detect-non-literal-require + return require(file); + } catch (e) { + if (e instanceof SyntaxError) { + throw e; + } + return undefined; + } +} diff --git a/package.json b/package.json index cecf918bf..fe558b4e4 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@types/glob": "^8.1.0", "@types/google-protobuf": "^3.2.7", "@types/js-beautify": "^1.8.1", + "@types/js-yaml": "^4.0.5", "@types/json-stable-stringify": "^1.0.32", "@types/long": "^4.0.0", "@types/moo": "^0.5.0", @@ -71,6 +72,7 @@ "grpc-web-client": "^0.5.0", "handy-redis": "^1.8.3", "js-beautify": "^1.10.2", + "js-yaml": "^4.1.0", "jsdoc": "^3.6.11", "json-stable-stringify": "^1.0.1", "long": "^4.0.0", diff --git a/packages/@dataform/core/BUILD b/packages/@dataform/core/BUILD index b0d9cbfd8..df1218aff 100644 --- a/packages/@dataform/core/BUILD +++ b/packages/@dataform/core/BUILD @@ -14,6 +14,7 @@ ts_library( ) externals = [ + "js-yaml", "protobufjs", "tarjan-graph", "semver", diff --git a/testing/BUILD b/testing/BUILD index 00e83e883..51f9ffaf9 100644 --- a/testing/BUILD +++ b/testing/BUILD @@ -6,18 +6,25 @@ load("//tools:ts_library.bzl", "ts_library") ts_library( name = "testing", - srcs = glob( - ["*.ts"], - exclude = ["*.spec.ts"], - ), + srcs = [ + "child_process.ts", + "fixtures.ts", + "hook.ts", + "index.ts", + "runner.ts", + "suite.ts", + "test.ts", + ], deps = [ "//:modules-fix", "@npm//@types/diff", "@npm//@types/json-stable-stringify", "@npm//@types/node", + "@npm//@types/rimraf", "@npm//chalk", "@npm//diff", "@npm//json-stable-stringify", + "@npm//rimraf", ], ) @@ -25,7 +32,7 @@ load("//testing:index.bzl", "ts_test_suite") ts_test_suite( name = "tests", - srcs = glob(["*.spec.ts"]), + srcs = ["index_test.ts"], deps = [ ":testing", "@npm//@types/chai", diff --git a/tests/utils/fixtures.ts b/testing/fixtures.ts similarity index 100% rename from tests/utils/fixtures.ts rename to testing/fixtures.ts diff --git a/testing/index.spec.ts b/testing/index_test.ts similarity index 100% rename from testing/index.spec.ts rename to testing/index_test.ts diff --git a/tests/core/core.spec.ts b/tests/core/core.spec.ts index bf21022d1..29a584a68 100644 --- a/tests/core/core.spec.ts +++ b/tests/core/core.spec.ts @@ -9,6 +9,8 @@ import { dataform } from "df/protos/ts"; import { suite, test } from "df/testing"; import { asPlainObject } from "df/tests/utils"; +// TODO(ekrekr): migrate the tests in this file to core/main_test.ts. + class TestConfigs { public static bigquery: dataform.IProjectConfig = { warehouse: "bigquery", diff --git a/tests/utils/BUILD b/tests/utils/BUILD index d5a42c37c..42c7914d8 100644 --- a/tests/utils/BUILD +++ b/tests/utils/BUILD @@ -1,14 +1,8 @@ load("//tools:ts_library.bzl", "ts_library") -package(default_visibility = ["//tests:__subpackages__", "//examples:__subpackages__"]) +package(default_visibility = ["//visibility:public"]) ts_library( name = "utils", srcs = glob(["**/*.ts"]), - deps = [ - "//testing", - "@npm//@types/node", - "@npm//@types/rimraf", - "@npm//rimraf", - ], ) diff --git a/yarn.lock b/yarn.lock index 1865fc193..10ac9bf89 100644 --- a/yarn.lock +++ b/yarn.lock @@ -871,6 +871,11 @@ resolved "https://registry.yarnpkg.com/@types/js-beautify/-/js-beautify-1.8.1.tgz#c210b3206bece04dc240b1deeec185f7b5b03534" integrity "sha1-whCzIGvs4E3CQLHe7sGF97WwNTQ= sha512-B1Br8yE27obcYvFx5ECZswT/947aAFNb9lHqnkUOhtOfvJqaa6Axibo4T+5G6iQlUfjgSd8am9R/9j9UBfRlrw==" +"@types/js-yaml@^4.0.5": + version "4.0.9" + resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.9.tgz#cd82382c4f902fed9691a2ed79ec68c5898af4c2" + integrity sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg== + "@types/json-schema@^7.0.5": version "7.0.7" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad" @@ -5280,6 +5285,13 @@ js-yaml@^3.13.1: argparse "^1.0.7" esprima "^4.0.0" +js-yaml@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.0.tgz#c1fb65f8f5017901cdd2c951864ba18458a10602" + integrity sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== + dependencies: + argparse "^2.0.1" + js2xmlparser@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/js2xmlparser/-/js2xmlparser-4.0.2.tgz#2a1fdf01e90585ef2ae872a01bc169c6a8d5e60a"