From 83f9457c491043486cd029f40fb8aaeb27e7b177 Mon Sep 17 00:00:00 2001 From: Elias Kassell Date: Wed, 15 Nov 2023 15:09:02 +0000 Subject: [PATCH] Remove gen index in favor of main (#1571) * Remove gen index * Fix semver * Simplify error message --- cli/api/commands/compile.ts | 21 +- cli/vm/BUILD | 2 + cli/vm/compile.ts | 101 +- cli/vm/create_config.ts | 61 - core/gen_index.ts | 72 - core/index.ts | 3 +- core/main.ts | 1 + protos/core.proto | 5 +- tests/api/BUILD | 2 - tests/api/projects.spec.ts | 1374 ++++++++--------- .../projects/backwards_compatibility/BUILD | 19 - .../backwards_compatibility/dataform.json | 7 - .../definitions/example.sql | 2 - .../definitions/example_assertion.assert.sql | 1 - .../includes/macros.js | 3 - 15 files changed, 744 insertions(+), 930 deletions(-) delete mode 100644 cli/vm/create_config.ts delete mode 100644 core/gen_index.ts delete mode 100644 tests/api/projects/backwards_compatibility/BUILD delete mode 100644 tests/api/projects/backwards_compatibility/dataform.json delete mode 100644 tests/api/projects/backwards_compatibility/definitions/example.sql delete mode 100644 tests/api/projects/backwards_compatibility/definitions/example_assertion.assert.sql delete mode 100644 tests/api/projects/backwards_compatibility/includes/macros.js diff --git a/cli/api/commands/compile.ts b/cli/api/commands/compile.ts index 12c136ee7..c3c51e602 100644 --- a/cli/api/commands/compile.ts +++ b/cli/api/commands/compile.ts @@ -42,27 +42,10 @@ export async function compile( ); } - if (compileConfig.useMain === null || compileConfig.useMain === undefined) { - try { - const packageJson = JSON.parse( - fs.readFileSync(`${compileConfig.projectDir}/package.json`, "utf8") - ); - const dataformCoreVersion = packageJson.dependencies["@dataform/core"]; - compileConfig.useMain = semver.subset(dataformCoreVersion, ">=2.0.4"); - } catch (e) { - // Silently catch any thrown Error. Do not attempt to use `main` compilation. - } - } - const result = await CompileChildProcess.forkProcess().compile(compileConfig); - let compileResult: dataform.CompiledGraph; - if (compileConfig.useMain) { - const decodedResult = decode64(dataform.CoreExecutionResponse, result); - compileResult = dataform.CompiledGraph.create(decodedResult.compile.compiledGraph); - } else { - compileResult = decode64(dataform.CompiledGraph, result); - } + const decodedResult = decode64(dataform.CoreExecutionResponse, result); + const compileResult = dataform.CompiledGraph.create(decodedResult.compile.compiledGraph); compileResult.tables.forEach(setOrValidateTableEnumType); return compileResult; diff --git a/cli/vm/BUILD b/cli/vm/BUILD index 4416d6e45..ebfbe044b 100644 --- a/cli/vm/BUILD +++ b/cli/vm/BUILD @@ -14,7 +14,9 @@ ts_library( "//protos:ts", "@npm//@types/glob", "@npm//@types/node", + "@npm//@types/semver", "@npm//glob", + "@npm//semver", "@npm//vm2", ], ) diff --git a/cli/vm/compile.ts b/cli/vm/compile.ts index 18d513d60..6faa564ed 100644 --- a/cli/vm/compile.ts +++ b/cli/vm/compile.ts @@ -1,17 +1,15 @@ +import * as glob from "glob"; import * as path from "path"; +import * as semver from "semver"; import { CompilerFunction, NodeVM } from "vm2"; -import { createCoreExecutionRequest, createGenIndexConfig } from "df/cli/vm/create_config"; +import { encode64 } from "df/common/protos"; import { dataform } from "df/protos/ts"; -function missingValidCorePackageError() { - return new Error( - `Could not find a recent installed version of @dataform/core in the project. Ensure packages are installed and upgrade to a recent version.` - ); -} export function compile(compileConfig: dataform.ICompileConfig) { const vmIndexFileName = path.resolve(path.join(compileConfig.projectDir, "index.js")); + // First retrieve a compiler function for vm2 to process files. const indexGeneratorVm = new NodeVM({ wrapper: "none", require: { @@ -21,19 +19,13 @@ export function compile(compileConfig: dataform.ICompileConfig) { builtin: ["path"] } }); + const compiler: CompilerFunction = runDataformCoreVmScript( + indexGeneratorVm, + vmIndexFileName, + 'return require("@dataform/core").compiler' + ); - const findCompiler = (): CompilerFunction => { - try { - return indexGeneratorVm.run('return require("@dataform/core").compiler', vmIndexFileName); - } catch (e) { - throw missingValidCorePackageError(); - } - }; - const compiler = findCompiler(); - if (!compiler) { - throw missingValidCorePackageError(); - } - + // Then use vm2's native compiler integration to apply the compiler to files. const userCodeVm = new NodeVM({ wrapper: "none", require: { @@ -48,31 +40,20 @@ export function compile(compileConfig: dataform.ICompileConfig) { compiler }); - if (compileConfig.useMain) { - try { - return userCodeVm.run( - `return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")`, - vmIndexFileName - ); - } catch (e) { - throw missingValidCorePackageError(); - } + const dataformCoreVersion: string = runDataformCoreVmScript( + userCodeVm, + vmIndexFileName, + 'return require("@dataform/core").version || "0.0.0"' + ); + if (semver.lt(dataformCoreVersion, "3.0.0")) { + throw new Error("@dataform/core ^3.0.0 required."); } - // Generate an index file and run it. - const findGenIndex = (): ((base64EncodedConfig: string) => string) => { - try { - return indexGeneratorVm.run( - 'return require("@dataform/core").indexFileGenerator', - vmIndexFileName - ); - } catch (e) { - throw missingValidCorePackageError(); - } - }; - const genIndex = findGenIndex(); - - return userCodeVm.run(genIndex(createGenIndexConfig(compileConfig)), vmIndexFileName); + return runDataformCoreVmScript( + userCodeVm, + vmIndexFileName, + `return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")` + ); } export function listenForCompileRequest() { @@ -90,6 +71,44 @@ export function listenForCompileRequest() { }); } +function missingValidCorePackageError() { + return new Error( + "Could not find a recent installed version of @dataform/core in the project. Ensure packages " + + "are installed and upgrade to a recent version." + ); +} + +function runDataformCoreVmScript(nodeVM: NodeVM, vmIndexFileName: string, script: string): any { + // Missing valid core package errors are thrown because if @dataform/core isn't installed, + // the properties of it can't be found. + const getResult = (): any => { + try { + return nodeVM.run(script, vmIndexFileName); + } catch (e) { + throw missingValidCorePackageError(); + } + }; + const result = getResult(); + if (!result) { + throw missingValidCorePackageError(); + } + return result as any; +} + if (require.main === module) { listenForCompileRequest(); } + +/** + * @returns a base64 encoded {@see dataform.CoreExecutionRequest} proto. + */ +function createCoreExecutionRequest(compileConfig: dataform.ICompileConfig): string { + const filePaths = Array.from( + new Set(glob.sync("!(node_modules)/**/*.*", { cwd: compileConfig.projectDir })) + ); + + return encode64(dataform.CoreExecutionRequest, { + // Add the list of file paths to the compile config if not already set. + compile: { compileConfig: { filePaths, ...compileConfig } } + }); +} diff --git a/cli/vm/create_config.ts b/cli/vm/create_config.ts deleted file mode 100644 index 888b51405..000000000 --- a/cli/vm/create_config.ts +++ /dev/null @@ -1,61 +0,0 @@ -import * as glob from "glob"; - -import { encode64 } from "df/common/protos"; -import { dataform } from "df/protos/ts"; - -export function createGenIndexConfig(compileConfig: dataform.ICompileConfig): string { - const includePaths: string[] = []; - glob - .sync("includes/*.js", { cwd: compileConfig.projectDir }) - .sort(alphabetically) - .forEach(path => { - if (includePaths.indexOf(path) < 0) { - includePaths.push(path); - } - }); - - const definitionPaths: string[] = []; - glob - .sync("definitions/**/*.{js,sql,sqlx}", { cwd: compileConfig.projectDir }) - .sort(alphabetically) - .forEach(path => { - if (definitionPaths.indexOf(path) < 0) { - definitionPaths.push(path); - } - }); - // Support projects that don't use the new project structure. - glob - .sync("models/**/*.{js,sql,sqlx}", { cwd: compileConfig.projectDir }) - .sort(alphabetically) - .forEach(path => { - if (definitionPaths.indexOf(path) < 0) { - definitionPaths.push(path); - } - }); - return encode64(dataform.GenerateIndexConfig, { - compileConfig, - includePaths, - definitionPaths - }); -} - -/** - * @returns a base64 encoded {@see dataform.CoreExecutionRequest} proto. - */ -export function createCoreExecutionRequest(compileConfig: dataform.ICompileConfig): string { - const filePaths = Array.from( - new Set( - glob.sync("!(node_modules)/**/*.*", { cwd: compileConfig.projectDir }).sort(alphabetically) - ) - ); - - return encode64(dataform.CoreExecutionRequest, { - // Add the list of file paths to the compile config if not already set. - compile: { compileConfig: { filePaths, ...compileConfig } } - }); -} - -// NOTE: this is used to sort results of `glob.sync()` above. -// This sort is only required for compatibility with @dataform/core <= 2.6.5. -// If/when the CLI drops support for those versions, we can remove the sorting. -const alphabetically = (a: string, b: string) => a.localeCompare(b); diff --git a/core/gen_index.ts b/core/gen_index.ts deleted file mode 100644 index dbffd545b..000000000 --- a/core/gen_index.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { decode64 } from "df/common/protos"; -import * as utils from "df/core/utils"; -import { dataform } from "df/protos/ts"; - -export function genIndex(base64EncodedConfig: string): string { - const config = decode64(dataform.GenerateIndexConfig, base64EncodedConfig); - - const includeRequires = config.includePaths - .map(path => { - return ` - try { global.${utils.baseFilename(path)} = require("./${path}"); } catch (e) { - session.compileError(e, "${path}"); - }`; - }) - .join("\n"); - const definitionRequires = config.definitionPaths - .map(path => { - return ` - try { require("./${path}"); } catch (e) { - session.compileError(e, "${path}"); - }`; - }) - .join("\n"); - - const projectOverridesJsonString = JSON.stringify( - dataform.ProjectConfig.create(config.compileConfig.projectConfigOverride).toJSON() - ); - - // NOTE: - // - The returned script must be valid JavaScript (not TypeScript) - // - The returned script may not require() any package that is not "@dataform/core" - return ` -// Read the project config. -const originalProjectConfig = require("./dataform.json"); - -// Stop using the deprecated 'gcloudProjectId' field. -if (!originalProjectConfig.defaultDatabase) { - originalProjectConfig.defaultDatabase = originalProjectConfig.gcloudProjectId; -} -delete originalProjectConfig.gcloudProjectId; - -let projectConfig = { ...originalProjectConfig }; - -// Merge in general project config overrides. -projectConfig = { ...projectConfig, ...${projectOverridesJsonString}, vars: { ...projectConfig.vars, ...${projectOverridesJsonString}.vars } }; - -// Initialize the compilation session. -const session = require("@dataform/core").session; -session.init("${config.compileConfig.projectDir.replace( - /\\/g, - "\\\\" - )}", projectConfig, originalProjectConfig); - -// Allow "includes" files to use the current session object. -global.dataform = session; - -// Require "includes" *.js files. -${includeRequires} - -// Bind various @dataform/core APIs to the 'global' object. -global.publish = session.publish.bind(session); -global.operate = session.operate.bind(session); -global.assert = session.assert.bind(session); -global.declare = session.declare.bind(session); -global.test = session.test.bind(session); - -// Require all "definitions" files (attaching them to the session). -${definitionRequires} - -// Return a base64 encoded proto via NodeVM. Returning a Uint8Array directly causes issues. -return session.compileToBase64();`; -} diff --git a/core/index.ts b/core/index.ts index 9d7f79734..ac9d69d6c 100644 --- a/core/index.ts +++ b/core/index.ts @@ -1,5 +1,4 @@ import { compile as compiler } from "df/core/compilers"; -import { genIndex as indexFileGenerator } from "df/core/gen_index"; import { main } from "df/core/main"; import { Session } from "df/core/session"; import { version } from "df/core/version"; @@ -20,4 +19,4 @@ const supportedFeatures = [dataform.SupportedFeatures.ARRAY_BUFFER_IPC]; // These exports constitute the public API of @dataform/core. // Changes to these will break @dataform/api, so take care! -export { compiler, indexFileGenerator, main, session, supportedFeatures, version }; +export { compiler, main, session, supportedFeatures, version }; diff --git a/core/main.ts b/core/main.ts index 39dd7e0aa..1fc1d08b7 100644 --- a/core/main.ts +++ b/core/main.ts @@ -79,6 +79,7 @@ export function main(coreExecutionRequest: Uint8Array | string): Uint8Array | st compileRequest.compileConfig.filePaths .filter(path => path.startsWith(`definitions${utils.pathSeperator}`)) .filter(path => path.endsWith(".js") || path.endsWith(".sqlx")) + .sort() .forEach(definitionPath => { try { // tslint:disable-next-line: tsr-detect-non-literal-require diff --git a/protos/core.proto b/protos/core.proto index c5a32b87a..f79db4084 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -56,10 +56,7 @@ message CompileConfig { // Override compilation timeout settings. int32 timeout_millis = 6; - // Whether to use the new main function instead of gen index. - bool use_main = 9; - - reserved 2, 4, 5, 7; + reserved 2, 4, 5, 7, 9; } message Target { diff --git a/tests/api/BUILD b/tests/api/BUILD index 522a41296..62ca8ccd7 100644 --- a/tests/api/BUILD +++ b/tests/api/BUILD @@ -4,8 +4,6 @@ ts_test_suite( name = "tests", srcs = glob(["**/*.ts"]), data = [ - "//tests/api/projects/backwards_compatibility:files", - "//tests/api/projects/backwards_compatibility:node_modules", "//tests/api/projects/common_v2:files", "//tests/api/projects/common_v2:node_modules", "//tests/api/projects/invalid_dataform_json:files", diff --git a/tests/api/projects.spec.ts b/tests/api/projects.spec.ts index 9795ea800..3b880ca6e 100644 --- a/tests/api/projects.spec.ts +++ b/tests/api/projects.spec.ts @@ -10,745 +10,725 @@ import { cleanSql } from "df/tests/utils"; suite("examples", () => { suite("common_v2 bigquery", async () => { - for (const useMain of [true, false]) { - for (const databaseSuffix of ["", "foo"]) { - for (const schemaSuffix of ["", "bar"]) { - const databaseWithSuffix = (database: string) => - databaseSuffix ? `${database}_${databaseSuffix}` : database; - const schemaWithSuffix = (schema: string) => - schemaSuffix ? `${schema}_${schemaSuffix}` : schema; - - test(`compiles with database suffix "${databaseSuffix}", schema suffix "${schemaSuffix}"`, async () => { - const graph = await compile({ - projectDir: path.resolve("tests/api/projects/common_v2"), - projectConfigOverride: { schemaSuffix, databaseSuffix, warehouse: "bigquery" }, - useMain - }); - expect( - graph.graphErrors.compilationErrors.map(({ fileName, message }) => ({ - fileName, - message - })) - ).deep.equals([ - { - fileName: "includes/example_ignore.js", - message: "publish is not defined" - }, - { - fileName: "definitions/has_compile_errors/assertion_with_bigquery.sqlx", - message: - 'Unexpected property "bigquery" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' - }, - { - fileName: "definitions/has_compile_errors/assertion_with_materialized.sqlx", - message: - 'Unexpected property "materialized" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' - }, - { - fileName: "definitions/has_compile_errors/assertion_with_output.sqlx", - message: - 'Unexpected property "hasOutput" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' - }, - { - fileName: "definitions/has_compile_errors/assertion_with_postops.sqlx", - message: "Actions may only include post_operations if they create a dataset." - }, - { - fileName: "definitions/has_compile_errors/assertion_with_preops.sqlx", - message: "Actions may only include pre_operations if they create a dataset." - }, - { - fileName: "definitions/has_compile_errors/protected_assertion.sqlx", - message: - "Actions may only specify 'protected: true' if they are of type 'incremental'." - }, - { - fileName: "definitions/has_compile_errors/protected_assertion.sqlx", - message: - 'Unexpected property "protected" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' - }, - { - fileName: "definitions/has_compile_errors/view_with_incremental.sqlx", - message: - "Actions may only include incremental_where if they are of type 'incremental'." - }, - { - fileName: "definitions/has_compile_errors/view_with_multiple_statements.sqlx", - message: - "Actions may only contain more than one SQL statement if they are of type 'operations'." - }, - { - fileName: "definitions/has_compile_errors/view_with_semi_colon_at_end.sqlx", - message: "Semi-colons are not allowed at the end of SQL statements." - }, - { - fileName: "definitions/has_compile_errors/table_with_materialized.sqlx", - message: "The 'materialized' option is only valid for BigQuery views" - }, - { - fileName: "definitions/has_compile_errors/view_without_hermetic.sqlx", - message: - "Zero-dependency actions which create datasets are required to explicitly declare 'hermetic: (true|false)' when run caching is turned on." - } - ]); - - // Check JS blocks get processed. - const exampleJsBlocks = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_js_blocks" - ) - ); - expect(exampleJsBlocks.type).equals("table"); - expect(exampleJsBlocks.enumType).equals(dataform.TableType.TABLE); - expect(exampleJsBlocks.query.trim()).equals("select 1 as foo"); - - // Check we can import and use an external package. - const exampleIncremental = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_incremental" - ) - ); - expect(exampleIncremental.protected).eql(true); - expect(exampleIncremental.query.trim()).equals("select current_timestamp() as ts"); - expect(exampleIncremental.where.trim()).equals( - `ts > (select max(ts) from \`${dotJoined( + for (const databaseSuffix of ["", "foo"]) { + for (const schemaSuffix of ["", "bar"]) { + const databaseWithSuffix = (database: string) => + databaseSuffix ? `${database}_${databaseSuffix}` : database; + const schemaWithSuffix = (schema: string) => + schemaSuffix ? `${schema}_${schemaSuffix}` : schema; + + test(`compiles with database suffix "${databaseSuffix}", schema suffix "${schemaSuffix}"`, async () => { + const graph = await compile({ + projectDir: path.resolve("tests/api/projects/common_v2"), + projectConfigOverride: { schemaSuffix, databaseSuffix, warehouse: "bigquery" } + }); + expect( + graph.graphErrors.compilationErrors.map(({ fileName, message }) => ({ + fileName, + message + })) + ).deep.equals([ + { + fileName: "includes/example_ignore.js", + message: "publish is not defined" + }, + { + fileName: "definitions/has_compile_errors/assertion_with_bigquery.sqlx", + message: + 'Unexpected property "bigquery" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' + }, + { + fileName: "definitions/has_compile_errors/assertion_with_materialized.sqlx", + message: + 'Unexpected property "materialized" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' + }, + { + fileName: "definitions/has_compile_errors/assertion_with_output.sqlx", + message: + 'Unexpected property "hasOutput" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' + }, + { + fileName: "definitions/has_compile_errors/assertion_with_postops.sqlx", + message: "Actions may only include post_operations if they create a dataset." + }, + { + fileName: "definitions/has_compile_errors/assertion_with_preops.sqlx", + message: "Actions may only include pre_operations if they create a dataset." + }, + { + fileName: "definitions/has_compile_errors/protected_assertion.sqlx", + message: + "Actions may only specify 'protected: true' if they are of type 'incremental'." + }, + { + fileName: "definitions/has_compile_errors/protected_assertion.sqlx", + message: + 'Unexpected property "protected" in assertion config. Supported properties are: ["database","dependencies","description","disabled","hermetic","name","schema","tags","type"]' + }, + { + fileName: "definitions/has_compile_errors/view_with_incremental.sqlx", + message: + "Actions may only include incremental_where if they are of type 'incremental'." + }, + { + fileName: "definitions/has_compile_errors/view_with_multiple_statements.sqlx", + message: + "Actions may only contain more than one SQL statement if they are of type 'operations'." + }, + { + fileName: "definitions/has_compile_errors/view_with_semi_colon_at_end.sqlx", + message: "Semi-colons are not allowed at the end of SQL statements." + }, + { + fileName: "definitions/has_compile_errors/table_with_materialized.sqlx", + message: "The 'materialized' option is only valid for BigQuery views" + }, + { + fileName: "definitions/has_compile_errors/view_without_hermetic.sqlx", + message: + "Zero-dependency actions which create datasets are required to explicitly declare 'hermetic: (true|false)' when run caching is turned on." + } + ]); + + // Check JS blocks get processed. + const exampleJsBlocks = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "example_incremental" - )}\`) or (select max(ts) from \`${dotJoined( + "example_js_blocks" + ) + ); + expect(exampleJsBlocks.type).equals("table"); + expect(exampleJsBlocks.enumType).equals(dataform.TableType.TABLE); + expect(exampleJsBlocks.query.trim()).equals("select 1 as foo"); + + // Check we can import and use an external package. + const exampleIncremental = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), "example_incremental" - )}\`) is null` - ); - - const exampleIsIncremental = graph.tables.filter( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_is_incremental" - ) - )[0]; - expect(cleanSql(exampleIsIncremental.query.trim())).equals( - "select * from (select current_timestamp() as ts)" - ); - expect(cleanSql(exampleIsIncremental.incrementalQuery)).equals( - cleanSql( - `select * from (select current_timestamp() as ts) + ) + ); + expect(exampleIncremental.protected).eql(true); + expect(exampleIncremental.query.trim()).equals("select current_timestamp() as ts"); + expect(exampleIncremental.where.trim()).equals( + `ts > (select max(ts) from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_incremental" + )}\`) or (select max(ts) from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_incremental" + )}\`) is null` + ); + + const exampleIsIncremental = graph.tables.filter( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_is_incremental" + ) + )[0]; + expect(cleanSql(exampleIsIncremental.query.trim())).equals( + "select * from (select current_timestamp() as ts)" + ); + expect(cleanSql(exampleIsIncremental.incrementalQuery)).equals( + cleanSql( + `select * from (select current_timestamp() as ts) where ts > (select max(ts) from \`${dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), "example_is_incremental" )}\`) or (select max(ts) from \`${dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_is_incremental" - )}\`) is null` + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_is_incremental" + )}\`) is null` + ) + ); + + expect(exampleIsIncremental.incrementalPreOps).to.eql(["\n select 1\n"]); + expect(exampleIsIncremental.incrementalPostOps).to.eql(["\n select 15\n"]); + + // Check tables defined in includes are not included. + const exampleIgnore = graph.tables.find( + (t: dataform.ITable) => targetAsReadableString(t.target) === "example_ignore" + ); + expect(exampleIgnore).equal(undefined); + const exampleIgnore2 = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_ignore" ) - ); - - expect(exampleIsIncremental.incrementalPreOps).to.eql(["\n select 1\n"]); - expect(exampleIsIncremental.incrementalPostOps).to.eql(["\n select 15\n"]); - - // Check tables defined in includes are not included. - const exampleIgnore = graph.tables.find( - (t: dataform.ITable) => targetAsReadableString(t.target) === "example_ignore" - ); - expect(exampleIgnore).equal(undefined); - const exampleIgnore2 = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_ignore" - ) - ); - expect(exampleIgnore2).equal(undefined); - - // Check SQL files with raw back-ticks get escaped. - const exampleBackticks = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_backticks" - ) - ); - expect(cleanSql(exampleBackticks.query)).equals( - "select * from `tada-analytics.df_integration_test.sample_data`" - ); - expect(exampleBackticks.preOps).to.eql([ - '\n GRANT SELECT ON `tada-analytics.df_integration_test.sample_data` TO GROUP "allusers@dataform.co"\n' - ]); - expect(exampleBackticks.postOps).to.eql([]); - - // Check deferred calls to table resolve to the correct definitions file. - const exampleDeferred = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_deferred" - ) - ); - expect(exampleDeferred.fileName).includes("definitions/example_deferred.js"); - - // Check inline tables - const exampleInline = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_inline" - ) - ); - expect(exampleInline.type).equals("inline"); - expect(exampleInline.enumType).equals(dataform.TableType.INLINE); - expect(exampleInline.query.trim()).equals( - `select * from \`${dotJoined( + ); + expect(exampleIgnore2).equal(undefined); + + // Check SQL files with raw back-ticks get escaped. + const exampleBackticks = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "sample_data" - )}\`` - ); - expect(exampleInline.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "sample_data" - }) - ]); - - // Check view - const exampleView = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_view" - ) - ); - expect(exampleView.type).equals("view"); - expect(exampleView.enumType).equals(dataform.TableType.VIEW); - expect(exampleView.query.trim()).equals( - `select * from \`${dotJoined( + "example_backticks" + ) + ); + expect(cleanSql(exampleBackticks.query)).equals( + "select * from `tada-analytics.df_integration_test.sample_data`" + ); + expect(exampleBackticks.preOps).to.eql([ + '\n GRANT SELECT ON `tada-analytics.df_integration_test.sample_data` TO GROUP "allusers@dataform.co"\n' + ]); + expect(exampleBackticks.postOps).to.eql([]); + + // Check deferred calls to table resolve to the correct definitions file. + const exampleDeferred = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "sample_data" - )}\`\n` + - `inner join select * from \`${dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("override_schema"), - "override_schema_example" - )}\`\n` + - `inner join select * from \`${dotJoined( - databaseWithSuffix("override_database"), - schemaWithSuffix("df_integration_test"), - "override_database_example" - )}\`` - ); - expect(exampleView.target).deep.equals( - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "example_view" - }) - ); - expect(exampleView.canonicalTarget).deep.equals( - dataform.Target.create({ - database: "tada-analytics", - schema: "df_integration_test", - name: "example_view" - }) - ); - expect(exampleView.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "sample_data" - }), - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("override_schema"), - name: "override_schema_example" - }), - dataform.Target.create({ - database: databaseWithSuffix("override_database"), - schema: schemaWithSuffix("df_integration_test"), - name: "override_database_example" - }) - ]); - expect(exampleView.tags).to.eql([]); - - // Check materialized view - const exampleMaterializedView = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_materialized_view" - ) - ); - expect(exampleMaterializedView.type).equals("view"); - expect(exampleMaterializedView.enumType).equals(dataform.TableType.VIEW); - expect(exampleMaterializedView.materialized).equals(true); - expect(exampleMaterializedView.query.trim()).equals( - `select * from \`${dotJoined( + "example_deferred" + ) + ); + expect(exampleDeferred.fileName).includes("definitions/example_deferred.js"); + + // Check inline tables + const exampleInline = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "sample_data" - )}\`\n` + `group by 1` - ); - expect(exampleMaterializedView.target).deep.equals( - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "example_materialized_view" - }) - ); - expect(exampleMaterializedView.canonicalTarget).deep.equals( - dataform.Target.create({ - database: "tada-analytics", - schema: "df_integration_test", - name: "example_materialized_view" - }) - ); - expect(exampleMaterializedView.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "sample_data" - }) - ]); - expect(exampleMaterializedView.tags).to.eql([]); - - // Check table - const exampleTable = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_table" - ) - ); - expect(exampleTable.type).equals("table"); - expect(exampleTable.enumType).equals(dataform.TableType.TABLE); - expect(exampleTable.query.trim()).equals( - `select * from \`${dotJoined( + "example_inline" + ) + ); + expect(exampleInline.type).equals("inline"); + expect(exampleInline.enumType).equals(dataform.TableType.INLINE); + expect(exampleInline.query.trim()).equals( + `select * from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "sample_data" + )}\`` + ); + expect(exampleInline.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "sample_data" + }) + ]); + + // Check view + const exampleView = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "sample_data" - )}\`\n\n-- here \${"is"} a \`comment\n\n/* \${"another"} \` backtick \` containing \`\`\`comment */` - ); - expect(exampleTable.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "sample_data" - }) - ]); - expect(exampleTable.preOps).to.eql([]); - expect(exampleTable.postOps).to.eql([ - `\n GRANT SELECT ON \`${dotJoined( + "example_view" + ) + ); + expect(exampleView.type).equals("view"); + expect(exampleView.enumType).equals(dataform.TableType.VIEW); + expect(exampleView.query.trim()).equals( + `select * from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "sample_data" + )}\`\n` + + `inner join select * from \`${dotJoined( databaseWithSuffix("tada-analytics"), + schemaWithSuffix("override_schema"), + "override_schema_example" + )}\`\n` + + `inner join select * from \`${dotJoined( + databaseWithSuffix("override_database"), schemaWithSuffix("df_integration_test"), - "example_table" - )}\` TO GROUP "allusers@dataform.co"\n`, - `\n GRANT SELECT ON \`${dotJoined( + "override_database_example" + )}\`` + ); + expect(exampleView.target).deep.equals( + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "example_view" + }) + ); + expect(exampleView.canonicalTarget).deep.equals( + dataform.Target.create({ + database: "tada-analytics", + schema: "df_integration_test", + name: "example_view" + }) + ); + expect(exampleView.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "sample_data" + }), + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("override_schema"), + name: "override_schema_example" + }), + dataform.Target.create({ + database: databaseWithSuffix("override_database"), + schema: schemaWithSuffix("df_integration_test"), + name: "override_database_example" + }) + ]); + expect(exampleView.tags).to.eql([]); + + // Check materialized view + const exampleMaterializedView = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "example_table" - )}\` TO GROUP "otherusers@dataform.co"\n` - ]); - expect(exampleTable.tags).to.eql([]); - - // Check Table with tags - const exampleTableWithTags = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_table_with_tags" - ) - ); - expect(exampleTableWithTags.disabled).eql(true); - expect(exampleTableWithTags.tags).to.eql(["tag1", "tag2", "tag3"]); - - // Check table-with-tags's unique key assertion - const exampleTableWithTagsUniqueKeyAssertion = graph.assertions.filter( - t => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test_assertions"), - "df_integration_test_example_table_with_tags_assertions_uniqueKey_0" - ) - )[0]; - expect(exampleTableWithTagsUniqueKeyAssertion.disabled).eql(true); - expect(cleanSql(exampleTableWithTagsUniqueKeyAssertion.query)).equals( - `select * from (select sample, count(1) as index_row_count from \`${dotJoined( + "example_materialized_view" + ) + ); + expect(exampleMaterializedView.type).equals("view"); + expect(exampleMaterializedView.enumType).equals(dataform.TableType.VIEW); + expect(exampleMaterializedView.materialized).equals(true); + expect(exampleMaterializedView.query.trim()).equals( + `select * from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "sample_data" + )}\`\n` + `group by 1` + ); + expect(exampleMaterializedView.target).deep.equals( + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "example_materialized_view" + }) + ); + expect(exampleMaterializedView.canonicalTarget).deep.equals( + dataform.Target.create({ + database: "tada-analytics", + schema: "df_integration_test", + name: "example_materialized_view" + }) + ); + expect(exampleMaterializedView.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "sample_data" + }) + ]); + expect(exampleMaterializedView.tags).to.eql([]); + + // Check table + const exampleTable = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "example_table_with_tags" - )}\` group by sample) as data where index_row_count > 1` - ); - expect(exampleTableWithTagsUniqueKeyAssertion.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "example_table_with_tags" - }) - ]); - expect(exampleTableWithTagsUniqueKeyAssertion.tags).eql(["tag1", "tag2", "tag3"]); - - // Check table-with-tags's row conditions assertion - const exampleTableWithTagsRowConditionsAssertion = graph.assertions.filter( - t => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test_assertions"), - "df_integration_test_example_table_with_tags_assertions_rowConditions" - ) - )[0]; - expect(exampleTableWithTagsRowConditionsAssertion.disabled).eql(true); - expect(cleanSql(exampleTableWithTagsRowConditionsAssertion.query)).equals( - `select 'sample is not null' as failing_row_condition, * from \`${dotJoined( + "example_table" + ) + ); + expect(exampleTable.type).equals("table"); + expect(exampleTable.enumType).equals(dataform.TableType.TABLE); + expect(exampleTable.query.trim()).equals( + `select * from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "sample_data" + )}\`\n\n-- here \${"is"} a \`comment\n\n/* \${"another"} \` backtick \` containing \`\`\`comment */` + ); + expect(exampleTable.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "sample_data" + }) + ]); + expect(exampleTable.preOps).to.eql([]); + expect(exampleTable.postOps).to.eql([ + `\n GRANT SELECT ON \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_table" + )}\` TO GROUP "allusers@dataform.co"\n`, + `\n GRANT SELECT ON \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_table" + )}\` TO GROUP "otherusers@dataform.co"\n` + ]); + expect(exampleTable.tags).to.eql([]); + + // Check Table with tags + const exampleTableWithTags = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), "example_table_with_tags" - )}\` where not (sample is not null)` - ); - expect(exampleTableWithTagsRowConditionsAssertion.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "example_table_with_tags" - }) - ]); - - // Check sample data - const exampleSampleData = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "sample_data" - ) - ); - expect(exampleSampleData.type).equals("view"); - expect(exampleSampleData.enumType).equals(dataform.TableType.VIEW); - expect(exampleSampleData.query.trim()).equals( - "select 1 as sample union all\nselect 2 as sample union all\nselect 3 as sample" - ); - expect(exampleSampleData.preOps).eql([]); - expect(exampleSampleData.dependencyTargets).eql([]); - expect(exampleSampleData.actionDescriptor).to.eql( - dataform.ActionDescriptor.create({ - description: "This is some sample data.", - columns: [ - dataform.ColumnDescriptor.create({ - description: "Sample integers.", - path: ["sample"] - }) - ] - }) - ); - - // Check database override defined in "config {}". - const exampleUsingOverriddenDatabase = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("override_database"), - schemaWithSuffix("df_integration_test"), - "override_database_example" - ) - ); - - expect(exampleUsingOverriddenDatabase.target.database).equals( - databaseWithSuffix("override_database") - ); - expect(exampleUsingOverriddenDatabase.type).equals("view"); - expect(exampleUsingOverriddenDatabase.enumType).equals(dataform.TableType.VIEW); - expect(exampleUsingOverriddenDatabase.query.trim()).equals( - "select 1 as test_database_override" - ); - - // Check schema overrides defined in "config {}" - const exampleUsingOverriddenSchema = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("override_schema"), - "override_schema_example" - ) - ); - - expect(exampleUsingOverriddenSchema.target.schema).equals( - schemaWithSuffix("override_schema") - ); - expect(exampleUsingOverriddenSchema.type).equals("view"); - expect(exampleUsingOverriddenSchema.enumType).equals(dataform.TableType.VIEW); - expect(exampleUsingOverriddenSchema.query.trim()).equals( - "select 1 as test_schema_override" - ); - - // Check schema overrides defined in "config {}" -- case with schema unchanged - const exampleUsingOverriddenSchemaUnchanged = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "override_schema_example_unchanged" - ) - ); - - expect(exampleUsingOverriddenSchemaUnchanged.target.schema).equals( - schemaWithSuffix("df_integration_test") - ); - expect(exampleUsingOverriddenSchemaUnchanged.type).equals("view"); - expect(exampleUsingOverriddenSchemaUnchanged.enumType).equals(dataform.TableType.VIEW); - expect(exampleUsingOverriddenSchemaUnchanged.query.trim()).equals( - "select 1 as test_schema_override" - ); - - // Check assertion - const exampleAssertion = graph.assertions.find( - (a: dataform.IAssertion) => - targetAsReadableString(a.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("hi_there"), - "example_assertion" - ) - ); - expect(exampleAssertion.target.schema).equals(schemaWithSuffix("hi_there")); - expect(exampleAssertion.query.trim()).equals( - `select * from \`${dotJoined( + ) + ); + expect(exampleTableWithTags.disabled).eql(true); + expect(exampleTableWithTags.tags).to.eql(["tag1", "tag2", "tag3"]); + + // Check table-with-tags's unique key assertion + const exampleTableWithTagsUniqueKeyAssertion = graph.assertions.filter( + t => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test_assertions"), + "df_integration_test_example_table_with_tags_assertions_uniqueKey_0" + ) + )[0]; + expect(exampleTableWithTagsUniqueKeyAssertion.disabled).eql(true); + expect(cleanSql(exampleTableWithTagsUniqueKeyAssertion.query)).equals( + `select * from (select sample, count(1) as index_row_count from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_table_with_tags" + )}\` group by sample) as data where index_row_count > 1` + ); + expect(exampleTableWithTagsUniqueKeyAssertion.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "example_table_with_tags" + }) + ]); + expect(exampleTableWithTagsUniqueKeyAssertion.tags).eql(["tag1", "tag2", "tag3"]); + + // Check table-with-tags's row conditions assertion + const exampleTableWithTagsRowConditionsAssertion = graph.assertions.filter( + t => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test_assertions"), + "df_integration_test_example_table_with_tags_assertions_rowConditions" + ) + )[0]; + expect(exampleTableWithTagsRowConditionsAssertion.disabled).eql(true); + expect(cleanSql(exampleTableWithTagsRowConditionsAssertion.query)).equals( + `select 'sample is not null' as failing_row_condition, * from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_table_with_tags" + )}\` where not (sample is not null)` + ); + expect(exampleTableWithTagsRowConditionsAssertion.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "example_table_with_tags" + }) + ]); + + // Check sample data + const exampleSampleData = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), "sample_data" - )}\` where sample = 100` - ); - expect(exampleAssertion.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "sample_data" - }) - ]); - expect(exampleAssertion.tags).to.eql([]); - expect(exampleAssertion.actionDescriptor).to.eql( - dataform.ActionDescriptor.create({ - description: "An example assertion looking for incorrect 'sample' values." - }) - ); - - // Check Assertion with tags - const exampleAssertionWithTags = graph.assertions.find( - (a: dataform.IAssertion) => - targetAsReadableString(a.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test_assertions"), - "example_assertion_with_tags" - ) - ); - expect(exampleAssertionWithTags.target.schema).equals( - schemaWithSuffix("df_integration_test_assertions") - ); - expect(exampleAssertionWithTags.tags).to.eql(["tag1", "tag2"]); - - // Check example operations file - const exampleOperations = graph.operations.find( - (o: dataform.IOperation) => - targetAsReadableString(o.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_operations" - ) - ); - expect(exampleOperations.hasOutput).equals(false); - expect(exampleOperations.queries).to.eql([ - "\n\nCREATE OR REPLACE VIEW someschema.someview AS (SELECT 1 AS test)\n", - `\nDROP VIEW IF EXISTS \`${dotJoined( + ) + ); + expect(exampleSampleData.type).equals("view"); + expect(exampleSampleData.enumType).equals(dataform.TableType.VIEW); + expect(exampleSampleData.query.trim()).equals( + "select 1 as sample union all\nselect 2 as sample union all\nselect 3 as sample" + ); + expect(exampleSampleData.preOps).eql([]); + expect(exampleSampleData.dependencyTargets).eql([]); + expect(exampleSampleData.actionDescriptor).to.eql( + dataform.ActionDescriptor.create({ + description: "This is some sample data.", + columns: [ + dataform.ColumnDescriptor.create({ + description: "Sample integers.", + path: ["sample"] + }) + ] + }) + ); + + // Check database override defined in "config {}". + const exampleUsingOverriddenDatabase = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("override_database"), + schemaWithSuffix("df_integration_test"), + "override_database_example" + ) + ); + + expect(exampleUsingOverriddenDatabase.target.database).equals( + databaseWithSuffix("override_database") + ); + expect(exampleUsingOverriddenDatabase.type).equals("view"); + expect(exampleUsingOverriddenDatabase.enumType).equals(dataform.TableType.VIEW); + expect(exampleUsingOverriddenDatabase.query.trim()).equals( + "select 1 as test_database_override" + ); + + // Check schema overrides defined in "config {}" + const exampleUsingOverriddenSchema = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("override_schema"), "override_schema_example" - )}\`\n`, - `\nDROP VIEW IF EXISTS \`${dotJoined( - databaseWithSuffix("override_database"), + ) + ); + + expect(exampleUsingOverriddenSchema.target.schema).equals( + schemaWithSuffix("override_schema") + ); + expect(exampleUsingOverriddenSchema.type).equals("view"); + expect(exampleUsingOverriddenSchema.enumType).equals(dataform.TableType.VIEW); + expect(exampleUsingOverriddenSchema.query.trim()).equals( + "select 1 as test_schema_override" + ); + + // Check schema overrides defined in "config {}" -- case with schema unchanged + const exampleUsingOverriddenSchemaUnchanged = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), - "override_database_example" - )}\`\n` - ]); - expect(exampleOperations.dependencyTargets).eql([ - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("override_schema"), - name: "override_schema_example" - }), - dataform.Target.create({ - database: databaseWithSuffix("override_database"), - schema: schemaWithSuffix("df_integration_test"), - name: "override_database_example" - }), - dataform.Target.create({ - database: databaseWithSuffix("tada-analytics"), - schema: schemaWithSuffix("df_integration_test"), - name: "sample_data" - }) - ]); - expect(exampleOperations.tags).to.eql([]); - - // Check example operation with output. - const exampleOperationWithOutput = graph.operations.find( - (o: dataform.IOperation) => - targetAsReadableString(o.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_operation_with_output" - ) - ); - expect(exampleOperationWithOutput.target.schema).equals( - schemaWithSuffix("df_integration_test") - ); - expect(exampleOperationWithOutput.target.name).equals("example_operation_with_output"); - expect(exampleOperationWithOutput.queries).to.eql([ - `\nCREATE OR REPLACE VIEW \`${dotJoined( + "override_schema_example_unchanged" + ) + ); + + expect(exampleUsingOverriddenSchemaUnchanged.target.schema).equals( + schemaWithSuffix("df_integration_test") + ); + expect(exampleUsingOverriddenSchemaUnchanged.type).equals("view"); + expect(exampleUsingOverriddenSchemaUnchanged.enumType).equals(dataform.TableType.VIEW); + expect(exampleUsingOverriddenSchemaUnchanged.query.trim()).equals( + "select 1 as test_schema_override" + ); + + // Check assertion + const exampleAssertion = graph.assertions.find( + (a: dataform.IAssertion) => + targetAsReadableString(a.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("hi_there"), + "example_assertion" + ) + ); + expect(exampleAssertion.target.schema).equals(schemaWithSuffix("hi_there")); + expect(exampleAssertion.query.trim()).equals( + `select * from \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "sample_data" + )}\` where sample = 100` + ); + expect(exampleAssertion.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "sample_data" + }) + ]); + expect(exampleAssertion.tags).to.eql([]); + expect(exampleAssertion.actionDescriptor).to.eql( + dataform.ActionDescriptor.create({ + description: "An example assertion looking for incorrect 'sample' values." + }) + ); + + // Check Assertion with tags + const exampleAssertionWithTags = graph.assertions.find( + (a: dataform.IAssertion) => + targetAsReadableString(a.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test_assertions"), + "example_assertion_with_tags" + ) + ); + expect(exampleAssertionWithTags.target.schema).equals( + schemaWithSuffix("df_integration_test_assertions") + ); + expect(exampleAssertionWithTags.tags).to.eql(["tag1", "tag2"]); + + // Check example operations file + const exampleOperations = graph.operations.find( + (o: dataform.IOperation) => + targetAsReadableString(o.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_operations" + ) + ); + expect(exampleOperations.hasOutput).equals(false); + expect(exampleOperations.queries).to.eql([ + "\n\nCREATE OR REPLACE VIEW someschema.someview AS (SELECT 1 AS test)\n", + `\nDROP VIEW IF EXISTS \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("override_schema"), + "override_schema_example" + )}\`\n`, + `\nDROP VIEW IF EXISTS \`${dotJoined( + databaseWithSuffix("override_database"), + schemaWithSuffix("df_integration_test"), + "override_database_example" + )}\`\n` + ]); + expect(exampleOperations.dependencyTargets).eql([ + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("override_schema"), + name: "override_schema_example" + }), + dataform.Target.create({ + database: databaseWithSuffix("override_database"), + schema: schemaWithSuffix("df_integration_test"), + name: "override_database_example" + }), + dataform.Target.create({ + database: databaseWithSuffix("tada-analytics"), + schema: schemaWithSuffix("df_integration_test"), + name: "sample_data" + }) + ]); + expect(exampleOperations.tags).to.eql([]); + + // Check example operation with output. + const exampleOperationWithOutput = graph.operations.find( + (o: dataform.IOperation) => + targetAsReadableString(o.target) === + dotJoined( databaseWithSuffix("tada-analytics"), schemaWithSuffix("df_integration_test"), "example_operation_with_output" - )}\` AS (SELECT * FROM \`some_database_name.some_external_schema_name.very_important_external_table\`)` - ]); - expect(exampleOperationWithOutput.dependencyTargets).eql([ - dataform.Target.create({ - database: "some_database_name", - schema: "some_external_schema_name", - name: "very_important_external_table" - }) - ]); - expect(exampleOperationWithOutput.actionDescriptor).to.eql( - dataform.ActionDescriptor.create({ - description: "An example operations file which outputs a dataset.", - columns: [ - dataform.ColumnDescriptor.create({ - description: "Just 1!", - path: ["TEST"] - }) - ] - }) - ); - - // Check Operation with tags - const exampleOperationsWithTags = graph.operations.find( - (o: dataform.IOperation) => - targetAsReadableString(o.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_operations_with_tags" - ) - ); - expect(exampleOperationsWithTags.tags).to.eql(["tag1"]); - - // Check declaration. - const exampleDeclaration = graph.declarations.find( - d => - targetAsReadableString(d.target) === - "some_database_name.some_external_schema_name.very_important_external_table" - ); - expect(exampleDeclaration.target).eql( - dataform.Target.create({ - database: "some_database_name", - schema: "some_external_schema_name", - name: "very_important_external_table" - }) - ); - expect(exampleDeclaration.actionDescriptor.description).to.equal( - "This table is not generated by Dataform!" - ); - - // Check testcases. - const testCase = graph.tests.find(t => t.name === "example_test_case"); - expect(testCase.testQuery.trim()).equals( - "select * from (\n select 'hi' as faked union all\n select 'ben' as faked union all\n select 'sup?' as faked\n)\n\n-- here ${\"is\"} a `comment\n\n/* ${\"another\"} ` backtick ` containing ```comment */" - ); - expect(testCase.expectedOutputQuery.trim()).equals( - "select 'hi' as faked union all\nselect 'ben' as faked union all\nselect 'sup?' as faked" - ); - - const testCaseFQ = graph.tests.find(t => t.name === "example_test_case_fq_ref"); - expect(testCaseFQ.testQuery.trim()).equals( - "select * from (\n select 'hi' as faked union all\n select 'ben' as faked union all\n select 'sup?' as faked\n)\n\n-- here ${\"is\"} a `comment\n\n/* ${\"another\"} ` backtick ` containing ```comment */" - ); - expect(testCaseFQ.expectedOutputQuery.trim()).equals( - "select 'hi' as faked union all\nselect 'ben' as faked union all\nselect 'sup?' as faked" - ); - - // Check double backslashes don't get converted to singular. - const exampleDoubleBackslash = graph.tables.find( - (t: dataform.ITable) => - targetAsReadableString(t.target) === - dotJoined( - databaseWithSuffix("tada-analytics"), - schemaWithSuffix("df_integration_test"), - "example_double_backslash" - ) - ); - expect(cleanSql(exampleDoubleBackslash.query)).equals( - "select * from regexp_extract('01a_data_engine', '^(\\\\d{2}\\\\w)') select * from regexp_extract('01a_data_engine', r'^(\\d{2}\\w)')" - ); - expect(cleanSql(exampleDoubleBackslash.preOps[0])).equals( - "select * from regexp_extract('\\\\\\\\', '\\\\')" - ); - }); - } + ) + ); + expect(exampleOperationWithOutput.target.schema).equals( + schemaWithSuffix("df_integration_test") + ); + expect(exampleOperationWithOutput.target.name).equals("example_operation_with_output"); + expect(exampleOperationWithOutput.queries).to.eql([ + `\nCREATE OR REPLACE VIEW \`${dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_operation_with_output" + )}\` AS (SELECT * FROM \`some_database_name.some_external_schema_name.very_important_external_table\`)` + ]); + expect(exampleOperationWithOutput.dependencyTargets).eql([ + dataform.Target.create({ + database: "some_database_name", + schema: "some_external_schema_name", + name: "very_important_external_table" + }) + ]); + expect(exampleOperationWithOutput.actionDescriptor).to.eql( + dataform.ActionDescriptor.create({ + description: "An example operations file which outputs a dataset.", + columns: [ + dataform.ColumnDescriptor.create({ + description: "Just 1!", + path: ["TEST"] + }) + ] + }) + ); + + // Check Operation with tags + const exampleOperationsWithTags = graph.operations.find( + (o: dataform.IOperation) => + targetAsReadableString(o.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_operations_with_tags" + ) + ); + expect(exampleOperationsWithTags.tags).to.eql(["tag1"]); + + // Check declaration. + const exampleDeclaration = graph.declarations.find( + d => + targetAsReadableString(d.target) === + "some_database_name.some_external_schema_name.very_important_external_table" + ); + expect(exampleDeclaration.target).eql( + dataform.Target.create({ + database: "some_database_name", + schema: "some_external_schema_name", + name: "very_important_external_table" + }) + ); + expect(exampleDeclaration.actionDescriptor.description).to.equal( + "This table is not generated by Dataform!" + ); + + // Check testcases. + const testCase = graph.tests.find(t => t.name === "example_test_case"); + expect(testCase.testQuery.trim()).equals( + "select * from (\n select 'hi' as faked union all\n select 'ben' as faked union all\n select 'sup?' as faked\n)\n\n-- here ${\"is\"} a `comment\n\n/* ${\"another\"} ` backtick ` containing ```comment */" + ); + expect(testCase.expectedOutputQuery.trim()).equals( + "select 'hi' as faked union all\nselect 'ben' as faked union all\nselect 'sup?' as faked" + ); + + const testCaseFQ = graph.tests.find(t => t.name === "example_test_case_fq_ref"); + expect(testCaseFQ.testQuery.trim()).equals( + "select * from (\n select 'hi' as faked union all\n select 'ben' as faked union all\n select 'sup?' as faked\n)\n\n-- here ${\"is\"} a `comment\n\n/* ${\"another\"} ` backtick ` containing ```comment */" + ); + expect(testCaseFQ.expectedOutputQuery.trim()).equals( + "select 'hi' as faked union all\nselect 'ben' as faked union all\nselect 'sup?' as faked" + ); + + // Check double backslashes don't get converted to singular. + const exampleDoubleBackslash = graph.tables.find( + (t: dataform.ITable) => + targetAsReadableString(t.target) === + dotJoined( + databaseWithSuffix("tada-analytics"), + schemaWithSuffix("df_integration_test"), + "example_double_backslash" + ) + ); + expect(cleanSql(exampleDoubleBackslash.query)).equals( + "select * from regexp_extract('01a_data_engine', '^(\\\\d{2}\\\\w)') select * from regexp_extract('01a_data_engine', r'^(\\d{2}\\w)')" + ); + expect(cleanSql(exampleDoubleBackslash.preOps[0])).equals( + "select * from regexp_extract('\\\\\\\\', '\\\\')" + ); + }); } } }); - test("backwards_compatibility", async () => { - // The purpose of this test is partially to make sure we don't break the compilation "contract" - // between the old versions of core and the new versions of the CLI. - const graph = await compile({ projectDir: "tests/api/projects/backwards_compatibility" }); - - const tableNames = graph.tables.map((t: dataform.ITable) => t.target.name); - - // Make sure it compiles. - expect(tableNames).includes("example"); - const example = graph.tables.filter((t: dataform.ITable) => t.target.name === "example")[0]; - expect(example.type).equals("table"); - expect(example.query.trim()).equals("select 1 as foo_bar"); - - // Make sure we can dry run. - new Builder(graph, {}, { tables: [] }).build(); - }); - test("times out after timeout period during compilation", async () => { try { await compile({ projectDir: "tests/api/projects/never_finishes_compiling" }); diff --git a/tests/api/projects/backwards_compatibility/BUILD b/tests/api/projects/backwards_compatibility/BUILD deleted file mode 100644 index 3c714a2f8..000000000 --- a/tests/api/projects/backwards_compatibility/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -package(default_visibility = ["//tests:__subpackages__"]) - -load("//tools:node_modules.bzl", "node_modules") - -filegroup( - name = "files", - srcs = glob([ - "**/*.*", - ]), -) - -node_modules( - name = "node_modules", - packages = [ - "protobufjs@6.8.8", - "@dataform/core@1.0.3", - "@dataform/protos@1.0.3", - ], -) diff --git a/tests/api/projects/backwards_compatibility/dataform.json b/tests/api/projects/backwards_compatibility/dataform.json deleted file mode 100644 index 1227b7cef..000000000 --- a/tests/api/projects/backwards_compatibility/dataform.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "warehouse": "bigquery", - "defaultDatabase": "tada-analytics", - "defaultSchema": "dataform_example", - "assertionSchema": "dataform_assertions", - "defaultLocation": "US" -} diff --git a/tests/api/projects/backwards_compatibility/definitions/example.sql b/tests/api/projects/backwards_compatibility/definitions/example.sql deleted file mode 100644 index bd218ffee..000000000 --- a/tests/api/projects/backwards_compatibility/definitions/example.sql +++ /dev/null @@ -1,2 +0,0 @@ -${type("table")} -select 1 as ${macros.foo("bar")} diff --git a/tests/api/projects/backwards_compatibility/definitions/example_assertion.assert.sql b/tests/api/projects/backwards_compatibility/definitions/example_assertion.assert.sql deleted file mode 100644 index 6721af860..000000000 --- a/tests/api/projects/backwards_compatibility/definitions/example_assertion.assert.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as value where false diff --git a/tests/api/projects/backwards_compatibility/includes/macros.js b/tests/api/projects/backwards_compatibility/includes/macros.js deleted file mode 100644 index edcd0359f..000000000 --- a/tests/api/projects/backwards_compatibility/includes/macros.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - foo: value => `foo_${value}`, -}