Skip to content

Commit

Permalink
merge from main
Browse files Browse the repository at this point in the history
  • Loading branch information
Ekrekr committed Dec 27, 2024
2 parents a913c20 + b951c38 commit 336d42c
Show file tree
Hide file tree
Showing 12 changed files with 3,427 additions and 3,368 deletions.
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* @dataform-co/dataform-reviewers
18 changes: 9 additions & 9 deletions cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -586,18 +586,18 @@ export function runCli() {
format: `format [${projectDirMustExistOption.name}]`,
description: "Format the dataform project's files.",
positionalOptions: [projectDirMustExistOption],
options: [
actionsOption
],
options: [actionsOption],
processFn: async argv => {
let actions = ["{definitions,includes}/**/*.{js,sqlx}"]
let actions = ["{definitions,includes}/**/*.{js,sqlx}"];
if (actionsOption.name in argv && argv[actionsOption.name].length > 0) {
actions = argv[actionsOption.name]
actions = argv[actionsOption.name];
}
const filenames = actions.map((action: string) =>
glob.sync(action, {cwd: argv[projectDirMustExistOption.name]})
).flat();
const results: Array<{ filename: string; err?: Error; }> = await Promise.all(
const filenames = actions
.map((action: string) =>
glob.sync(action, { cwd: argv[projectDirMustExistOption.name] })
)
.flat();
const results: Array<{ filename: string; err?: Error }> = await Promise.all(
filenames.map(async (filename: string) => {
try {
await formatFile(path.resolve(argv[projectDirMustExistOption.name], filename), {
Expand Down
2 changes: 1 addition & 1 deletion common/errors/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export function coerceAsError<T extends Error | any>(errorLike: T): T extends Er
if ((errorLike as any) instanceof Error) {
return errorLike as any;
}
const error = (errorLike as any) as Error
const error = (errorLike as any) as Error;
// Otherwise, attempt to reconstruct an error class from the object.
const message = error.message ? String(error.message) : String(errorLike);
const coercedError = new Error(message);
Expand Down
70 changes: 33 additions & 37 deletions core/actions/data_preparation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,23 +38,23 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
const dataPreparationAsJson = nativeRequire(config.filename).asJson;

// Find targets
const targets = this.getTargets(dataPreparationAsJson as {
[key: string]: any;
});
const targets = this.getTargets(
dataPreparationAsJson as {
[key: string]: any;
}
);
const resolvedTargets = targets.map(target =>
this.applySessionToTarget(target, session.projectConfig, config.filename, true)
)
// Finalize list of targets.
this.proto.targets = resolvedTargets.map(target =>
this.finalizeTarget(target)
this.applySessionToTarget(target, session.projectConfig, config.filename, true)
);
// Finalize list of targets.
this.proto.targets = resolvedTargets.map(target => this.finalizeTarget(target));
this.proto.canonicalTargets = targets.map(target =>
this.applySessionToTarget(target, session.canonicalProjectConfig)
);

// Resolve all table references with compilation overrides and encode resolved proto instance
const resolvedDefinition = this.applySessionToDataPreparationContents(dataPreparationAsJson);
this.proto.dataPreparationYaml = dumpYaml(resolvedDefinition)
this.proto.dataPreparationYaml = dumpYaml(resolvedDefinition);

// Set the unique target key as the first target defined.
// TODO: Remove once multiple targets are supported.
Expand Down Expand Up @@ -116,77 +116,75 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
);
}

private applySessionToDataPreparationContents(
definition: {[key: string]: any}
): {[key: string]: any} {
private applySessionToDataPreparationContents(definition: {
[key: string]: any;
}): { [key: string]: any } {
// Resolve error table, if set
// @ts-ignore
const errorTable = definition.configuration?.errorTable;
if (errorTable) {
definition.configuration.errorTable =
this.applySessionToTableReference(errorTable as {[key: string]: string});
definition.configuration.errorTable = this.applySessionToTableReference(
errorTable as { [key: string]: string }
);
}

// Loop through all nodes and resolve the compilation overrides for
// all source and destination tables.
if (definition.nodes) {
(definition.nodes as Array<{ [key: string]: any }>).forEach((node, index) => {

// Resolve source tables, if set.
const sourceTable = node.source?.table;
if (sourceTable) {
definition.nodes[index].source.table =
this.applySessionToTableReference(sourceTable as {[key: string]: string});
definition.nodes[index].source.table = this.applySessionToTableReference(
sourceTable as { [key: string]: string }
);
}

// Resolve destination tables, if set.
const destinationTable = node.destination?.table;
if (destinationTable) {
definition.nodes[index].destination.table =
this.applySessionToTableReference(destinationTable as {[key: string]: string});
definition.nodes[index].destination.table = this.applySessionToTableReference(
destinationTable as { [key: string]: string }
);
}
});
}

return definition;
}

private applySessionToTableReference(
tableReference: {[key: string]: string}
): object {
private applySessionToTableReference(tableReference: { [key: string]: string }): object {
const target: dataform.ITarget = {
database: tableReference.project,
schema: tableReference.dataset,
name: tableReference.table
}
const resolvedTarget =
this.applySessionToTarget(
dataform.Target.create(target),
this.session.projectConfig)
};
const resolvedTarget = this.applySessionToTarget(
dataform.Target.create(target),
this.session.projectConfig
);
// Convert resolved target into a Data Preparation Table Reference
let resolvedTableReference : {[key: string]: string} = {
table: this.session.finalizeName(resolvedTarget.name),
}
let resolvedTableReference: { [key: string]: string } = {
table: this.session.finalizeName(resolvedTarget.name)
};

// Ensure project and dataset field are added in order
if (resolvedTarget.schema) {
resolvedTableReference = {
dataset: this.session.finalizeSchema(resolvedTarget.schema),
...resolvedTableReference
}
};
}
if (resolvedTarget.database) {
resolvedTableReference = {
project: this.session.finalizeDatabase(resolvedTarget.database),
...resolvedTableReference
}
};
}
return resolvedTableReference;
}

private getTargets(definition: {
[key: string]: any;
}): dataform.Target[] {
private getTargets(definition: { [key: string]: any }): dataform.Target[] {
const targets: dataform.Target[] = [];

if (definition.nodes) {
Expand All @@ -199,12 +197,10 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
name: table.table
};
targets.push(dataform.Target.create(compiledGraphTarget));

}
});
}

return targets;
}
}

4 changes: 2 additions & 2 deletions core/actions/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {
LegacyConfigConverter
} from "df/core/actions";
import { Assertion } from "df/core/actions/assertion";
import { IncrementalTable } from "df/core/actions/incremental_table";
import { View } from "df/core/actions/view";
import { ColumnDescriptors } from "df/core/column_descriptors";
import { Contextable, Resolvable } from "df/core/common";
import * as Path from "df/core/path";
Expand All @@ -21,8 +23,6 @@ import {
validateQueryString
} from "df/core/utils";
import { dataform } from "df/protos/ts";
import { IncrementalTable } from "./incremental_table";
import { View } from "./view";

/**
* @hidden
Expand Down
2 changes: 1 addition & 1 deletion core/actions/view.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos";
import { ActionBuilder, LegacyConfigConverter, ITableContext } from "df/core/actions";
import { ActionBuilder, ITableContext, LegacyConfigConverter } from "df/core/actions";
import { Assertion } from "df/core/actions/assertion";
import { ColumnDescriptors } from "df/core/column_descriptors";
import { Contextable, Resolvable } from "df/core/common";
Expand Down
Loading

0 comments on commit 336d42c

Please sign in to comment.