-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature] Add project scaffolding option (#5)
To make `tbd` a true alternative to `dbt init`, it needs to optionally be able to scaffold a baseline dbt project. It can now do this! If you choose this option your files will be generated inside the `staging` folder of the scaffolded project. It also adds an important guard that prevents users choosing an existing, populated directory to build into. Rather than deal with how to handle overwrites, I think it's much better to just error out and ask users to either specify a new directory or choose an empty one.
- Loading branch information
Showing
8 changed files
with
368 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,30 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "errors" | ||
| "log" | ||
| "os" | ||
| ) | ||
|
|
||
| func PrepBuildDir(buildDir string) { | ||
| _, err := os.Stat(buildDir) | ||
| func PrepBuildDir(bd string) error { | ||
| _, err := os.Stat(bd) | ||
| if os.IsNotExist(err) { | ||
| dirErr := os.MkdirAll(buildDir, 0755) | ||
| dirErr := os.MkdirAll(bd, 0755) | ||
| if dirErr != nil { | ||
| log.Fatalf("Failed to create directory %v", dirErr) | ||
| return dirErr | ||
| } | ||
| } else if err == nil { | ||
| files, err := os.ReadDir(bd) | ||
| if err != nil { | ||
| log.Fatalf("Failed to check build target directory %v", err) | ||
| } | ||
| if len(files) == 0 { | ||
| return nil | ||
| } else { | ||
| return errors.New("build directory is not empty") | ||
| } | ||
| } else { | ||
| return err | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "log" | ||
| "os" | ||
| "path" | ||
| "text/template" | ||
|
|
||
| "github.com/gwenwindflower/tbd/shared" | ||
| ) | ||
|
|
||
| func WriteProfile(cd shared.ConnectionDetails, bd string) { | ||
| pt := ` | ||
| {{.ConnType}}: | ||
| target: dev | ||
| outputs: | ||
| dev: | ||
| type: {{.ConnType}} | ||
| {{- if eq .ConnType "snowflake"}} | ||
| authenticator: externalbrowser | ||
| {{- end}} | ||
| {{- if eq .ConnType "bigquery"}} | ||
| method: oauth | ||
| {{- end}} | ||
| {{- if .Account}} | ||
| account: {{.Account}} | ||
| {{- end}} | ||
| {{- if .Username}} | ||
| user: {{.Username}} | ||
| {{- end}} | ||
| {{- if .Database}} | ||
| database: {{.Database}} | ||
| {{- end}} | ||
| {{- if .Project}} | ||
| project: {{.Project}} | ||
| {{- end}} | ||
| {{- if .Schema}} | ||
| schema: {{.Schema}} | ||
| {{- end}} | ||
| {{- if .Dataset}} | ||
| dataset: {{.Dataset}} | ||
| {{- end}} | ||
| {{- if .Path}} | ||
| path: {{.Path}} | ||
| {{- end}} | ||
| threads: 8 | ||
| ` | ||
| tmpl, err := template.New("profiles").Parse(pt) | ||
| if err != nil { | ||
| log.Fatalf("Failed to parse template %v\n", err) | ||
| } | ||
| p := path.Join(bd, "profiles.yml") | ||
| o, err := os.Create(p) | ||
| if err != nil { | ||
| log.Fatalf("Failed to create profiles.yml file %v\n", err) | ||
| } | ||
| defer o.Close() | ||
| err = tmpl.Execute(o, cd) | ||
| if err != nil { | ||
| log.Fatalf("Failed to execute template %v\n", err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "os" | ||
| "path" | ||
| "testing" | ||
|
|
||
| "github.com/gwenwindflower/tbd/shared" | ||
| ) | ||
|
|
||
| func TestWriteProfile(t *testing.T) { | ||
| cd := shared.ConnectionDetails{ | ||
| ConnType: "snowflake", | ||
| Username: "aragorn", | ||
| Account: "dunedain.snowflakecomputing.com", | ||
| Database: "gondor", | ||
| Schema: "minas_tirith", | ||
| } | ||
| tmpDir := t.TempDir() | ||
| WriteProfile(cd, tmpDir) | ||
|
|
||
| expected := []byte(` | ||
| snowflake: | ||
| target: dev | ||
| outputs: | ||
| dev: | ||
| type: snowflake | ||
| authenticator: externalbrowser | ||
| account: dunedain.snowflakecomputing.com | ||
| user: aragorn | ||
| database: gondor | ||
| schema: minas_tirith | ||
| threads: 8 | ||
| `) | ||
| tpp := path.Join(tmpDir, "profiles.yml") | ||
| got, err := os.ReadFile(tpp) | ||
| if err != nil { | ||
| t.Fatalf("Failed to read profiles.yml: %v", err) | ||
| } | ||
| // os.Remove("profiles.yml") | ||
| if string(got) != string(expected) { | ||
| t.Errorf("Expected %s, got %s", expected, got) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "log" | ||
| "os" | ||
| "path" | ||
| "text/template" | ||
|
|
||
| "github.com/gwenwindflower/tbd/shared" | ||
| ) | ||
|
|
||
| func WriteScaffoldProject(cd shared.ConnectionDetails, bd string, pn string) (string, error) { | ||
| folders := []string{"models", "analyses", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} | ||
| emptyFolders := []string{"analyses", "macros", "seeds", "snapshots", "data-tests", "models/marts"} | ||
| for _, folder := range folders { | ||
| p := path.Join(bd, folder) | ||
| err := os.MkdirAll(p, 0755) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| } | ||
| for _, folder := range emptyFolders { | ||
| p := path.Join(bd, folder, ".gitkeep") | ||
| err := os.MkdirAll(p, 0755) | ||
| if err != nil { | ||
| log.Fatalf("Failed to create .gitkeep in %s folder %v\n", folder, err) | ||
| } | ||
| } | ||
| projectYamlTemplate := `config-version: 2 | ||
| name: {{.ProjectName}} | ||
| profile: {{.ConnType}} | ||
| model-paths: ["models"] | ||
| analysis-paths: ["analyses"] | ||
| test-paths: ["data-tests"] | ||
| seed-paths: ["seeds"] | ||
| macro-paths: ["macros"] | ||
| snapshot-paths: ["snapshots"] | ||
| target-path: "target" | ||
| clean-targets: | ||
| - "target" | ||
| - "dbt_packages" | ||
| models: | ||
| {{.ProjectName}}: | ||
| staging: | ||
| +materialized: view | ||
| marts: | ||
| +materialized: table | ||
| ` | ||
| gitignore := []byte(`.venv | ||
| venv | ||
| .env | ||
| env | ||
| target/ | ||
| dbt_packages/ | ||
| logs/ | ||
| profiles.yml | ||
| .DS_Store | ||
| .user.yml | ||
| .ruff_cache | ||
| __pycache__ | ||
| `) | ||
|
|
||
| tmpl, err := template.New("dbt_project.yml").Parse(projectYamlTemplate) | ||
| if err != nil { | ||
| log.Fatalf("Failed to parse dbt_project.yml template %v\n", err) | ||
| } | ||
| p := path.Join(bd, "dbt_project.yml") | ||
| o, err := os.Create(p) | ||
| if err != nil { | ||
| log.Fatalf("Failed to create dbt_project.yml file %v\n", err) | ||
| } | ||
| defer o.Close() | ||
| cd.ProjectName = pn | ||
| err = tmpl.Execute(o, cd) | ||
| if err != nil { | ||
| log.Fatalf("Failed to execute dbt_project.yml template %v\n", err) | ||
| } | ||
| gi := path.Join(bd, ".gitignore") | ||
| err = os.WriteFile(gi, gitignore, 0644) | ||
| if err != nil { | ||
| log.Fatalf("Failed to write .gitignore file %v\n", err) | ||
| } | ||
| s := path.Join(bd, "models/staging", cd.Schema) | ||
| err = os.MkdirAll(s, 0755) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return s, nil | ||
| } |
Oops, something went wrong.