From 1c880b5343915c54888fbaa9fa806777aa6ee264 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Tue, 16 Apr 2024 10:40:01 -0500 Subject: [PATCH 1/3] feat(scaffold new dbt project): Add project scaffolding feature `tbd` now functions as a drop-in replacement for `dbt init` by adding features which optionally scaffold a dbt project. --- forms.go | 30 ++++++++++++++++++ main.go | 10 ++++++ shared/types.go | 17 +++++----- write_profile.go | 62 ++++++++++++++++++++++++++++++++++++ write_profile_test.go | 41 ++++++++++++++++++++++++ write_scaffold_project.go | 66 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 write_profile.go create mode 100644 write_profile_test.go create mode 100644 write_scaffold_project.go diff --git a/forms.go b/forms.go index 15f8f17..e8aa6de 100644 --- a/forms.go +++ b/forms.go @@ -23,6 +23,9 @@ type FormResponse struct { UseDbtProfile bool DbtProfile string DbtProfileOutput string + CreateProfile bool + ScaffoldProject bool + ProjectName string } func Forms() (formResponse FormResponse) { @@ -75,8 +78,23 @@ You'll need: huh.NewConfirm().Affirmative("Yes!").Negative("Nah"). Title("Do you have a dbt profile you'd like to connect with?\n(you can enter your credentials manually if not)"). Value(&formResponse.UseDbtProfile), + huh.NewConfirm().Affirmative("Yeah!").Negative("Nope"). + Title("Would you like to scaffold a basic dbt project into the output directory?"). + Value(&formResponse.ScaffoldProject), ), ) + project_name_form := huh.NewForm( + huh.NewGroup(huh.NewInput(). + Title("What is the name of your dbt project?"). + Value(&formResponse.ProjectName). + Placeholder("gondor_patrol_analytics"), + )) + profile_create_form := huh.NewForm( + huh.NewGroup( + huh.NewConfirm().Affirmative("Yes, pls").Negative("No, thx"). + Title("Would you like to generate a profiles.yml file from the info you provide next?"). + Value(&formResponse.CreateProfile), + )) dbt_form := huh.NewForm( huh.NewGroup( huh.NewInput(). @@ -173,6 +191,8 @@ tbd will overwrite any existing files of the same name.`), ), ) intro_form.WithTheme(huh.ThemeCatppuccin()) + profile_create_form.WithTheme(huh.ThemeCatppuccin()) + project_name_form.WithTheme(huh.ThemeCatppuccin()) dbt_form.WithTheme(huh.ThemeCatppuccin()) warehouse_form.WithTheme(huh.ThemeCatppuccin()) snowflake_form.WithTheme(huh.ThemeCatppuccin()) @@ -191,6 +211,16 @@ tbd will overwrite any existing files of the same name.`), log.Fatalf("Error running dbt form %v\n", err) } } else { + err = profile_create_form.Run() + if err != nil { + log.Fatalf("Error running profile create form %v\n", err) + } + if formResponse.ScaffoldProject { + err = project_name_form.Run() + if err != nil { + log.Fatalf("Error running project name form %v\n", err) + } + } err = warehouse_form.Run() if err != nil { log.Fatalf("Error running warehouse form %v\n", err) diff --git a/main.go b/main.go index da8975c..b7dcd81 100644 --- a/main.go +++ b/main.go @@ -50,7 +50,17 @@ func main() { if formResponse.GenerateDescriptions { GenerateColumnDescriptions(ts) } + if formResponse.CreateProfile { + WriteProfile(cd, bd) + } PrepBuildDir(bd) + if formResponse.ScaffoldProject { + s, err := WriteScaffoldProject(cd, bd, formResponse.ProjectName) + if err != nil { + log.Fatalf("Error scaffolding project: %v\n", err) + } + bd = s + } err = WriteFiles(ts, bd) if err != nil { log.Fatalf("Error writing files: %v\n", err) diff --git a/shared/types.go b/shared/types.go index 2bd0709..e7f28fb 100644 --- a/shared/types.go +++ b/shared/types.go @@ -18,12 +18,13 @@ type SourceTables struct { } type ConnectionDetails struct { - ConnType string - Username string - Account string - Database string - Schema string - Project string - Dataset string - Path string + ConnType string + Username string + Account string + Database string + Schema string + Project string + Dataset string + Path string + ProjectName string } diff --git a/write_profile.go b/write_profile.go new file mode 100644 index 0000000..bfa9dfb --- /dev/null +++ b/write_profile.go @@ -0,0 +1,62 @@ +package main + +import ( + "log" + "os" + "path" + "text/template" + + "github.com/gwenwindflower/tbd/shared" +) + +func WriteProfile(cd shared.ConnectionDetails, bd string) { + pt := ` +{{.ConnType}}: + target: dev + outputs: + dev: + type: {{.ConnType}} + {{- if eq .ConnType "snowflake"}} + auth: externalbrowser + {{- end}} + {{- if eq .ConnType "bigquery"}} + method: oauth + {{- end}} + {{- if .Account}} + account: {{.Account}} + {{- end}} + {{- if .Username}} + user: {{.Username}} + {{- end}} + {{- if .Database}} + database: {{.Database}} + {{- end}} + {{- if .Project}} + project: {{.Project}} + {{- end}} + {{- if .Schema}} + schema: {{.Schema}} + {{- end}} + {{- if .Dataset}} + dataset: {{.Dataset}} + {{- end}} + {{- if .Path}} + path: {{.Path}} + {{- end}} + threads: 8 +` + tmpl, err := template.New("profiles.yml").Parse(pt) + if err != nil { + log.Fatalf("Failed to parse template %v\n", err) + } + p := path.Join(bd, "profiles.yml") + o, err := os.Create(p) + if err != nil { + log.Fatalf("Failed to create profiles.yml file %v\n", err) + } + defer o.Close() + err = tmpl.Execute(o, cd) + if err != nil { + log.Fatalf("Failed to execute template %v\n", err) + } +} diff --git a/write_profile_test.go b/write_profile_test.go new file mode 100644 index 0000000..d60819b --- /dev/null +++ b/write_profile_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "os" + "testing" + + "github.com/gwenwindflower/tbd/shared" +) + +func TestWriteProfile(t *testing.T) { + cd := shared.ConnectionDetails{ + ConnType: "snowflake", + Username: "aragorn", + Account: "dunedain.snowflakecomputing.com", + Database: "gondor", + Schema: "minas_tirith", + } + tmpDir := t.TempDir() + WriteProfile(cd, tmpDir) + + expected := []byte(` +snowflake: + target: dev + outputs: + dev: + type: snowflake + account: dunedain.snowflakecomputing.com + user: aragorn + database: gondor + schema: minas_tirith + threads: 8 +`) + got, err := os.ReadFile("profiles.yml") + if err != nil { + t.Fatalf("Failed to read profiles.yml: %v", err) + } + // os.Remove("profiles.yml") + if string(got) != string(expected) { + t.Errorf("Expected %s, got %s", expected, got) + } +} diff --git a/write_scaffold_project.go b/write_scaffold_project.go new file mode 100644 index 0000000..e32d860 --- /dev/null +++ b/write_scaffold_project.go @@ -0,0 +1,66 @@ +package main + +import ( + "log" + "os" + "path" + "text/template" + + "github.com/gwenwindflower/tbd/shared" +) + +func WriteScaffoldProject(cd shared.ConnectionDetails, bd string, pn string) (string, error) { + folders := []string{"models", "analyses", "data", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} + for _, folder := range folders { + p := path.Join(bd, folder) + err := os.MkdirAll(p, 0755) + if err != nil { + return "", err + } + } + projectYamlTemplate := `config-version: 2 + +name: {{.ProjectName}} +profile: {{.ConnType}} + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["data-tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + {{.ProjectName}}: + staging: + +materialized: view + marts: + +materialized: table +` + tmpl, err := template.New("dbt_project.yml").Parse(projectYamlTemplate) + if err != nil { + log.Fatalf("Failed to parse dbt_project.yml template %v\n", err) + } + p := path.Join(bd, "dbt_project.yml") + o, err := os.Create(p) + if err != nil { + log.Fatalf("Failed to create dbt_project.yml file %v\n", err) + } + defer o.Close() + cd.ProjectName = pn + err = tmpl.Execute(o, cd) + if err != nil { + log.Fatalf("Failed to execute dbt_project.yml template %v\n", err) + } + s := path.Join(bd, "models/staging", cd.Schema) + err = os.MkdirAll(s, 0755) + if err != nil { + return "", err + } + return s, nil +} From 49090afc533b95dcd395d199645782ed05126dab Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Tue, 16 Apr 2024 11:56:14 -0500 Subject: [PATCH 2/3] fix(empty directories)!: Require target folder to be empty or new This commit adds an intentional error if the target folder exists and is not empty. It also adds .gitkeep files in all the empty scaffolded folders to allow the directories to be tracked if there's an inital commit of the generated project. --- forms.go | 6 +++--- main.go | 5 ++++- prep_build_dir.go | 22 ++++++++++++++++++---- write_profile.go | 2 +- write_scaffold_project.go | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/forms.go b/forms.go index e8aa6de..97598d3 100644 --- a/forms.go +++ b/forms.go @@ -44,7 +44,7 @@ Generates: For each table in the designated schema/dataset. To prepare, make sure you have the following: -✴︎ An existing dbt profile.yml file to reference +✴︎ An existing dbt profiles.yml file to reference *_OR_* ✴︎ The necessary connection details for your warehouse @@ -173,8 +173,8 @@ Relative to pwd e.g. if db is in this dir -> cool_ducks.db`). huh.NewNote(). Title("🚧🚨 Choose your build directory carefully! 🚨🚧"). Description(`Choose a _new_ or _empty_ directory. -If you use an existing directory, -tbd will overwrite any existing files of the same name.`), +If you choose an existing, populated directory +tbd will _intentionally error out_.`), ), huh.NewGroup( huh.NewInput(). diff --git a/main.go b/main.go index b7dcd81..92d5fc8 100644 --- a/main.go +++ b/main.go @@ -50,10 +50,13 @@ func main() { if formResponse.GenerateDescriptions { GenerateColumnDescriptions(ts) } + err = PrepBuildDir(bd) + if err != nil { + log.Fatalf("Error preparing build directory: %v\n", err) + } if formResponse.CreateProfile { WriteProfile(cd, bd) } - PrepBuildDir(bd) if formResponse.ScaffoldProject { s, err := WriteScaffoldProject(cd, bd, formResponse.ProjectName) if err != nil { diff --git a/prep_build_dir.go b/prep_build_dir.go index 2fac191..172688b 100644 --- a/prep_build_dir.go +++ b/prep_build_dir.go @@ -1,16 +1,30 @@ package main import ( + "errors" "log" "os" ) -func PrepBuildDir(buildDir string) { - _, err := os.Stat(buildDir) +func PrepBuildDir(bd string) error { + _, err := os.Stat(bd) if os.IsNotExist(err) { - dirErr := os.MkdirAll(buildDir, 0755) + dirErr := os.MkdirAll(bd, 0755) if dirErr != nil { - log.Fatalf("Failed to create directory %v", dirErr) + return dirErr } + } else if err == nil { + files, err := os.ReadDir(bd) + if err != nil { + log.Fatalf("Failed to check build target directory %v", err) + } + if len(files) == 0 { + return nil + } else { + return errors.New("build directory is not empty") + } + } else { + return err } + return nil } diff --git a/write_profile.go b/write_profile.go index bfa9dfb..ccdd97e 100644 --- a/write_profile.go +++ b/write_profile.go @@ -45,7 +45,7 @@ func WriteProfile(cd shared.ConnectionDetails, bd string) { {{- end}} threads: 8 ` - tmpl, err := template.New("profiles.yml").Parse(pt) + tmpl, err := template.New("profiles").Parse(pt) if err != nil { log.Fatalf("Failed to parse template %v\n", err) } diff --git a/write_scaffold_project.go b/write_scaffold_project.go index e32d860..d2c40b1 100644 --- a/write_scaffold_project.go +++ b/write_scaffold_project.go @@ -10,7 +10,8 @@ import ( ) func WriteScaffoldProject(cd shared.ConnectionDetails, bd string, pn string) (string, error) { - folders := []string{"models", "analyses", "data", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} + folders := []string{"models", "analyses", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} + emptyFolders := []string{"analyses", "macros", "seeds", "snapshots", "data-tests", "models/marts"} for _, folder := range folders { p := path.Join(bd, folder) err := os.MkdirAll(p, 0755) @@ -18,6 +19,13 @@ func WriteScaffoldProject(cd shared.ConnectionDetails, bd string, pn string) (st return "", err } } + for _, folder := range emptyFolders { + p := path.Join(bd, folder, ".gitkeep") + err := os.MkdirAll(p, 0755) + if err != nil { + log.Fatalf("Failed to create .gitkeep in %s folder %v\n", folder, err) + } + } projectYamlTemplate := `config-version: 2 name: {{.ProjectName}} @@ -42,6 +50,24 @@ models: marts: +materialized: table ` + gitignore := []byte(`.venv +venv +.env +env + +target/ +dbt_packages/ +logs/ +profiles.yml + +.DS_Store + +.user.yml + +.ruff_cache +__pycache__ +`) + tmpl, err := template.New("dbt_project.yml").Parse(projectYamlTemplate) if err != nil { log.Fatalf("Failed to parse dbt_project.yml template %v\n", err) @@ -57,6 +83,11 @@ models: if err != nil { log.Fatalf("Failed to execute dbt_project.yml template %v\n", err) } + gi := path.Join(bd, ".gitignore") + err = os.WriteFile(gi, gitignore, 0644) + if err != nil { + log.Fatalf("Failed to write .gitignore file %v\n", err) + } s := path.Join(bd, "models/staging", cd.Schema) err = os.MkdirAll(s, 0755) if err != nil { From 5a38b7071ebb4aec1ea5d3d8a3c9cdcc837c6839 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Tue, 16 Apr 2024 12:25:05 -0500 Subject: [PATCH 3/3] test(scaffolding project): Add tests for project scaffolding Adds tests that folders are created, gitignore is matching, and dbt_project.yml is templated correctly based on ConnectionDetails. --- write_profile.go | 2 +- write_profile_test.go | 5 +- write_scaffold_project_test.go | 91 ++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 write_scaffold_project_test.go diff --git a/write_profile.go b/write_profile.go index ccdd97e..24a4a85 100644 --- a/write_profile.go +++ b/write_profile.go @@ -17,7 +17,7 @@ func WriteProfile(cd shared.ConnectionDetails, bd string) { dev: type: {{.ConnType}} {{- if eq .ConnType "snowflake"}} - auth: externalbrowser + authenticator: externalbrowser {{- end}} {{- if eq .ConnType "bigquery"}} method: oauth diff --git a/write_profile_test.go b/write_profile_test.go index d60819b..5c361d0 100644 --- a/write_profile_test.go +++ b/write_profile_test.go @@ -2,6 +2,7 @@ package main import ( "os" + "path" "testing" "github.com/gwenwindflower/tbd/shared" @@ -24,13 +25,15 @@ snowflake: outputs: dev: type: snowflake + authenticator: externalbrowser account: dunedain.snowflakecomputing.com user: aragorn database: gondor schema: minas_tirith threads: 8 `) - got, err := os.ReadFile("profiles.yml") + tpp := path.Join(tmpDir, "profiles.yml") + got, err := os.ReadFile(tpp) if err != nil { t.Fatalf("Failed to read profiles.yml: %v", err) } diff --git a/write_scaffold_project_test.go b/write_scaffold_project_test.go new file mode 100644 index 0000000..fc2f65d --- /dev/null +++ b/write_scaffold_project_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "os" + "path" + "testing" + + "github.com/gwenwindflower/tbd/shared" +) + +func TestWriteScaffoldProject(t *testing.T) { + cd := shared.ConnectionDetails{ + ConnType: "snowflake", + Username: "user", + Account: "account", + Database: "database", + Schema: "schema", + Project: "project", + } + bd := t.TempDir() + pn := "project" + _, err := WriteScaffoldProject(cd, bd, pn) + if err != nil { + t.Fatalf("Error scaffolding project: %v\n", err) + } + // Check that the directories were created + for _, folder := range []string{"models", "analyses", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} { + if _, err := os.Stat(path.Join(bd, folder)); os.IsNotExist(err) { + t.Fatalf("Directory %s was not created\n", folder) + } + } + // Check that .gitignore was created correctly + gitignore := []byte(`.venv +venv +.env +env + +target/ +dbt_packages/ +logs/ +profiles.yml + +.DS_Store + +.user.yml + +.ruff_cache +__pycache__ +`) + gi := path.Join(bd, ".gitignore") + got, err := os.ReadFile(gi) + if err != nil { + t.Fatalf("Failed to read .gitignore: %v", err) + } + if string(got) != string(gitignore) { + t.Errorf("Expected %s, got %s", gitignore, got) + } + // Check that project.yml was created correctly + projectYaml := []byte(`config-version: 2 + +name: project +profile: snowflake + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["data-tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + project: + staging: + +materialized: view + marts: + +materialized: table +`) + py := path.Join(bd, "dbt_project.yml") + got, err = os.ReadFile(py) + if err != nil { + t.Fatalf("Failed to read dbt_project.yml: %v", err) + } + if string(got) != string(projectYaml) { + t.Errorf("Expected %s, got %s", projectYaml, got) + } +}