diff --git a/forms.go b/forms.go index 15f8f17..97598d3 100644 --- a/forms.go +++ b/forms.go @@ -23,6 +23,9 @@ type FormResponse struct { UseDbtProfile bool DbtProfile string DbtProfileOutput string + CreateProfile bool + ScaffoldProject bool + ProjectName string } func Forms() (formResponse FormResponse) { @@ -41,7 +44,7 @@ Generates: For each table in the designated schema/dataset. To prepare, make sure you have the following: -✴︎ An existing dbt profile.yml file to reference +✴︎ An existing dbt profiles.yml file to reference *_OR_* ✴︎ The necessary connection details for your warehouse @@ -75,8 +78,23 @@ You'll need: huh.NewConfirm().Affirmative("Yes!").Negative("Nah"). Title("Do you have a dbt profile you'd like to connect with?\n(you can enter your credentials manually if not)"). Value(&formResponse.UseDbtProfile), + huh.NewConfirm().Affirmative("Yeah!").Negative("Nope"). + Title("Would you like to scaffold a basic dbt project into the output directory?"). + Value(&formResponse.ScaffoldProject), ), ) + project_name_form := huh.NewForm( + huh.NewGroup(huh.NewInput(). + Title("What is the name of your dbt project?"). + Value(&formResponse.ProjectName). + Placeholder("gondor_patrol_analytics"), + )) + profile_create_form := huh.NewForm( + huh.NewGroup( + huh.NewConfirm().Affirmative("Yes, pls").Negative("No, thx"). + Title("Would you like to generate a profiles.yml file from the info you provide next?"). + Value(&formResponse.CreateProfile), + )) dbt_form := huh.NewForm( huh.NewGroup( huh.NewInput(). @@ -155,8 +173,8 @@ Relative to pwd e.g. if db is in this dir -> cool_ducks.db`). huh.NewNote(). Title("🚧🚨 Choose your build directory carefully! 🚨🚧"). Description(`Choose a _new_ or _empty_ directory. -If you use an existing directory, -tbd will overwrite any existing files of the same name.`), +If you choose an existing, populated directory +tbd will _intentionally error out_.`), ), huh.NewGroup( huh.NewInput(). @@ -173,6 +191,8 @@ tbd will overwrite any existing files of the same name.`), ), ) intro_form.WithTheme(huh.ThemeCatppuccin()) + profile_create_form.WithTheme(huh.ThemeCatppuccin()) + project_name_form.WithTheme(huh.ThemeCatppuccin()) dbt_form.WithTheme(huh.ThemeCatppuccin()) warehouse_form.WithTheme(huh.ThemeCatppuccin()) snowflake_form.WithTheme(huh.ThemeCatppuccin()) @@ -191,6 +211,16 @@ tbd will overwrite any existing files of the same name.`), log.Fatalf("Error running dbt form %v\n", err) } } else { + err = profile_create_form.Run() + if err != nil { + log.Fatalf("Error running profile create form %v\n", err) + } + if formResponse.ScaffoldProject { + err = project_name_form.Run() + if err != nil { + log.Fatalf("Error running project name form %v\n", err) + } + } err = warehouse_form.Run() if err != nil { log.Fatalf("Error running warehouse form %v\n", err) diff --git a/main.go b/main.go index da8975c..92d5fc8 100644 --- a/main.go +++ b/main.go @@ -50,7 +50,20 @@ func main() { if formResponse.GenerateDescriptions { GenerateColumnDescriptions(ts) } - PrepBuildDir(bd) + err = PrepBuildDir(bd) + if err != nil { + log.Fatalf("Error preparing build directory: %v\n", err) + } + if formResponse.CreateProfile { + WriteProfile(cd, bd) + } + if formResponse.ScaffoldProject { + s, err := WriteScaffoldProject(cd, bd, formResponse.ProjectName) + if err != nil { + log.Fatalf("Error scaffolding project: %v\n", err) + } + bd = s + } err = WriteFiles(ts, bd) if err != nil { log.Fatalf("Error writing files: %v\n", err) diff --git a/prep_build_dir.go b/prep_build_dir.go index 2fac191..172688b 100644 --- a/prep_build_dir.go +++ b/prep_build_dir.go @@ -1,16 +1,30 @@ package main import ( + "errors" "log" "os" ) -func PrepBuildDir(buildDir string) { - _, err := os.Stat(buildDir) +func PrepBuildDir(bd string) error { + _, err := os.Stat(bd) if os.IsNotExist(err) { - dirErr := os.MkdirAll(buildDir, 0755) + dirErr := os.MkdirAll(bd, 0755) if dirErr != nil { - log.Fatalf("Failed to create directory %v", dirErr) + return dirErr } + } else if err == nil { + files, err := os.ReadDir(bd) + if err != nil { + log.Fatalf("Failed to check build target directory %v", err) + } + if len(files) == 0 { + return nil + } else { + return errors.New("build directory is not empty") + } + } else { + return err } + return nil } diff --git a/shared/types.go b/shared/types.go index 2bd0709..e7f28fb 100644 --- a/shared/types.go +++ b/shared/types.go @@ -18,12 +18,13 @@ type SourceTables struct { } type ConnectionDetails struct { - ConnType string - Username string - Account string - Database string - Schema string - Project string - Dataset string - Path string + ConnType string + Username string + Account string + Database string + Schema string + Project string + Dataset string + Path string + ProjectName string } diff --git a/write_profile.go b/write_profile.go new file mode 100644 index 0000000..24a4a85 --- /dev/null +++ b/write_profile.go @@ -0,0 +1,62 @@ +package main + +import ( + "log" + "os" + "path" + "text/template" + + "github.com/gwenwindflower/tbd/shared" +) + +func WriteProfile(cd shared.ConnectionDetails, bd string) { + pt := ` +{{.ConnType}}: + target: dev + outputs: + dev: + type: {{.ConnType}} + {{- if eq .ConnType "snowflake"}} + authenticator: externalbrowser + {{- end}} + {{- if eq .ConnType "bigquery"}} + method: oauth + {{- end}} + {{- if .Account}} + account: {{.Account}} + {{- end}} + {{- if .Username}} + user: {{.Username}} + {{- end}} + {{- if .Database}} + database: {{.Database}} + {{- end}} + {{- if .Project}} + project: {{.Project}} + {{- end}} + {{- if .Schema}} + schema: {{.Schema}} + {{- end}} + {{- if .Dataset}} + dataset: {{.Dataset}} + {{- end}} + {{- if .Path}} + path: {{.Path}} + {{- end}} + threads: 8 +` + tmpl, err := template.New("profiles").Parse(pt) + if err != nil { + log.Fatalf("Failed to parse template %v\n", err) + } + p := path.Join(bd, "profiles.yml") + o, err := os.Create(p) + if err != nil { + log.Fatalf("Failed to create profiles.yml file %v\n", err) + } + defer o.Close() + err = tmpl.Execute(o, cd) + if err != nil { + log.Fatalf("Failed to execute template %v\n", err) + } +} diff --git a/write_profile_test.go b/write_profile_test.go new file mode 100644 index 0000000..5c361d0 --- /dev/null +++ b/write_profile_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "os" + "path" + "testing" + + "github.com/gwenwindflower/tbd/shared" +) + +func TestWriteProfile(t *testing.T) { + cd := shared.ConnectionDetails{ + ConnType: "snowflake", + Username: "aragorn", + Account: "dunedain.snowflakecomputing.com", + Database: "gondor", + Schema: "minas_tirith", + } + tmpDir := t.TempDir() + WriteProfile(cd, tmpDir) + + expected := []byte(` +snowflake: + target: dev + outputs: + dev: + type: snowflake + authenticator: externalbrowser + account: dunedain.snowflakecomputing.com + user: aragorn + database: gondor + schema: minas_tirith + threads: 8 +`) + tpp := path.Join(tmpDir, "profiles.yml") + got, err := os.ReadFile(tpp) + if err != nil { + t.Fatalf("Failed to read profiles.yml: %v", err) + } + // os.Remove("profiles.yml") + if string(got) != string(expected) { + t.Errorf("Expected %s, got %s", expected, got) + } +} diff --git a/write_scaffold_project.go b/write_scaffold_project.go new file mode 100644 index 0000000..d2c40b1 --- /dev/null +++ b/write_scaffold_project.go @@ -0,0 +1,97 @@ +package main + +import ( + "log" + "os" + "path" + "text/template" + + "github.com/gwenwindflower/tbd/shared" +) + +func WriteScaffoldProject(cd shared.ConnectionDetails, bd string, pn string) (string, error) { + folders := []string{"models", "analyses", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} + emptyFolders := []string{"analyses", "macros", "seeds", "snapshots", "data-tests", "models/marts"} + for _, folder := range folders { + p := path.Join(bd, folder) + err := os.MkdirAll(p, 0755) + if err != nil { + return "", err + } + } + for _, folder := range emptyFolders { + p := path.Join(bd, folder, ".gitkeep") + err := os.MkdirAll(p, 0755) + if err != nil { + log.Fatalf("Failed to create .gitkeep in %s folder %v\n", folder, err) + } + } + projectYamlTemplate := `config-version: 2 + +name: {{.ProjectName}} +profile: {{.ConnType}} + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["data-tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + {{.ProjectName}}: + staging: + +materialized: view + marts: + +materialized: table +` + gitignore := []byte(`.venv +venv +.env +env + +target/ +dbt_packages/ +logs/ +profiles.yml + +.DS_Store + +.user.yml + +.ruff_cache +__pycache__ +`) + + tmpl, err := template.New("dbt_project.yml").Parse(projectYamlTemplate) + if err != nil { + log.Fatalf("Failed to parse dbt_project.yml template %v\n", err) + } + p := path.Join(bd, "dbt_project.yml") + o, err := os.Create(p) + if err != nil { + log.Fatalf("Failed to create dbt_project.yml file %v\n", err) + } + defer o.Close() + cd.ProjectName = pn + err = tmpl.Execute(o, cd) + if err != nil { + log.Fatalf("Failed to execute dbt_project.yml template %v\n", err) + } + gi := path.Join(bd, ".gitignore") + err = os.WriteFile(gi, gitignore, 0644) + if err != nil { + log.Fatalf("Failed to write .gitignore file %v\n", err) + } + s := path.Join(bd, "models/staging", cd.Schema) + err = os.MkdirAll(s, 0755) + if err != nil { + return "", err + } + return s, nil +} diff --git a/write_scaffold_project_test.go b/write_scaffold_project_test.go new file mode 100644 index 0000000..fc2f65d --- /dev/null +++ b/write_scaffold_project_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "os" + "path" + "testing" + + "github.com/gwenwindflower/tbd/shared" +) + +func TestWriteScaffoldProject(t *testing.T) { + cd := shared.ConnectionDetails{ + ConnType: "snowflake", + Username: "user", + Account: "account", + Database: "database", + Schema: "schema", + Project: "project", + } + bd := t.TempDir() + pn := "project" + _, err := WriteScaffoldProject(cd, bd, pn) + if err != nil { + t.Fatalf("Error scaffolding project: %v\n", err) + } + // Check that the directories were created + for _, folder := range []string{"models", "analyses", "macros", "seeds", "snapshots", "data-tests", "models/staging", "models/marts"} { + if _, err := os.Stat(path.Join(bd, folder)); os.IsNotExist(err) { + t.Fatalf("Directory %s was not created\n", folder) + } + } + // Check that .gitignore was created correctly + gitignore := []byte(`.venv +venv +.env +env + +target/ +dbt_packages/ +logs/ +profiles.yml + +.DS_Store + +.user.yml + +.ruff_cache +__pycache__ +`) + gi := path.Join(bd, ".gitignore") + got, err := os.ReadFile(gi) + if err != nil { + t.Fatalf("Failed to read .gitignore: %v", err) + } + if string(got) != string(gitignore) { + t.Errorf("Expected %s, got %s", gitignore, got) + } + // Check that project.yml was created correctly + projectYaml := []byte(`config-version: 2 + +name: project +profile: snowflake + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["data-tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + project: + staging: + +materialized: view + marts: + +materialized: table +`) + py := path.Join(bd, "dbt_project.yml") + got, err = os.ReadFile(py) + if err != nil { + t.Fatalf("Failed to read dbt_project.yml: %v", err) + } + if string(got) != string(projectYaml) { + t.Errorf("Expected %s, got %s", projectYaml, got) + } +}