From f27f9cd23eafe64aac31cd135d4d5037ac23b578 Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Tue, 12 Nov 2024 11:51:43 -0500 Subject: [PATCH] move transform command into http service --- .gitignore | 1 + README.md | 49 +++-- internal/handlers/transform.go | 347 +++++++++++++++++++++++++++++++++ main.go | 338 +------------------------------- scripts/download-transform.sh | 15 +- 5 files changed, 401 insertions(+), 349 deletions(-) create mode 100644 internal/handlers/transform.go diff --git a/.gitignore b/.gitignore index 919c33f..56c581f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ fabricator fabricator_pid.txt nohup.out *.csv +*.zip diff --git a/README.md b/README.md index cebbdd4..844dd3f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # fabricator -Prepare a CSV to load via Islandora Workbench - -This is a convenience utility to allow a more user friendly spreadsheet to then be converted to the format Workbench expects. Can be thought of as middleware between normal spreadsheet curation and the format workbench expects. +Transform a Google Sheet URL into a fully executed [Islandora Workbench](https://github.com/mjordan/islandora_workbench) task. ## Overview +- Content creators can work in Google Sheets to prepare a spreadsheet +- [A Google Appscript](./google/appsscript) is embeded in the sheet to allow easily checking their work +- When the spreadsheet is ready, it can be ingested into Islandora/Drupal via Islandora Workbench by supplying the sheet URL in [the GitHub Action](./.github/workflows/run.yml) + ```mermaid sequenceDiagram actor Alice @@ -13,24 +15,41 @@ sequenceDiagram Alice->>Google Sheets: Edit 2 Alice->>Google Sheets: Edit ... Alice->>Google Sheets: Edit N - Google Sheets->>Alice:
Download CSV - Alice->>Fabricator: template.csv - Fabricator->>Fabricator: processing/validating - Fabricator->>Alice: workbench.csv - Alice->>Islandora Workbench: workbench.csv + Alice->>Google Sheets: Click check my work + Google Sheets->>Fabricator: Check this CSV + Fabricator->>Alice: Looks good 🚀 + Alice->>GitHub: Run workbench workflow + GitHub->>Self-hosted Runner: Run workbench workflow + Self-hosted Runner->>Islandora Workbench: python3 workbench Islandora Workbench->>Drupal: entity CUD + Islandora Workbench->>GitHub: logs + Alice->>GitHub: View action logs ``` ## Getting started +### Start the server + ``` -go run main.go \ - --source /path/to/google/sheet.csv \ - --target workbench.csv +export SHARED_SECRET=changeme +go build +nohup ./fabrictor & +echo $! > pid ``` +### Get a workbench CSV from a google sheet CSV -## TODO -- [ ] HTTP service to allow a Google Sheets Apps script to validate a spreadsheet -- [ ] Validator service -- [ ] CSV transform service +``` +curl -s \ + -H "X-Secret: $SHARED_SECRET" \ + -XPOST \ + -o target.zip \ + --upload-file source.csv \ + http://localhost:8080/workbench/transform +``` + +### Stop the service + +``` +kill $(cat pid) +``` diff --git a/internal/handlers/transform.go b/internal/handlers/transform.go new file mode 100644 index 0000000..b041145 --- /dev/null +++ b/internal/handlers/transform.go @@ -0,0 +1,347 @@ +package handlers + +import ( + "archive/zip" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + + "github.com/lehigh-university-libraries/fabricator/internal/contributor" + "github.com/lehigh-university-libraries/fabricator/internal/tgn" + "github.com/lehigh-university-libraries/go-islandora/workbench" +) + +func TransformCsv(w http.ResponseWriter, r *http.Request) { + target := "/tmp/target.csv" + headers, rows, linkedAgents, err := readCSVWithJSONTags(r) + if err != nil { + slog.Error("Failed to read CSV", "err", err) + http.Error(w, "Error parsing CSV", http.StatusBadRequest) + return + } + + file, err := os.Create(target) + if err != nil { + slog.Error("Failed to create file", "err", err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + defer file.Close() + + writer := csv.NewWriter(file) + defer writer.Flush() + + firstRow := make([]string, 0, len(headers)) + for header := range headers { + firstRow = append(firstRow, header) + } + + // finally, write the header to the CSV + if err := writer.Write(firstRow); err != nil { + slog.Error("Failed to write record to CSV", "err", err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + + // write the rows to the CSV + for _, row := range rows { + record := []string{} + for _, header := range firstRow { + record = append(record, strings.Join(row[header], "|")) + } + if err := writer.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "err", err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + } + + files := []string{ + target, + } + if len(linkedAgents) > 1 { + csvFile := strings.Replace(target, ".csv", ".agents.csv", 1) + files = append(files, csvFile) + aFile, err := os.Create(csvFile) + if err != nil { + slog.Error("Failed to create file", "file", csvFile, "err", err) + os.Exit(1) + } + defer aFile.Close() + + aWriter := csv.NewWriter(aFile) + defer aWriter.Flush() + + for _, row := range linkedAgents { + if err := aWriter.Write(row); err != nil { + slog.Error("Failed to write record to CSV", "err", err) + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + } + } + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", "attachment; filename=files.zip") + + zipWriter := zip.NewWriter(w) + defer zipWriter.Close() + + // Iterate over each file path in the slice and add it to the zip archive + for _, filePath := range files { + file, err := os.Open(filePath) + if err != nil { + http.Error(w, "Error opening file: "+err.Error(), http.StatusInternalServerError) + return + } + defer file.Close() + + fileName := filepath.Base(filePath) + zipFile, err := zipWriter.Create(fileName) + if err != nil { + http.Error(w, "Error creating zip entry: "+err.Error(), http.StatusInternalServerError) + return + } + + _, err = io.Copy(zipFile, file) + if err != nil { + http.Error(w, "Error writing to zip: "+err.Error(), http.StatusInternalServerError) + return + } + os.Remove(filePath) + } + +} + +func getJSONFieldName(tag string) string { + if commaIndex := strings.Index(tag, ","); commaIndex != -1 { + return tag[:commaIndex] + } + return tag +} + +func readCSVWithJSONTags(r *http.Request) (map[string]bool, []map[string][]string, [][]string, error) { + defer r.Body.Close() + re := regexp.MustCompile(`^\d{1,4}$`) + reader := csv.NewReader(r.Body) + headers, err := reader.Read() + if err != nil { + return nil, nil, nil, err + } + + var rows []map[string][]string + newHeaders := map[string]bool{} + + var linkedAgents [][]string + linkedAgents = append(linkedAgents, []string{ + "term_name", + "field_contributor_status", + "field_relationships", + "field_email", + "field_identifier", + }) + newCsv := &workbench.SheetsCsv{} + tgnCache := make(map[string]string) + for { + record, err := reader.Read() + if err != nil { + break + } + + row := map[string][]string{} + v := reflect.ValueOf(newCsv).Elem() + t := v.Type() + + for i, header := range headers { + for j := 0; j < t.NumField(); j++ { + field := t.Field(j) + jsonTag := getJSONFieldName(field.Tag.Get("json")) + if jsonTag == header { + value := v.FieldByName(field.Name) + if value.IsValid() && value.CanSet() { + if record[i] == "" { + continue + } + column := getJSONFieldName(field.Tag.Get("csv")) + if column == "" { + return nil, nil, nil, fmt.Errorf("unknown column: %s", jsonTag) + } + originalColumn := column + + values := []string{} + for _, str := range strings.Split(record[i], " ; ") { + switch originalColumn { + case "field_linked_agent": + var c contributor.Contributor + err := json.Unmarshal([]byte(str), &c) + if err != nil { + return nil, nil, nil, fmt.Errorf("error unmarshalling contributor: %s %v", str, err) + } + + str = c.Name + if c.Institution != "" { + str = fmt.Sprintf("%s - %s", str, c.Institution) + } + if c.Status != "" || c.Email != "" || c.Institution != "" || c.Orcid != "" { + name := strings.Split(str, ":") + if len(name) < 4 { + return nil, nil, nil, fmt.Errorf("poorly formatted contributor: %s %v", str, err) + } + agent := []string{ + strings.Join(name[3:], ":"), + c.Status, + fmt.Sprintf("schema:worksFor:corporate_body:%s", c.Institution), + c.Email, + fmt.Sprintf(`{"attr0": "orcid", "value": "%s"}`, c.Orcid), + } + if c.Institution == "" { + agent[2] = "" + } + if c.Orcid == "" { + agent[4] = "" + } + + linkedAgents = append(linkedAgents, agent) + } + + case "field_add_coverpage", "published": + switch str { + case "Yes": + str = "1" + case "No": + str = "0" + default: + return nil, nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) + } + case "id", "parent_id": + if !re.MatchString(str) { + return nil, nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) + } + case "field_weight": + _, err := strconv.Atoi(str) + if err != nil { + return nil, nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) + } + str = strings.TrimLeft(str, "0") + case "field_subject_hierarchical_geo": + if _, ok := tgnCache[str]; ok { + str = tgnCache[str] + break + } + + tgn, err := tgn.GetLocationFromTGN(str) + if err != nil { + return nil, nil, nil, fmt.Errorf("unknown TGN: %s %v", str, err) + } + + locationJSON, err := json.Marshal(tgn) + if err != nil { + return nil, nil, nil, fmt.Errorf("error marshalling TGN: %s %v", str, err) + } + tgnCache[str] = string(locationJSON) + str = tgnCache[str] + + case "field_rights": + switch str { + case "IN COPYRIGHT": + str = "http://rightsstatements.org/vocab/InC/1.0/" + case "IN COPYRIGHT - EU ORPHAN WORK": + str = "http://rightsstatements.org/vocab/InC-OW-EU/1.0/" + case "IN COPYRIGHT - EDUCATIONAL USE PERMITTED": + str = "http://rightsstatements.org/vocab/InC-EDU/1.0/" + case "IN COPYRIGHT - NON-COMMERCIAL USE PERMITTED": + str = "http://rightsstatements.org/vocab/InC-NC/1.0/" + case "IN COPYRIGHT - RIGHTS-HOLDER(S) UNLOCATABLE OR UNIDENTIFIABLE": + str = "http://rightsstatements.org/vocab/InC-RUU/1.0/" + case "NO COPYRIGHT - CONTRACTUAL RESTRICTIONS": + str = "http://rightsstatements.org/vocab/NoC-CR/1.0/" + case "NO COPYRIGHT - NON-COMMERCIAL USE ONLY": + str = "http://rightsstatements.org/vocab/NoC-NC/1.0/" + case "NO COPYRIGHT - OTHER KNOWN LEGAL RESTRICTIONS": + str = "http://rightsstatements.org/vocab/NoC-OKLR/1.0/" + case "NO COPYRIGHT - UNITED STATES": + str = "http://rightsstatements.org/vocab/NoC-US/1.0/" + case "COPYRIGHT NOT EVALUATED": + str = "http://rightsstatements.org/vocab/CNE/1.0/" + case "COPYRIGHT UNDETERMINED": + str = "http://rightsstatements.org/vocab/UND/1.0/" + case "NO KNOWN COPYRIGHT": + str = "http://rightsstatements.org/vocab/NKC/1.0/" + default: + return nil, nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) + } + case "field_extent.attr0=page", + "field_extent.attr0=dimensions", + "field_extent.attr0=bytes", + "field_extent.attr0=minutes", + "field_abstract.attr0=description", + "field_abstract.attr0=abstract", + "field_note.attr0=preferred-citation", + "field_note.attr0=capture-device", + "field_note.attr0=ppi", + "field_note.attr0=collection", + "field_note.attr0=box", + "field_note.attr0=series", + "field_note.attr0=folder", + "field_part_detail.attr0=volume", + "field_part_detail.attr0=issue", + "field_part_detail.attr0=page", + "field_identifier.attr0=doi", + "field_identifier.attr0=uri", + "field_identifier.attr0=call-number", + "field_identifier.attr0=report-number": + components := strings.Split(originalColumn, ".attr0=") + str = strings.ReplaceAll(str, `\`, `\\`) + str = strings.ReplaceAll(str, `"`, `\"`) + column = components[0] + if column == "field_part_detail" { + str = fmt.Sprintf(`{"number":"%s","type":"%s"}`, str, components[1]) + + } else { + str = fmt.Sprintf(`{"value":"%s","attr0":"%s"}`, str, components[1]) + } + case "field_geographic_subject.vid=geographic_naf", + "field_geographic_subject.vid=geographic_local": + components := strings.Split(originalColumn, ".vid=") + column = components[0] + str = fmt.Sprintf("%s:%s", components[1], str) + case "field_related_item.title": + column = "field_related_item" + str = fmt.Sprintf(`{"title": "%s"}`, str) + case "field_related_item.identifier_type=issn": + column = "field_related_item" + str = fmt.Sprintf(`{"type": "issn", "identifier": "%s"}`, str) + case "file": + str = strings.ReplaceAll(str, `\`, `/`) + str = strings.TrimLeft(str, "/") + if len(str) > 3 && str[0:3] != "mnt" { + str = fmt.Sprintf("/mnt/islandora_staging/%s", str) + } + } + + str = strings.TrimSpace(str) + values = append(values, str) + } + + newHeaders[column] = true + // replace the locally defined google sheets cell delimiter + // with workbench's pipe delimiter + row[column] = append(row[column], strings.Join(values, "|")) + } + } + } + } + + rows = append(rows, row) + } + + return newHeaders, rows, linkedAgents, nil +} diff --git a/main.go b/main.go index 664afec..71c9546 100644 --- a/main.go +++ b/main.go @@ -1,346 +1,18 @@ package main import ( - "encoding/csv" - "encoding/json" - "flag" - "fmt" "log/slog" "net/http" - "os" - "reflect" - "regexp" - "strconv" - "strings" - "github.com/lehigh-university-libraries/fabricator/internal/contributor" "github.com/lehigh-university-libraries/fabricator/internal/handlers" - "github.com/lehigh-university-libraries/fabricator/internal/tgn" - "github.com/lehigh-university-libraries/go-islandora/workbench" ) -var linkedAgents [][]string - -func getJSONFieldName(tag string) string { - if commaIndex := strings.Index(tag, ","); commaIndex != -1 { - return tag[:commaIndex] - } - return tag -} - -func readCSVWithJSONTags(filePath string) (map[string]bool, []map[string][]string, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, nil, err - } - defer file.Close() - re := regexp.MustCompile(`^\d{1,4}$`) - reader := csv.NewReader(file) - headers, err := reader.Read() - if err != nil { - return nil, nil, err - } - - var rows []map[string][]string - newHeaders := map[string]bool{} - linkedAgents = append(linkedAgents, []string{ - "term_name", - "field_contributor_status", - "field_relationships", - "field_email", - "field_identifier", - }) - newCsv := &workbench.SheetsCsv{} - tgnCache := make(map[string]string) - for { - record, err := reader.Read() - if err != nil { - break - } - - row := map[string][]string{} - v := reflect.ValueOf(newCsv).Elem() - t := v.Type() - - for i, header := range headers { - for j := 0; j < t.NumField(); j++ { - field := t.Field(j) - jsonTag := getJSONFieldName(field.Tag.Get("json")) - if jsonTag == header { - value := v.FieldByName(field.Name) - if value.IsValid() && value.CanSet() { - if record[i] == "" { - continue - } - column := getJSONFieldName(field.Tag.Get("csv")) - if column == "" { - return nil, nil, fmt.Errorf("unknown column: %s", jsonTag) - } - originalColumn := column - - values := []string{} - for _, str := range strings.Split(record[i], " ; ") { - switch originalColumn { - case "field_linked_agent": - var c contributor.Contributor - err := json.Unmarshal([]byte(str), &c) - if err != nil { - return nil, nil, fmt.Errorf("error unmarshalling contributor: %s %v", str, err) - } - - str = c.Name - if c.Institution != "" { - str = fmt.Sprintf("%s - %s", str, c.Institution) - } - if c.Status != "" || c.Email != "" || c.Institution != "" || c.Orcid != "" { - name := strings.Split(str, ":") - if len(name) < 4 { - return nil, nil, fmt.Errorf("poorly formatted contributor: %s %v", str, err) - } - agent := []string{ - strings.Join(name[3:], ":"), - c.Status, - fmt.Sprintf("schema:worksFor:corporate_body:%s", c.Institution), - c.Email, - fmt.Sprintf(`{"attr0": "orcid", "value": "%s"}`, c.Orcid), - } - if c.Institution == "" { - agent[2] = "" - } - if c.Orcid == "" { - agent[4] = "" - } - - linkedAgents = append(linkedAgents, agent) - } - - case "field_add_coverpage", "published": - switch str { - case "Yes": - str = "1" - case "No": - str = "0" - default: - return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) - } - case "id", "parent_id": - if !re.MatchString(str) { - return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) - } - case "field_weight": - _, err := strconv.Atoi(str) - if err != nil { - return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) - } - str = strings.TrimLeft(str, "0") - case "field_subject_hierarchical_geo": - if _, ok := tgnCache[str]; ok { - str = tgnCache[str] - break - } - - tgn, err := tgn.GetLocationFromTGN(str) - if err != nil { - return nil, nil, fmt.Errorf("unknown TGN: %s %v", str, err) - } - - locationJSON, err := json.Marshal(tgn) - if err != nil { - return nil, nil, fmt.Errorf("error marshalling TGN: %s %v", str, err) - } - tgnCache[str] = string(locationJSON) - str = tgnCache[str] - - case "field_rights": - switch str { - case "IN COPYRIGHT": - str = "http://rightsstatements.org/vocab/InC/1.0/" - case "IN COPYRIGHT - EU ORPHAN WORK": - str = "http://rightsstatements.org/vocab/InC-OW-EU/1.0/" - case "IN COPYRIGHT - EDUCATIONAL USE PERMITTED": - str = "http://rightsstatements.org/vocab/InC-EDU/1.0/" - case "IN COPYRIGHT - NON-COMMERCIAL USE PERMITTED": - str = "http://rightsstatements.org/vocab/InC-NC/1.0/" - case "IN COPYRIGHT - RIGHTS-HOLDER(S) UNLOCATABLE OR UNIDENTIFIABLE": - str = "http://rightsstatements.org/vocab/InC-RUU/1.0/" - case "NO COPYRIGHT - CONTRACTUAL RESTRICTIONS": - str = "http://rightsstatements.org/vocab/NoC-CR/1.0/" - case "NO COPYRIGHT - NON-COMMERCIAL USE ONLY": - str = "http://rightsstatements.org/vocab/NoC-NC/1.0/" - case "NO COPYRIGHT - OTHER KNOWN LEGAL RESTRICTIONS": - str = "http://rightsstatements.org/vocab/NoC-OKLR/1.0/" - case "NO COPYRIGHT - UNITED STATES": - str = "http://rightsstatements.org/vocab/NoC-US/1.0/" - case "COPYRIGHT NOT EVALUATED": - str = "http://rightsstatements.org/vocab/CNE/1.0/" - case "COPYRIGHT UNDETERMINED": - str = "http://rightsstatements.org/vocab/UND/1.0/" - case "NO KNOWN COPYRIGHT": - str = "http://rightsstatements.org/vocab/NKC/1.0/" - default: - return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str) - } - case "field_extent.attr0=page", - "field_extent.attr0=dimensions", - "field_extent.attr0=bytes", - "field_extent.attr0=minutes", - "field_abstract.attr0=description", - "field_abstract.attr0=abstract", - "field_note.attr0=preferred-citation", - "field_note.attr0=capture-device", - "field_note.attr0=ppi", - "field_note.attr0=collection", - "field_note.attr0=box", - "field_note.attr0=series", - "field_note.attr0=folder", - "field_part_detail.attr0=volume", - "field_part_detail.attr0=issue", - "field_part_detail.attr0=page", - "field_identifier.attr0=doi", - "field_identifier.attr0=uri", - "field_identifier.attr0=call-number", - "field_identifier.attr0=report-number": - components := strings.Split(originalColumn, ".attr0=") - str = strings.ReplaceAll(str, `\`, `\\`) - str = strings.ReplaceAll(str, `"`, `\"`) - column = components[0] - if column == "field_part_detail" { - str = fmt.Sprintf(`{"number":"%s","type":"%s"}`, str, components[1]) - - } else { - str = fmt.Sprintf(`{"value":"%s","attr0":"%s"}`, str, components[1]) - } - case "field_geographic_subject.vid=geographic_naf", - "field_geographic_subject.vid=geographic_local": - components := strings.Split(originalColumn, ".vid=") - column = components[0] - str = fmt.Sprintf("%s:%s", components[1], str) - case "field_related_item.title": - column = "field_related_item" - str = fmt.Sprintf(`{"title": "%s"}`, str) - case "field_related_item.identifier_type=issn": - column = "field_related_item" - str = fmt.Sprintf(`{"type": "issn", "identifier": "%s"}`, str) - case "file": - str = strings.ReplaceAll(str, `\`, `/`) - str = strings.TrimLeft(str, "/") - if len(str) > 3 && str[0:3] != "mnt" { - str = fmt.Sprintf("/mnt/islandora_staging/%s", str) - } - } - - str = strings.TrimSpace(str) - values = append(values, str) - } - - newHeaders[column] = true - // replace the locally defined google sheets cell delimiter - // with workbench's pipe delimiter - row[column] = append(row[column], strings.Join(values, "|")) - } - } - } - } - - rows = append(rows, row) - } - - return newHeaders, rows, nil -} - func main() { - var serverMode bool - - flag.BoolVar(&serverMode, "server", false, "Set to true to run as server") - source := flag.String("source", "", "Path to the source CSV file") - target := flag.String("target", "", "Path to the target CSV file") - flag.Parse() - - if serverMode { - // Start HTTP server - http.HandleFunc("/workbench/check", handlers.CheckMyWork) - - slog.Info("Starting server on :8080") - if err := http.ListenAndServe(":8080", nil); err != nil { - panic(err) - } - } - - // Define the source and target flags - - if *source == "" || *target == "" { - fmt.Println("Source and target flags are required") - flag.Usage() - return - } - headers, rows, err := readCSVWithJSONTags(*source) - if err != nil { - slog.Error("Failed to read CSV", "err", err) - os.Exit(1) - } - - file, err := os.Create(*target) - if err != nil { - slog.Error("Failed to create file", "err", err) - os.Exit(1) - } - defer file.Close() - - writer := csv.NewWriter(file) - defer writer.Flush() - - firstRow := make([]string, 0, len(headers)) - for header := range headers { - firstRow = append(firstRow, header) - } - - // finally, write the header to the CSV - if err := writer.Write(firstRow); err != nil { - slog.Error("Failed to write record to CSV", "err", err) - os.Exit(1) - } - - // write the rows to the CSV - for _, row := range rows { - record := []string{} - for _, header := range firstRow { - record = append(record, strings.Join(row[header], "|")) - } - if err := writer.Write(record); err != nil { - slog.Error("Failed to write record to CSV", "err", err) - os.Exit(1) - } - } - slog.Info("CSV file has been written successfully") - if len(linkedAgents) == 1 { - return - } - - csvFile := strings.Replace(*target, ".csv", ".agents.csv", 1) - aFile, err := os.Create(csvFile) - if err != nil { - slog.Error("Failed to create file", "file", csvFile, "err", err) - os.Exit(1) - } - defer aFile.Close() - - aWriter := csv.NewWriter(aFile) - defer aWriter.Flush() - - for _, row := range linkedAgents { - if err := aWriter.Write(row); err != nil { - slog.Error("Failed to write record to CSV", "err", err) - os.Exit(1) - } - } - slog.Info("Linked Agent CSV file has been written successfully") -} + http.HandleFunc("/workbench/check", handlers.CheckMyWork) + http.HandleFunc("/workbench/transform", handlers.TransformCsv) -func StrInSlice(s string, sl []string) bool { - for _, a := range sl { - if a == s { - return true - } + slog.Info("Starting server on :8080") + if err := http.ListenAndServe(":8080", nil); err != nil { + panic(err) } - return false } diff --git a/scripts/download-transform.sh b/scripts/download-transform.sh index b026b38..502b280 100755 --- a/scripts/download-transform.sh +++ b/scripts/download-transform.sh @@ -43,7 +43,20 @@ if [ "${STATUS}" -gt 299 ] || [ "${STATUS}" -lt 200 ]; then fi # transform google sheet to a workbench CSV -./fabricator --source source.csv --target target.csv +STATUS=$(curl -v \ + -w '%{http_code}' \ + -H "X-Secret: $SHARED_SECRET" \ + -XPOST \ + -o target.zip \ + --upload-file source.csv \ + http://localhost:8080/workbench/transform) +if [ "${STATUS}" -gt 299 ] || [ "${STATUS}" -lt 200 ]; then + echo "CSV transform failed" + exit 1 +fi + +unzip target.zip +rm target.zip # make sure source and target CSVs line count match SOURCE=$(wc -l < source.csv)