From 9332ea1ce7dd8448ba7d234c80552591644ddde1 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 19 Nov 2024 05:51:58 +0000 Subject: [PATCH 01/13] added sbo --- .../storagebatchoperations/product.yaml | 21 ++ mmv1/products/storagebatchoperations/sbo.yaml | 54 +++ ...tch_operations_storage_batch_operations.go | 344 ++++++++++++++++++ ...orage_batch_operations_generated_meta.yaml | 5 + 4 files changed, 424 insertions(+) create mode 100644 mmv1/products/storagebatchoperations/product.yaml create mode 100644 mmv1/products/storagebatchoperations/sbo.yaml create mode 100644 mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go create mode 100644 mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml diff --git a/mmv1/products/storagebatchoperations/product.yaml b/mmv1/products/storagebatchoperations/product.yaml new file mode 100644 index 000000000000..63afb5d70b56 --- /dev/null +++ b/mmv1/products/storagebatchoperations/product.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: 'StorageBatchOperations' +display_name: 'Cloud Storage Batch Operations' +versions: + - name: 'ga' + base_url: 'https://storagebatchoperations.googleapis.com/v1/' +scopes: + - 'https://www.googleapis.com/auth/cloud-platform' diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml new file mode 100644 index 000000000000..f45df034f467 --- /dev/null +++ b/mmv1/products/storagebatchoperations/sbo.yaml @@ -0,0 +1,54 @@ +# Copyright 2024 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: 'StorageBatchOperations' +kind: 'storagebatchoperations#jobs' +id_format: '{{job_id}}' +base_url: 'project/{{project}}/location/{{location}}/jobs' +self_link: 'project/{{project}}/location/{{location}}/jobs' +has_self_link: true +delete_url: 'project/{{project}}/location/{{location}}/jobs' +timeouts: + insert_minutes: 20 + update_minutes: 20 + delete_minutes: 20 +exclude_sweeper: true +description: | + Storage Batch Operations (SBO) is a Cloud Storage management feature that offers a + seamless experience to perform single batch operations on millions of GCS objects in a + serverless manner. +parameters: + - name: 'location' + type: String + description: 'The location of the job.' + immutable: true + imports: 'location' + - name: 'prefixListFile' + type: String + description: one of prefixlist or manifestLocation allowed + - name: 'manifestLocation' + type: String + description: on eof prefixlistfile or manifestlocation alloowed + +properties: + - name: 'createTime' + type: String + description: | + The timestamp at which this managed folder was created. + output: true + - name: 'updateTime' + type: String + description: | + The timestamp at which this managed folder was most recently updated. + output: true diff --git a/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go b/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go new file mode 100644 index 000000000000..4cf9c458cc49 --- /dev/null +++ b/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go @@ -0,0 +1,344 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// ---------------------------------------------------------------------------- +// +// *** AUTO GENERATED CODE *** Type: MMv1 *** +// +// ---------------------------------------------------------------------------- +// +// This file is automatically generated by Magic Modules and manual +// changes will be clobbered when the file is regenerated. +// +// Please read more about how to change this file in +// .github/CONTRIBUTING.md. +// +// ---------------------------------------------------------------------------- + +package storagebatchoperations + +import ( + "fmt" + "log" + "net/http" + "reflect" + "time" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + + "github.com/hashicorp/terraform-provider-google/google/tpgresource" + transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" +) + +func ResourceStorageBatchOperationsStorageBatchOperations() *schema.Resource { + return &schema.Resource{ + Create: resourceStorageBatchOperationsStorageBatchOperationsCreate, + Read: resourceStorageBatchOperationsStorageBatchOperationsRead, + Update: resourceStorageBatchOperationsStorageBatchOperationsUpdate, + Delete: resourceStorageBatchOperationsStorageBatchOperationsDelete, + + Importer: &schema.ResourceImporter{ + State: resourceStorageBatchOperationsStorageBatchOperationsImport, + }, + + Timeouts: &schema.ResourceTimeout{ + Create: schema.DefaultTimeout(20 * time.Minute), + Update: schema.DefaultTimeout(20 * time.Minute), + Delete: schema.DefaultTimeout(20 * time.Minute), + }, + + CustomizeDiff: customdiff.All( + tpgresource.DefaultProviderProject, + ), + + Schema: map[string]*schema.Schema{ + "location": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `The location of the job.`, + }, + "create_time": { + Type: schema.TypeString, + Computed: true, + Description: `The timestamp at which this managed folder was created.`, + }, + "update_time": { + Type: schema.TypeString, + Computed: true, + Description: `The timestamp at which this managed folder was most recently updated.`, + }, + "project": { + Type: schema.TypeString, + Optional: true, + Computed: true, + ForceNew: true, + }, + "self_link": { + Type: schema.TypeString, + Computed: true, + }, + }, + UseJSONNumber: true, + } +} + +func resourceStorageBatchOperationsStorageBatchOperationsCreate(d *schema.ResourceData, meta interface{}) error { + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + obj := make(map[string]interface{}) + locationProp, err := expandStorageBatchOperationsStorageBatchOperationsLocation(d.Get("location"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("location"); !tpgresource.IsEmptyValue(reflect.ValueOf(locationProp)) && (ok || !reflect.DeepEqual(v, locationProp)) { + obj["location"] = locationProp + } + + url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") + if err != nil { + return err + } + + log.Printf("[DEBUG] Creating new StorageBatchOperations: %#v", obj) + billingProject := "" + + project, err := tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) + } + billingProject = project + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + headers := make(http.Header) + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "POST", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, + Body: obj, + Timeout: d.Timeout(schema.TimeoutCreate), + Headers: headers, + }) + if err != nil { + return fmt.Errorf("Error creating StorageBatchOperations: %s", err) + } + + // Store the ID now + id, err := tpgresource.ReplaceVars(d, config, "{{job_id}}") + if err != nil { + return fmt.Errorf("Error constructing id: %s", err) + } + d.SetId(id) + + log.Printf("[DEBUG] Finished creating StorageBatchOperations %q: %#v", d.Id(), res) + + return resourceStorageBatchOperationsStorageBatchOperationsRead(d, meta) +} + +func resourceStorageBatchOperationsStorageBatchOperationsRead(d *schema.ResourceData, meta interface{}) error { + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") + if err != nil { + return err + } + + billingProject := "" + + project, err := tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) + } + billingProject = project + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + headers := make(http.Header) + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "GET", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, + Headers: headers, + }) + if err != nil { + return transport_tpg.HandleNotFoundError(err, d, fmt.Sprintf("StorageBatchOperationsStorageBatchOperations %q", d.Id())) + } + + if err := d.Set("project", project); err != nil { + return fmt.Errorf("Error reading StorageBatchOperations: %s", err) + } + + if err := d.Set("create_time", flattenStorageBatchOperationsStorageBatchOperationsCreateTime(res["createTime"], d, config)); err != nil { + return fmt.Errorf("Error reading StorageBatchOperations: %s", err) + } + if err := d.Set("update_time", flattenStorageBatchOperationsStorageBatchOperationsUpdateTime(res["updateTime"], d, config)); err != nil { + return fmt.Errorf("Error reading StorageBatchOperations: %s", err) + } + if err := d.Set("location", flattenStorageBatchOperationsStorageBatchOperationsLocation(res["location"], d, config)); err != nil { + return fmt.Errorf("Error reading StorageBatchOperations: %s", err) + } + if err := d.Set("self_link", tpgresource.ConvertSelfLinkToV1(res["selfLink"].(string))); err != nil { + return fmt.Errorf("Error reading StorageBatchOperations: %s", err) + } + + return nil +} + +func resourceStorageBatchOperationsStorageBatchOperationsUpdate(d *schema.ResourceData, meta interface{}) error { + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + billingProject := "" + + project, err := tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) + } + billingProject = project + + obj := make(map[string]interface{}) + locationProp, err := expandStorageBatchOperationsStorageBatchOperationsLocation(d.Get("location"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("location"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, locationProp)) { + obj["location"] = locationProp + } + + url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") + if err != nil { + return err + } + + log.Printf("[DEBUG] Updating StorageBatchOperations %q: %#v", d.Id(), obj) + headers := make(http.Header) + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "PUT", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, + Body: obj, + Timeout: d.Timeout(schema.TimeoutUpdate), + Headers: headers, + }) + + if err != nil { + return fmt.Errorf("Error updating StorageBatchOperations %q: %s", d.Id(), err) + } else { + log.Printf("[DEBUG] Finished updating StorageBatchOperations %q: %#v", d.Id(), res) + } + + return resourceStorageBatchOperationsStorageBatchOperationsRead(d, meta) +} + +func resourceStorageBatchOperationsStorageBatchOperationsDelete(d *schema.ResourceData, meta interface{}) error { + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + billingProject := "" + + project, err := tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) + } + billingProject = project + + url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") + if err != nil { + return err + } + + var obj map[string]interface{} + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + headers := make(http.Header) + + log.Printf("[DEBUG] Deleting StorageBatchOperations %q", d.Id()) + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "DELETE", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, + Body: obj, + Timeout: d.Timeout(schema.TimeoutDelete), + Headers: headers, + }) + if err != nil { + return transport_tpg.HandleNotFoundError(err, d, "StorageBatchOperations") + } + + log.Printf("[DEBUG] Finished deleting StorageBatchOperations %q: %#v", d.Id(), res) + return nil +} + +func resourceStorageBatchOperationsStorageBatchOperationsImport(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) { + config := meta.(*transport_tpg.Config) + if err := tpgresource.ParseImportId([]string{ + "^project/(?P[^/]+)/location/(?P[^/]+)/jobs/(?P[^/]+)$", + "^(?P[^/]+)/(?P[^/]+)/(?P[^/]+)$", + "^(?P[^/]+)/(?P[^/]+)$", + }, d, config); err != nil { + return nil, err + } + + // Replace import id for the resource id + id, err := tpgresource.ReplaceVars(d, config, "{{job_id}}") + if err != nil { + return nil, fmt.Errorf("Error constructing id: %s", err) + } + d.SetId(id) + + return []*schema.ResourceData{d}, nil +} + +func flattenStorageBatchOperationsStorageBatchOperationsCreateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { + return v +} + +func flattenStorageBatchOperationsStorageBatchOperationsUpdateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { + return v +} + +func flattenStorageBatchOperationsStorageBatchOperationsLocation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { + return v +} + +func expandStorageBatchOperationsStorageBatchOperationsLocation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { + return v, nil +} diff --git a/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml b/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml new file mode 100644 index 000000000000..814ff1ed5a3a --- /dev/null +++ b/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml @@ -0,0 +1,5 @@ +resource: 'google_storage_batch_operations_storage_batch_operations' +generation_type: 'mmv1' +api_service_name: 'storagebatchoperations.googleapis.com' +api_version: 'v1' +api_resource_type_kind: 'StorageBatchOperations' From f88b07f736dea993d969b22b727c2a6679779ca1 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 19 Nov 2024 06:06:02 +0000 Subject: [PATCH 02/13] yaml sybtax --- mmv1/products/storagebatchoperations/sbo.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml index f45df034f467..81fd9cbe5eca 100644 --- a/mmv1/products/storagebatchoperations/sbo.yaml +++ b/mmv1/products/storagebatchoperations/sbo.yaml @@ -25,8 +25,8 @@ timeouts: delete_minutes: 20 exclude_sweeper: true description: | - Storage Batch Operations (SBO) is a Cloud Storage management feature that offers a - seamless experience to perform single batch operations on millions of GCS objects in a + Storage Batch Operations (SBO) is a Cloud Storage management feature that offers a + seamless experience to perform single batch operations on millions of GCS objects in a serverless manner. parameters: - name: 'location' @@ -39,7 +39,7 @@ parameters: description: one of prefixlist or manifestLocation allowed - name: 'manifestLocation' type: String - description: on eof prefixlistfile or manifestlocation alloowed + description: one of prefixlistfile or manifestlocation allowed properties: - name: 'createTime' From 1e68c653326e5a9ad55bfd974b4176a63d40e1b9 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 19 Nov 2024 06:16:47 +0000 Subject: [PATCH 03/13] deleted --- ...tch_operations_storage_batch_operations.go | 344 ------------------ ...orage_batch_operations_generated_meta.yaml | 5 - 2 files changed, 349 deletions(-) delete mode 100644 mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go delete mode 100644 mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml diff --git a/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go b/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go deleted file mode 100644 index 4cf9c458cc49..000000000000 --- a/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations.go +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -// ---------------------------------------------------------------------------- -// -// *** AUTO GENERATED CODE *** Type: MMv1 *** -// -// ---------------------------------------------------------------------------- -// -// This file is automatically generated by Magic Modules and manual -// changes will be clobbered when the file is regenerated. -// -// Please read more about how to change this file in -// .github/CONTRIBUTING.md. -// -// ---------------------------------------------------------------------------- - -package storagebatchoperations - -import ( - "fmt" - "log" - "net/http" - "reflect" - "time" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - - "github.com/hashicorp/terraform-provider-google/google/tpgresource" - transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" -) - -func ResourceStorageBatchOperationsStorageBatchOperations() *schema.Resource { - return &schema.Resource{ - Create: resourceStorageBatchOperationsStorageBatchOperationsCreate, - Read: resourceStorageBatchOperationsStorageBatchOperationsRead, - Update: resourceStorageBatchOperationsStorageBatchOperationsUpdate, - Delete: resourceStorageBatchOperationsStorageBatchOperationsDelete, - - Importer: &schema.ResourceImporter{ - State: resourceStorageBatchOperationsStorageBatchOperationsImport, - }, - - Timeouts: &schema.ResourceTimeout{ - Create: schema.DefaultTimeout(20 * time.Minute), - Update: schema.DefaultTimeout(20 * time.Minute), - Delete: schema.DefaultTimeout(20 * time.Minute), - }, - - CustomizeDiff: customdiff.All( - tpgresource.DefaultProviderProject, - ), - - Schema: map[string]*schema.Schema{ - "location": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Description: `The location of the job.`, - }, - "create_time": { - Type: schema.TypeString, - Computed: true, - Description: `The timestamp at which this managed folder was created.`, - }, - "update_time": { - Type: schema.TypeString, - Computed: true, - Description: `The timestamp at which this managed folder was most recently updated.`, - }, - "project": { - Type: schema.TypeString, - Optional: true, - Computed: true, - ForceNew: true, - }, - "self_link": { - Type: schema.TypeString, - Computed: true, - }, - }, - UseJSONNumber: true, - } -} - -func resourceStorageBatchOperationsStorageBatchOperationsCreate(d *schema.ResourceData, meta interface{}) error { - config := meta.(*transport_tpg.Config) - userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) - if err != nil { - return err - } - - obj := make(map[string]interface{}) - locationProp, err := expandStorageBatchOperationsStorageBatchOperationsLocation(d.Get("location"), d, config) - if err != nil { - return err - } else if v, ok := d.GetOkExists("location"); !tpgresource.IsEmptyValue(reflect.ValueOf(locationProp)) && (ok || !reflect.DeepEqual(v, locationProp)) { - obj["location"] = locationProp - } - - url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") - if err != nil { - return err - } - - log.Printf("[DEBUG] Creating new StorageBatchOperations: %#v", obj) - billingProject := "" - - project, err := tpgresource.GetProject(d, config) - if err != nil { - return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) - } - billingProject = project - - // err == nil indicates that the billing_project value was found - if bp, err := tpgresource.GetBillingProject(d, config); err == nil { - billingProject = bp - } - - headers := make(http.Header) - res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "POST", - Project: billingProject, - RawURL: url, - UserAgent: userAgent, - Body: obj, - Timeout: d.Timeout(schema.TimeoutCreate), - Headers: headers, - }) - if err != nil { - return fmt.Errorf("Error creating StorageBatchOperations: %s", err) - } - - // Store the ID now - id, err := tpgresource.ReplaceVars(d, config, "{{job_id}}") - if err != nil { - return fmt.Errorf("Error constructing id: %s", err) - } - d.SetId(id) - - log.Printf("[DEBUG] Finished creating StorageBatchOperations %q: %#v", d.Id(), res) - - return resourceStorageBatchOperationsStorageBatchOperationsRead(d, meta) -} - -func resourceStorageBatchOperationsStorageBatchOperationsRead(d *schema.ResourceData, meta interface{}) error { - config := meta.(*transport_tpg.Config) - userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) - if err != nil { - return err - } - - url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") - if err != nil { - return err - } - - billingProject := "" - - project, err := tpgresource.GetProject(d, config) - if err != nil { - return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) - } - billingProject = project - - // err == nil indicates that the billing_project value was found - if bp, err := tpgresource.GetBillingProject(d, config); err == nil { - billingProject = bp - } - - headers := make(http.Header) - res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: userAgent, - Headers: headers, - }) - if err != nil { - return transport_tpg.HandleNotFoundError(err, d, fmt.Sprintf("StorageBatchOperationsStorageBatchOperations %q", d.Id())) - } - - if err := d.Set("project", project); err != nil { - return fmt.Errorf("Error reading StorageBatchOperations: %s", err) - } - - if err := d.Set("create_time", flattenStorageBatchOperationsStorageBatchOperationsCreateTime(res["createTime"], d, config)); err != nil { - return fmt.Errorf("Error reading StorageBatchOperations: %s", err) - } - if err := d.Set("update_time", flattenStorageBatchOperationsStorageBatchOperationsUpdateTime(res["updateTime"], d, config)); err != nil { - return fmt.Errorf("Error reading StorageBatchOperations: %s", err) - } - if err := d.Set("location", flattenStorageBatchOperationsStorageBatchOperationsLocation(res["location"], d, config)); err != nil { - return fmt.Errorf("Error reading StorageBatchOperations: %s", err) - } - if err := d.Set("self_link", tpgresource.ConvertSelfLinkToV1(res["selfLink"].(string))); err != nil { - return fmt.Errorf("Error reading StorageBatchOperations: %s", err) - } - - return nil -} - -func resourceStorageBatchOperationsStorageBatchOperationsUpdate(d *schema.ResourceData, meta interface{}) error { - config := meta.(*transport_tpg.Config) - userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) - if err != nil { - return err - } - - billingProject := "" - - project, err := tpgresource.GetProject(d, config) - if err != nil { - return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) - } - billingProject = project - - obj := make(map[string]interface{}) - locationProp, err := expandStorageBatchOperationsStorageBatchOperationsLocation(d.Get("location"), d, config) - if err != nil { - return err - } else if v, ok := d.GetOkExists("location"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, locationProp)) { - obj["location"] = locationProp - } - - url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") - if err != nil { - return err - } - - log.Printf("[DEBUG] Updating StorageBatchOperations %q: %#v", d.Id(), obj) - headers := make(http.Header) - - // err == nil indicates that the billing_project value was found - if bp, err := tpgresource.GetBillingProject(d, config); err == nil { - billingProject = bp - } - - res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "PUT", - Project: billingProject, - RawURL: url, - UserAgent: userAgent, - Body: obj, - Timeout: d.Timeout(schema.TimeoutUpdate), - Headers: headers, - }) - - if err != nil { - return fmt.Errorf("Error updating StorageBatchOperations %q: %s", d.Id(), err) - } else { - log.Printf("[DEBUG] Finished updating StorageBatchOperations %q: %#v", d.Id(), res) - } - - return resourceStorageBatchOperationsStorageBatchOperationsRead(d, meta) -} - -func resourceStorageBatchOperationsStorageBatchOperationsDelete(d *schema.ResourceData, meta interface{}) error { - config := meta.(*transport_tpg.Config) - userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) - if err != nil { - return err - } - - billingProject := "" - - project, err := tpgresource.GetProject(d, config) - if err != nil { - return fmt.Errorf("Error fetching project for StorageBatchOperations: %s", err) - } - billingProject = project - - url, err := tpgresource.ReplaceVars(d, config, "{{StorageBatchOperationsBasePath}}project/{{project}}/location/{{location}}/jobs") - if err != nil { - return err - } - - var obj map[string]interface{} - - // err == nil indicates that the billing_project value was found - if bp, err := tpgresource.GetBillingProject(d, config); err == nil { - billingProject = bp - } - - headers := make(http.Header) - - log.Printf("[DEBUG] Deleting StorageBatchOperations %q", d.Id()) - res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "DELETE", - Project: billingProject, - RawURL: url, - UserAgent: userAgent, - Body: obj, - Timeout: d.Timeout(schema.TimeoutDelete), - Headers: headers, - }) - if err != nil { - return transport_tpg.HandleNotFoundError(err, d, "StorageBatchOperations") - } - - log.Printf("[DEBUG] Finished deleting StorageBatchOperations %q: %#v", d.Id(), res) - return nil -} - -func resourceStorageBatchOperationsStorageBatchOperationsImport(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) { - config := meta.(*transport_tpg.Config) - if err := tpgresource.ParseImportId([]string{ - "^project/(?P[^/]+)/location/(?P[^/]+)/jobs/(?P[^/]+)$", - "^(?P[^/]+)/(?P[^/]+)/(?P[^/]+)$", - "^(?P[^/]+)/(?P[^/]+)$", - }, d, config); err != nil { - return nil, err - } - - // Replace import id for the resource id - id, err := tpgresource.ReplaceVars(d, config, "{{job_id}}") - if err != nil { - return nil, fmt.Errorf("Error constructing id: %s", err) - } - d.SetId(id) - - return []*schema.ResourceData{d}, nil -} - -func flattenStorageBatchOperationsStorageBatchOperationsCreateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { - return v -} - -func flattenStorageBatchOperationsStorageBatchOperationsUpdateTime(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { - return v -} - -func flattenStorageBatchOperationsStorageBatchOperationsLocation(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { - return v -} - -func expandStorageBatchOperationsStorageBatchOperationsLocation(v interface{}, d tpgresource.TerraformResourceData, config *transport_tpg.Config) (interface{}, error) { - return v, nil -} diff --git a/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml b/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml deleted file mode 100644 index 814ff1ed5a3a..000000000000 --- a/mmv1/third_party/terraform/services/storagebatchoperations/resource_storage_batch_operations_storage_batch_operations_generated_meta.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resource: 'google_storage_batch_operations_storage_batch_operations' -generation_type: 'mmv1' -api_service_name: 'storagebatchoperations.googleapis.com' -api_version: 'v1' -api_resource_type_kind: 'StorageBatchOperations' From 60b783a893c457f87c30e3682caeac085189f981 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 19 Nov 2024 06:34:33 +0000 Subject: [PATCH 04/13] added in kt files --- .../terraform/.teamcity/components/inputs/services_beta.kt | 5 +++++ .../terraform/.teamcity/components/inputs/services_ga.kt | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt b/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt index b025568eac6d..5d70cb937da6 100644 --- a/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt +++ b/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt @@ -726,6 +726,11 @@ var ServicesListBeta = mapOf( "displayName" to "Storagetransfer", "path" to "./google-beta/services/storagetransfer" ), + "storagebatchoperations" to mapOf( + "name" to "storagebatchoperations", + "displayName" to "Storagebatchoperations", + "path" to "./google/services/storagebatchoperations" + ), "tags" to mapOf( "name" to "tags", "displayName" to "Tags", diff --git a/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt b/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt index 5087bad11881..f07209c5f901 100644 --- a/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt +++ b/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt @@ -721,6 +721,11 @@ var ServicesListGa = mapOf( "displayName" to "Storagetransfer", "path" to "./google/services/storagetransfer" ), + "storagebatchoperations" to mapOf( + "name" to "storagebatchoperations", + "displayName" to "Storagebatchoperations", + "path" to "./google/services/storagebatchoperations" + ), "tags" to mapOf( "name" to "tags", "displayName" to "Tags", From 36f7a5744a800e9d2a356695a25efc07200e7ce8 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 19 Nov 2024 09:35:10 +0000 Subject: [PATCH 05/13] sbo --- mmv1/products/storagebatchoperations/sbo.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml index 81fd9cbe5eca..ad28476df075 100644 --- a/mmv1/products/storagebatchoperations/sbo.yaml +++ b/mmv1/products/storagebatchoperations/sbo.yaml @@ -34,12 +34,18 @@ parameters: description: 'The location of the job.' immutable: true imports: 'location' - - name: 'prefixListFile' + - name: 'prefix_list_file' type: String - description: one of prefixlist or manifestLocation allowed - - name: 'manifestLocation' + description: one of prefix_list_file or manifest_location allowed + exactly_one_of: + - 'manifest_location' + - 'prefix_list_file' + - name: 'manifest_location' type: String - description: one of prefixlistfile or manifestlocation allowed + description: one of prefix_list_file or manifest_location allowed + exactly_one_of: + - 'manifest_location' + - 'prefix_list_file' properties: - name: 'createTime' From bac7e8276878735412e8963c888b6a7ffe4fc094 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Mon, 9 Dec 2024 06:14:55 +0000 Subject: [PATCH 06/13] update --- mmv1/products/storagebatchoperations/sbo.yaml | 212 +++++++++++++++--- 1 file changed, 186 insertions(+), 26 deletions(-) diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml index ad28476df075..c49d5a1d529f 100644 --- a/mmv1/products/storagebatchoperations/sbo.yaml +++ b/mmv1/products/storagebatchoperations/sbo.yaml @@ -12,13 +12,15 @@ # limitations under the License. --- -name: 'StorageBatchOperations' +name: 'Jobs' kind: 'storagebatchoperations#jobs' id_format: '{{job_id}}' -base_url: 'project/{{project}}/location/{{location}}/jobs' -self_link: 'project/{{project}}/location/{{location}}/jobs' +base_url: 'projects/{{project}}/locations/{{location}}/jobs' +self_link: 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' +create_url: 'projects/{{project}}/locations/{{location}}/jobs?jobId={{job_id}}' has_self_link: true -delete_url: 'project/{{project}}/location/{{location}}/jobs' +import_format: + - 'projects/{{project}}/locations/{{location}}/jobs/{{jobId}}' timeouts: insert_minutes: 20 update_minutes: 20 @@ -28,33 +30,191 @@ description: | Storage Batch Operations (SBO) is a Cloud Storage management feature that offers a seamless experience to perform single batch operations on millions of GCS objects in a serverless manner. +examples: + - name: 'storage_batch_operations' parameters: - - name: 'location' - type: String - description: 'The location of the job.' - immutable: true - imports: 'location' - - name: 'prefix_list_file' - type: String - description: one of prefix_list_file or manifest_location allowed - exactly_one_of: - - 'manifest_location' - - 'prefix_list_file' - - name: 'manifest_location' - type: String - description: one of prefix_list_file or manifest_location allowed - exactly_one_of: - - 'manifest_location' - - 'prefix_list_file' - -properties: - name: 'createTime' type: String description: | - The timestamp at which this managed folder was created. + The timestamp at which this storage batch operation was created. output: true - name: 'updateTime' type: String description: | - The timestamp at which this managed folder was most recently updated. - output: true + The timestamp at which this storage batch operation was most recently updated. + output: true +properties: + - name: 'location' + type: String + description: 'The location of the job.' + immutable: true + required: true + url_param_only: true + - name: 'jobId' + type: String + description: 'The ID of the job.' + url_param_only: true + - name: 'prefixList' + type: NestedObject + description: All objects in the bucket that match the prefix will be in scope for the selected batch operation + immutable: true + exactly_one_of: + - 'prefix_list' + - 'manifest' + properties: + - name: 'bucketName' + type: string + required: true + immutable: true + description: 'bucket on which operations performed' + - name: 'objectPrefix' + type: string + required: true + immutable: true + description: 'object prefix allows to specify a prefix as the scope for batch operations' + - name: 'manifest' + type: NestedObject + immutable: true + description: one of prefix_list or manifest allowed + exactly_one_of: + - 'prefix_list' + - 'manifest' + properties: + - name: 'manifestLocation' + type: String + description: 'list of objects to perform operations in csv format on storage bucket' + immutable: true + - name: 'deleteObject' + type: NestedObject + description: 'allows batch operations to delete objects in bucket' + immutable: true + properties: + - name: 'permanentObjectDeletionEnabled' + type: Boolean + description: 'enable flag to permanently delete object and all object versions if versioning is enabled on bucket' + immutable: true + required: true + exactly_one_of: + - 'delete_object' + - 'put_metadata' + - 'put_kms_key' + - 'put_object_hold' + - name: 'putMetadata' + type: NestedObject + description: 'allows batch operations to update metadata for objects in bucket' + immutable: true + exactly_one_of: + - 'delete_object' + - 'put_metadata' + - 'put_kms_key' + - 'put_object_hold' + properties: + - name: 'contentDisposition' + type: String + description: 'Content-Disposition of the object data.' + immutable: true + at_least_one_of: + - 'put_metadata.0.content_disposition' + - 'put_metadata.0.content_encoding' + - 'put_metadata.0.content_language' + - 'put_metadata.0.content_type' + - 'put_metadata.0.cache_control' + - 'put_metadata.0.custom_metadata' + - name: 'contentEncoding' + type: String + description: 'Content Encoding of the object data.' + immutable: true + at_least_one_of: + - 'put_metadata.0.content_disposition' + - 'put_metadata.0.content_encoding' + - 'put_metadata.0.content_language' + - 'put_metadata.0.content_type' + - 'put_metadata.0.cache_control' + - 'put_metadata.0.custom_metadata' + - name: 'contentType' + type: String + description: 'Content-Type of the object data.' + immutable: true + at_least_one_of: + - 'put_metadata.0.content_disposition' + - 'put_metadata.0.content_encoding' + - 'put_metadata.0.content_language' + - 'put_metadata.0.content_type' + - 'put_metadata.0.cache_control' + - 'put_metadata.0.custom_metadata' + - name: 'contentLanguage' + type: String + description: 'Content-Language of the object data.' + immutable: true + at_least_one_of: + - 'put_metadata.0.content_disposition' + - 'put_metadata.0.content_encoding' + - 'put_metadata.0.content_language' + - 'put_metadata.0.content_type' + - 'put_metadata.0.cache_control' + - 'put_metadata.0.custom_metadata' + - name: 'cacheControl' + type: String + description: 'Cache-Control directive to specify caching behavior of object data. If omitted and object is accessible to all anonymous users, the default will be public, max-age=3600' + immutable: true + at_least_one_of: + - 'put_metadata.0.content_disposition' + - 'put_metadata.0.content_encoding' + - 'put_metadata.0.content_language' + - 'put_metadata.0.content_type' + - 'put_metadata.0.cache_control' + - 'put_metadata.0.custom_metadata' + - name: 'customMetadata' + type: KeyValuePairs + description: 'User-provided metadata, in key/value pairs.' + immutable: true + at_least_one_of: + - 'put_metadata.0.content_disposition' + - 'put_metadata.0.content_encoding' + - 'put_metadata.0.content_language' + - 'put_metadata.0.content_type' + - 'put_metadata.0.cache_control' + - 'put_metadata.0.custom_metadata' + - name: 'putKmsKey' + type: NestedObject + description: 'allows to update encryption key for objects in bucket' + immutable: true + exactly_one_of: + - 'delete_object' + - 'put_metadata' + - 'put_kms_key' + - 'put_object_hold' + properties: + - name: 'kmsKey' + type: String + description: 'valid kms key' + immutable: true + required: true + - name: 'putObjectHold' + type: NestedObject + description: 'allows to update temporary hold or eventBased hold for objects in bucket' + immutable: true + exactly_one_of: + - 'delete_object' + - 'put_metadata' + - 'put_kms_key' + - 'put_object_hold' + properties: + - name: 'eventBasedHold' + type: TypeString + immutable: true + description: 'set/unset to update event based hold for objects' + validation: + function: 'validation.StringInSlice([]string{"SET", "UNSET"}, false)' + at_least_one_of: + - 'putObjectHold.0.eventBasedHold' + - 'putObjectHold.0.temporaryHold' + - name: 'temporaryHold' + type: TypeString + immutable: true + description: 'set/unset to update temporary based hold for objects' + validation: + function: 'validation.StringInSlice([]string{"SET", "UNSET"}, false)' + at_least_one_of: + - 'putObjectHold.0.eventBasedHold' + - 'putObjectHold.0.temporaryHold' From 040125a0113e1a4f7b88efe840afd352b5fa67dd Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Thu, 26 Dec 2024 06:54:19 +0000 Subject: [PATCH 07/13] updates --- mmv1/products/storagebatchoperations/sbo.yaml | 41 ++++++++++++++----- .../storage_batch_operations_jobs.go.tmpl | 19 +++++++++ .../storage_batch_operations.go.tmpl | 10 +++++ .../storage_batch_operations.go.tmpl | 1 + 4 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl create mode 100644 mmv1/templates/terraform/custom_update/storage_batch_operations.go.tmpl create mode 100644 mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml index c49d5a1d529f..d8dbd027deee 100644 --- a/mmv1/products/storagebatchoperations/sbo.yaml +++ b/mmv1/products/storagebatchoperations/sbo.yaml @@ -14,24 +14,44 @@ --- name: 'Jobs' kind: 'storagebatchoperations#jobs' -id_format: '{{job_id}}' +id_format: 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' base_url: 'projects/{{project}}/locations/{{location}}/jobs' self_link: 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' create_url: 'projects/{{project}}/locations/{{location}}/jobs?jobId={{job_id}}' -has_self_link: true import_format: - 'projects/{{project}}/locations/{{location}}/jobs/{{jobId}}' timeouts: - insert_minutes: 20 - update_minutes: 20 + insert_minutes: 120 + update_minutes: 120 delete_minutes: 20 exclude_sweeper: true +exclude_delete: true +autogen_async: true +async: + actions: ['create', 'update'] + type: 'OpAsync' + operation: + base_url: '{{op_id}}' + result: + resource_inside_response: true description: | Storage Batch Operations (SBO) is a Cloud Storage management feature that offers a seamless experience to perform single batch operations on millions of GCS objects in a serverless manner. examples: - name: 'storage_batch_operations' +custom_code: + constants: 'templates/terraform/constants/storage_batch_operations_jobs.go.tmpl' + custom_update: 'templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl' + custom_delete: 'templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl' +custom_diff: + - 'verifyJobIdChange' +virtual_fields: + - name: 'delete_protection' + description: | + If set to `true`, the storage batch operation job will not be deleted and new job will be created. + type: Boolean + default_value: true parameters: - name: 'createTime' type: String @@ -42,7 +62,12 @@ parameters: type: String description: | The timestamp at which this storage batch operation was most recently updated. - output: true + output: true + - name: 'jobId' + type: String + description: 'The ID of the job.' + url_param_only: true + immutable: true properties: - name: 'location' type: String @@ -50,10 +75,6 @@ properties: immutable: true required: true url_param_only: true - - name: 'jobId' - type: String - description: 'The ID of the job.' - url_param_only: true - name: 'prefixList' type: NestedObject description: All objects in the bucket that match the prefix will be in scope for the selected batch operation @@ -69,7 +90,6 @@ properties: description: 'bucket on which operations performed' - name: 'objectPrefix' type: string - required: true immutable: true description: 'object prefix allows to specify a prefix as the scope for batch operations' - name: 'manifest' @@ -84,6 +104,7 @@ properties: type: String description: 'list of objects to perform operations in csv format on storage bucket' immutable: true + required: true - name: 'deleteObject' type: NestedObject description: 'allows batch operations to delete objects in bucket' diff --git a/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl b/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl new file mode 100644 index 000000000000..f53be743ed2e --- /dev/null +++ b/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl @@ -0,0 +1,19 @@ +func hasChange(d *schema.ResourceDiff) error { + for _, key := range []string{ + "delete_object", + "put_object_hold", + "put_kms_key", + "manifest", + "prefix_list", + "put_metadata", + } { + if d.HasChange(key) && !d.HasChange("job_id") { + return fmt.Errorf("Error, any change in parameters, should update job_id") + } + } + return nil +} + +func verifyJobIdChange(_ context.Context, diff *schema.ResourceDiff, meta interface{}) error { + return hasChange(diff) +} \ No newline at end of file diff --git a/mmv1/templates/terraform/custom_update/storage_batch_operations.go.tmpl b/mmv1/templates/terraform/custom_update/storage_batch_operations.go.tmpl new file mode 100644 index 000000000000..4dca4cfe9d6d --- /dev/null +++ b/mmv1/templates/terraform/custom_update/storage_batch_operations.go.tmpl @@ -0,0 +1,10 @@ +_ = config +// we can only get here if delete_protection was updated +if d.Get("delete_protection") != nil { + if err := d.Set("delete_protection", d.Get("delete_protection")); err != nil { + return fmt.Errorf("Error updating delete_protection: %s", err) + } +} + +// all other fields are immutable, don't do anything else +return nil diff --git a/mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl b/mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl new file mode 100644 index 000000000000..7d4bc3192d8d --- /dev/null +++ b/mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl @@ -0,0 +1 @@ +log.Printf("pre create") \ No newline at end of file From 2a7ca15afc34ff40b03ce2c2971d7acecda2b0e9 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Thu, 26 Dec 2024 06:55:23 +0000 Subject: [PATCH 08/13] updates --- .../storage_batch_operations_jobs.go.tmpl | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl diff --git a/mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl b/mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl new file mode 100644 index 000000000000..c721b50e03ab --- /dev/null +++ b/mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl @@ -0,0 +1,42 @@ +project, err := tpgresource.GetProject(d, config) +if err != nil { + return fmt.Errorf("Error fetching project for Jobs: %s", err) +} +billingProject = project + +if d.Get("delete_protection").(bool) { + log.Printf("[DEBUG] delete_protection set to true, existing resource on cloud will not be deleted, new resource will be created") + return nil +} + +url, err := tpgresource.ReplaceVars(d, config, "{{"{{"}}StorageBatchOperationsBasePath{{"}}"}}projects/{{"{{"}}project{{"}}"}}/locations/{{"{{"}}location{{"}}"}}/jobs/{{"{{"}}job_id{{"}}"}}") +if err != nil { + return err +} + +var obj map[string]interface{} + +// err == nil indicates that the billing_project value was found +if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp +} + +headers := make(http.Header) + +log.Printf("[DEBUG] Deleting Jobs %q", d.Id()) +res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "DELETE", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, + Body: obj, + Timeout: d.Timeout(schema.TimeoutDelete), + Headers: headers, +}) +if err != nil { + return transport_tpg.HandleNotFoundError(err, d, "Jobs") +} + +log.Printf("[DEBUG] Finished deleting Jobs %q: %#v", d.Id(), res) +return nil \ No newline at end of file From aa89b08035239342f17c122580d1060627914936 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Thu, 26 Dec 2024 06:56:54 +0000 Subject: [PATCH 09/13] remove pre create --- .../terraform/pre_create/storage_batch_operations.go.tmpl | 1 - 1 file changed, 1 deletion(-) delete mode 100644 mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl diff --git a/mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl b/mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl deleted file mode 100644 index 7d4bc3192d8d..000000000000 --- a/mmv1/templates/terraform/pre_create/storage_batch_operations.go.tmpl +++ /dev/null @@ -1 +0,0 @@ -log.Printf("pre create") \ No newline at end of file From 82bf6b1d0bc0188331e5fbc31ff097c5087a996e Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 31 Dec 2024 06:52:08 +0000 Subject: [PATCH 10/13] update --- mmv1/products/storagebatchoperations/sbo.yaml | 34 +++++++++++++------ .../storage_batch_operations_jobs.go.tmpl | 3 +- ... => storage_batch_operations_jobs.go.tmpl} | 0 3 files changed, 25 insertions(+), 12 deletions(-) rename mmv1/templates/terraform/custom_update/{storage_batch_operations.go.tmpl => storage_batch_operations_jobs.go.tmpl} (100%) diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml index d8dbd027deee..d81dced3e1d2 100644 --- a/mmv1/products/storagebatchoperations/sbo.yaml +++ b/mmv1/products/storagebatchoperations/sbo.yaml @@ -19,7 +19,7 @@ base_url: 'projects/{{project}}/locations/{{location}}/jobs' self_link: 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' create_url: 'projects/{{project}}/locations/{{location}}/jobs?jobId={{job_id}}' import_format: - - 'projects/{{project}}/locations/{{location}}/jobs/{{jobId}}' + - 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' timeouts: insert_minutes: 120 update_minutes: 120 @@ -67,7 +67,9 @@ parameters: type: String description: 'The ID of the job.' url_param_only: true - immutable: true + immutable: true + validation: + regex: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?.$' properties: - name: 'location' type: String @@ -83,15 +85,25 @@ properties: - 'prefix_list' - 'manifest' properties: - - name: 'bucketName' - type: string - required: true - immutable: true - description: 'bucket on which operations performed' - - name: 'objectPrefix' - type: string - immutable: true - description: 'object prefix allows to specify a prefix as the scope for batch operations' + - name: 'prefixes' + type: Array + min_size: 1 + max_size: 1 + description: contains object_prefix and bucket + immutable: true + item_type: + type: NestedObject + properties: + - name: 'bucket' + type: string + required: true + immutable: true + description: 'bucket on which operations performed' + - name: 'objectPrefix' + type: string + immutable: true + required: false + description: 'object prefix allows to specify a prefix as the scope for batch operations' - name: 'manifest' type: NestedObject immutable: true diff --git a/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl b/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl index f53be743ed2e..2841d96672b5 100644 --- a/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl +++ b/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl @@ -6,9 +6,10 @@ func hasChange(d *schema.ResourceDiff) error { "manifest", "prefix_list", "put_metadata", + "location", } { if d.HasChange(key) && !d.HasChange("job_id") { - return fmt.Errorf("Error, any change in parameters, should update job_id") + return fmt.Errorf("Error, job_id should be updated, if any change in parameters) } } return nil diff --git a/mmv1/templates/terraform/custom_update/storage_batch_operations.go.tmpl b/mmv1/templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl similarity index 100% rename from mmv1/templates/terraform/custom_update/storage_batch_operations.go.tmpl rename to mmv1/templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl From bc3262a9a706ce02c6e78a7a025603d213583ccb Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 31 Dec 2024 18:22:47 +0530 Subject: [PATCH 11/13] Update product.yaml --- mmv1/products/storagebatchoperations/product.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/storagebatchoperations/product.yaml b/mmv1/products/storagebatchoperations/product.yaml index 63afb5d70b56..78d2a664a542 100644 --- a/mmv1/products/storagebatchoperations/product.yaml +++ b/mmv1/products/storagebatchoperations/product.yaml @@ -15,7 +15,7 @@ name: 'StorageBatchOperations' display_name: 'Cloud Storage Batch Operations' versions: - - name: 'ga' + - name: 'alpha' base_url: 'https://storagebatchoperations.googleapis.com/v1/' scopes: - 'https://www.googleapis.com/auth/cloud-platform' From 9225c75a48ee6ebe29b1704ebf030e4c5432c8a0 Mon Sep 17 00:00:00 2001 From: Guru Sai Rama Subbarao Voleti Date: Tue, 31 Dec 2024 13:12:23 +0000 Subject: [PATCH 12/13] update --- mmv1/products/storagebatchoperations/sbo.yaml | 253 ------------------ .../storage_batch_operations_jobs.go.tmpl | 20 -- .../storage_batch_operations_jobs.go.tmpl | 42 --- .../storage_batch_operations_jobs.go.tmpl | 10 - .../components/inputs/services_beta.kt | 5 - .../components/inputs/services_ga.kt | 5 - 6 files changed, 335 deletions(-) delete mode 100644 mmv1/products/storagebatchoperations/sbo.yaml delete mode 100644 mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl delete mode 100644 mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl delete mode 100644 mmv1/templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl diff --git a/mmv1/products/storagebatchoperations/sbo.yaml b/mmv1/products/storagebatchoperations/sbo.yaml deleted file mode 100644 index d81dced3e1d2..000000000000 --- a/mmv1/products/storagebatchoperations/sbo.yaml +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2024 Google Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -name: 'Jobs' -kind: 'storagebatchoperations#jobs' -id_format: 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' -base_url: 'projects/{{project}}/locations/{{location}}/jobs' -self_link: 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' -create_url: 'projects/{{project}}/locations/{{location}}/jobs?jobId={{job_id}}' -import_format: - - 'projects/{{project}}/locations/{{location}}/jobs/{{job_id}}' -timeouts: - insert_minutes: 120 - update_minutes: 120 - delete_minutes: 20 -exclude_sweeper: true -exclude_delete: true -autogen_async: true -async: - actions: ['create', 'update'] - type: 'OpAsync' - operation: - base_url: '{{op_id}}' - result: - resource_inside_response: true -description: | - Storage Batch Operations (SBO) is a Cloud Storage management feature that offers a - seamless experience to perform single batch operations on millions of GCS objects in a - serverless manner. -examples: - - name: 'storage_batch_operations' -custom_code: - constants: 'templates/terraform/constants/storage_batch_operations_jobs.go.tmpl' - custom_update: 'templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl' - custom_delete: 'templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl' -custom_diff: - - 'verifyJobIdChange' -virtual_fields: - - name: 'delete_protection' - description: | - If set to `true`, the storage batch operation job will not be deleted and new job will be created. - type: Boolean - default_value: true -parameters: - - name: 'createTime' - type: String - description: | - The timestamp at which this storage batch operation was created. - output: true - - name: 'updateTime' - type: String - description: | - The timestamp at which this storage batch operation was most recently updated. - output: true - - name: 'jobId' - type: String - description: 'The ID of the job.' - url_param_only: true - immutable: true - validation: - regex: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?.$' -properties: - - name: 'location' - type: String - description: 'The location of the job.' - immutable: true - required: true - url_param_only: true - - name: 'prefixList' - type: NestedObject - description: All objects in the bucket that match the prefix will be in scope for the selected batch operation - immutable: true - exactly_one_of: - - 'prefix_list' - - 'manifest' - properties: - - name: 'prefixes' - type: Array - min_size: 1 - max_size: 1 - description: contains object_prefix and bucket - immutable: true - item_type: - type: NestedObject - properties: - - name: 'bucket' - type: string - required: true - immutable: true - description: 'bucket on which operations performed' - - name: 'objectPrefix' - type: string - immutable: true - required: false - description: 'object prefix allows to specify a prefix as the scope for batch operations' - - name: 'manifest' - type: NestedObject - immutable: true - description: one of prefix_list or manifest allowed - exactly_one_of: - - 'prefix_list' - - 'manifest' - properties: - - name: 'manifestLocation' - type: String - description: 'list of objects to perform operations in csv format on storage bucket' - immutable: true - required: true - - name: 'deleteObject' - type: NestedObject - description: 'allows batch operations to delete objects in bucket' - immutable: true - properties: - - name: 'permanentObjectDeletionEnabled' - type: Boolean - description: 'enable flag to permanently delete object and all object versions if versioning is enabled on bucket' - immutable: true - required: true - exactly_one_of: - - 'delete_object' - - 'put_metadata' - - 'put_kms_key' - - 'put_object_hold' - - name: 'putMetadata' - type: NestedObject - description: 'allows batch operations to update metadata for objects in bucket' - immutable: true - exactly_one_of: - - 'delete_object' - - 'put_metadata' - - 'put_kms_key' - - 'put_object_hold' - properties: - - name: 'contentDisposition' - type: String - description: 'Content-Disposition of the object data.' - immutable: true - at_least_one_of: - - 'put_metadata.0.content_disposition' - - 'put_metadata.0.content_encoding' - - 'put_metadata.0.content_language' - - 'put_metadata.0.content_type' - - 'put_metadata.0.cache_control' - - 'put_metadata.0.custom_metadata' - - name: 'contentEncoding' - type: String - description: 'Content Encoding of the object data.' - immutable: true - at_least_one_of: - - 'put_metadata.0.content_disposition' - - 'put_metadata.0.content_encoding' - - 'put_metadata.0.content_language' - - 'put_metadata.0.content_type' - - 'put_metadata.0.cache_control' - - 'put_metadata.0.custom_metadata' - - name: 'contentType' - type: String - description: 'Content-Type of the object data.' - immutable: true - at_least_one_of: - - 'put_metadata.0.content_disposition' - - 'put_metadata.0.content_encoding' - - 'put_metadata.0.content_language' - - 'put_metadata.0.content_type' - - 'put_metadata.0.cache_control' - - 'put_metadata.0.custom_metadata' - - name: 'contentLanguage' - type: String - description: 'Content-Language of the object data.' - immutable: true - at_least_one_of: - - 'put_metadata.0.content_disposition' - - 'put_metadata.0.content_encoding' - - 'put_metadata.0.content_language' - - 'put_metadata.0.content_type' - - 'put_metadata.0.cache_control' - - 'put_metadata.0.custom_metadata' - - name: 'cacheControl' - type: String - description: 'Cache-Control directive to specify caching behavior of object data. If omitted and object is accessible to all anonymous users, the default will be public, max-age=3600' - immutable: true - at_least_one_of: - - 'put_metadata.0.content_disposition' - - 'put_metadata.0.content_encoding' - - 'put_metadata.0.content_language' - - 'put_metadata.0.content_type' - - 'put_metadata.0.cache_control' - - 'put_metadata.0.custom_metadata' - - name: 'customMetadata' - type: KeyValuePairs - description: 'User-provided metadata, in key/value pairs.' - immutable: true - at_least_one_of: - - 'put_metadata.0.content_disposition' - - 'put_metadata.0.content_encoding' - - 'put_metadata.0.content_language' - - 'put_metadata.0.content_type' - - 'put_metadata.0.cache_control' - - 'put_metadata.0.custom_metadata' - - name: 'putKmsKey' - type: NestedObject - description: 'allows to update encryption key for objects in bucket' - immutable: true - exactly_one_of: - - 'delete_object' - - 'put_metadata' - - 'put_kms_key' - - 'put_object_hold' - properties: - - name: 'kmsKey' - type: String - description: 'valid kms key' - immutable: true - required: true - - name: 'putObjectHold' - type: NestedObject - description: 'allows to update temporary hold or eventBased hold for objects in bucket' - immutable: true - exactly_one_of: - - 'delete_object' - - 'put_metadata' - - 'put_kms_key' - - 'put_object_hold' - properties: - - name: 'eventBasedHold' - type: TypeString - immutable: true - description: 'set/unset to update event based hold for objects' - validation: - function: 'validation.StringInSlice([]string{"SET", "UNSET"}, false)' - at_least_one_of: - - 'putObjectHold.0.eventBasedHold' - - 'putObjectHold.0.temporaryHold' - - name: 'temporaryHold' - type: TypeString - immutable: true - description: 'set/unset to update temporary based hold for objects' - validation: - function: 'validation.StringInSlice([]string{"SET", "UNSET"}, false)' - at_least_one_of: - - 'putObjectHold.0.eventBasedHold' - - 'putObjectHold.0.temporaryHold' diff --git a/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl b/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl deleted file mode 100644 index 2841d96672b5..000000000000 --- a/mmv1/templates/terraform/constants/storage_batch_operations_jobs.go.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -func hasChange(d *schema.ResourceDiff) error { - for _, key := range []string{ - "delete_object", - "put_object_hold", - "put_kms_key", - "manifest", - "prefix_list", - "put_metadata", - "location", - } { - if d.HasChange(key) && !d.HasChange("job_id") { - return fmt.Errorf("Error, job_id should be updated, if any change in parameters) - } - } - return nil -} - -func verifyJobIdChange(_ context.Context, diff *schema.ResourceDiff, meta interface{}) error { - return hasChange(diff) -} \ No newline at end of file diff --git a/mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl b/mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl deleted file mode 100644 index c721b50e03ab..000000000000 --- a/mmv1/templates/terraform/custom_delete/storage_batch_operations_jobs.go.tmpl +++ /dev/null @@ -1,42 +0,0 @@ -project, err := tpgresource.GetProject(d, config) -if err != nil { - return fmt.Errorf("Error fetching project for Jobs: %s", err) -} -billingProject = project - -if d.Get("delete_protection").(bool) { - log.Printf("[DEBUG] delete_protection set to true, existing resource on cloud will not be deleted, new resource will be created") - return nil -} - -url, err := tpgresource.ReplaceVars(d, config, "{{"{{"}}StorageBatchOperationsBasePath{{"}}"}}projects/{{"{{"}}project{{"}}"}}/locations/{{"{{"}}location{{"}}"}}/jobs/{{"{{"}}job_id{{"}}"}}") -if err != nil { - return err -} - -var obj map[string]interface{} - -// err == nil indicates that the billing_project value was found -if bp, err := tpgresource.GetBillingProject(d, config); err == nil { - billingProject = bp -} - -headers := make(http.Header) - -log.Printf("[DEBUG] Deleting Jobs %q", d.Id()) -res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "DELETE", - Project: billingProject, - RawURL: url, - UserAgent: userAgent, - Body: obj, - Timeout: d.Timeout(schema.TimeoutDelete), - Headers: headers, -}) -if err != nil { - return transport_tpg.HandleNotFoundError(err, d, "Jobs") -} - -log.Printf("[DEBUG] Finished deleting Jobs %q: %#v", d.Id(), res) -return nil \ No newline at end of file diff --git a/mmv1/templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl b/mmv1/templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl deleted file mode 100644 index 4dca4cfe9d6d..000000000000 --- a/mmv1/templates/terraform/custom_update/storage_batch_operations_jobs.go.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -_ = config -// we can only get here if delete_protection was updated -if d.Get("delete_protection") != nil { - if err := d.Set("delete_protection", d.Get("delete_protection")); err != nil { - return fmt.Errorf("Error updating delete_protection: %s", err) - } -} - -// all other fields are immutable, don't do anything else -return nil diff --git a/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt b/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt index 8e2f834590d2..24abf96b5b4f 100644 --- a/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt +++ b/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt @@ -736,11 +736,6 @@ var ServicesListBeta = mapOf( "displayName" to "Storagetransfer", "path" to "./google-beta/services/storagetransfer" ), - "storagebatchoperations" to mapOf( - "name" to "storagebatchoperations", - "displayName" to "Storagebatchoperations", - "path" to "./google/services/storagebatchoperations" - ), "tags" to mapOf( "name" to "tags", "displayName" to "Tags", diff --git a/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt b/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt index 150e98d7da75..a5bb12bca8f1 100644 --- a/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt +++ b/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt @@ -731,11 +731,6 @@ var ServicesListGa = mapOf( "displayName" to "Storagetransfer", "path" to "./google/services/storagetransfer" ), - "storagebatchoperations" to mapOf( - "name" to "storagebatchoperations", - "displayName" to "Storagebatchoperations", - "path" to "./google/services/storagebatchoperations" - ), "tags" to mapOf( "name" to "tags", "displayName" to "Tags", From 67d788ed50049b4ead7896d51049bccd643ae21d Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 9 Jan 2025 20:42:02 +0530 Subject: [PATCH 13/13] Update product.yaml --- mmv1/products/storagebatchoperations/product.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/storagebatchoperations/product.yaml b/mmv1/products/storagebatchoperations/product.yaml index 78d2a664a542..63afb5d70b56 100644 --- a/mmv1/products/storagebatchoperations/product.yaml +++ b/mmv1/products/storagebatchoperations/product.yaml @@ -15,7 +15,7 @@ name: 'StorageBatchOperations' display_name: 'Cloud Storage Batch Operations' versions: - - name: 'alpha' + - name: 'ga' base_url: 'https://storagebatchoperations.googleapis.com/v1/' scopes: - 'https://www.googleapis.com/auth/cloud-platform'