Deployment search: Add flag to return all matches.

cmd/deployment/search.go

@@ -18,11 +18,12 @@
 package cmddeployment
 
 import (
+	"fmt"
 	"github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi"
+	"github.com/elastic/cloud-sdk-go/pkg/models"
 	"github.com/elastic/cloud-sdk-go/pkg/util/cmdutil"
-	"github.com/spf13/cobra"
-
 	"github.com/elastic/ecctl/pkg/ecctl"
+	"github.com/spf13/cobra"
 )
 
 const searchQueryLong = `Read more about Query DSL in https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html`
@@ -51,21 +52,57 @@ var searchCmd = &cobra.Command{
 			return err
 		}
 
-		res, err := deploymentapi.Search(deploymentapi.SearchParams{
-			API:     ecctl.Get().API,
-			Request: &sr,
-		})
+		returnAllMatches, _ := cmd.Flags().GetBool("all-matches")
+		if returnAllMatches && sr.Sort == nil {
+			return fmt.Errorf("The query must include a sort-field when using --all-matches. Example: \"sort\": [\"id\"]")
+		}
 
-		if err != nil {
-			return err
+		batchSize, _ := cmd.Flags().GetInt32("size")
+
+		var result *models.DeploymentsSearchResponse
+		var cursor string
+		for i := 0; i < 100; i++ {
+			sr.Cursor = cursor
+			if returnAllMatches {
+				// Custom batch-size to override any size already set in the input query
+				sr.Size = batchSize
+			}
+
+			res, err := deploymentapi.Search(deploymentapi.SearchParams{
+				API:     ecctl.Get().API,
+				Request: &sr,
+			})
+
+			if err != nil {
+				return err
+			}
+
+			cursor = res.Cursor
+
+			if result == nil {
+				result = res
+				result.Cursor = "" // Hide cursor in output
+			} else {
+				result.Deployments = append(result.Deployments, res.Deployments...)
+				newReturnCount := *result.ReturnCount + *res.ReturnCount
+				result.ReturnCount = &newReturnCount
+				result.MatchCount = newReturnCount
+			}
+
+			if *res.ReturnCount == 0 || !returnAllMatches {
+				break
+			}
 		}
 
-		return ecctl.Get().Formatter.Format("deployment/search", res)
+		return ecctl.Get().Formatter.Format("deployment/search", result)
 	},
 }
 
 func init() {
 	Command.AddCommand(searchCmd)
 	searchCmd.Flags().StringP("file", "f", "", "JSON file that contains JSON-style domain-specific language query")
 	searchCmd.MarkFlagRequired("file")
+	searchCmd.Flags().BoolP("all-matches", "a", false,
+		"Uses a cursor to return all matches of the query (ignoring the size in the query). This can be used to query more than 10k results.")
+	searchCmd.Flags().Int32("size", 500, "Defines the size per request when using the --all-matches option.")
 }

cmd/deployment/search_test.go

@@ -0,0 +1,181 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmddeployment
+
+import (
+	"github.com/elastic/cloud-sdk-go/pkg/api"
+	"github.com/elastic/cloud-sdk-go/pkg/api/mock"
+	"github.com/elastic/cloud-sdk-go/pkg/models"
+	"github.com/elastic/cloud-sdk-go/pkg/util/ec"
+	"github.com/elastic/ecctl/cmd/util/testutils"
+	"testing"
+)
+
+func Test_searchCmd(t *testing.T) {
+	tests := []struct {
+		name string
+		args testutils.Args
+		want testutils.Assertion
+	}{
+		{
+			name: "all-matches collects deployments using multiple requests",
+			args: testutils.Args{
+				Cmd: searchCmd,
+				Args: []string{
+					"search",
+					"-f",
+					"testdata/search_query.json",
+					"--all-matches",
+					"--size",
+					"250",
+				},
+				Cfg: testutils.MockCfg{
+					OutputFormat: "json",
+					Responses: []mock.Response{
+						mock.New200ResponseAssertion(
+							&mock.RequestAssertion{
+								Header: api.DefaultWriteMockHeaders,
+								Method: "POST",
+								Path:   "/api/v1/deployments/_search",
+								Host:   api.DefaultMockHost,
+								Body: mock.NewStructBody(models.SearchRequest{
+									Query: &models.QueryContainer{
+										MatchAll: struct{}{},
+									},
+									Size: 250,
+									Sort: []interface{}{"id"},
+								}),
+							},
+							mock.NewStructBody(models.DeploymentsSearchResponse{
+								Cursor: "cursor1",
+								Deployments: []*models.DeploymentSearchResponse{
+									{ID: ec.String("d1")},
+									{ID: ec.String("d2")},
+								},
+								MatchCount:      3,
+								MinimalMetadata: nil,
+								ReturnCount:     ec.Int32(2),
+							}),
+						),
+						mock.New200ResponseAssertion(
+							&mock.RequestAssertion{
+								Header: api.DefaultWriteMockHeaders,
+								Method: "POST",
+								Path:   "/api/v1/deployments/_search",
+								Host:   api.DefaultMockHost,
+								Body: mock.NewStructBody(models.SearchRequest{
+									Cursor: "cursor1",
+									Query: &models.QueryContainer{
+										MatchAll: struct{}{},
+									},
+									Size: 250,
+									Sort: []interface{}{"id"},
+								}),
+							},
+							mock.NewStructBody(models.DeploymentsSearchResponse{
+								Cursor: "cursor2",
+								Deployments: []*models.DeploymentSearchResponse{
+									{ID: ec.String("d3")},
+								},
+								MatchCount:      3,
+								MinimalMetadata: nil,
+								ReturnCount:     ec.Int32(1),
+							}),
+						),
+						mock.New200ResponseAssertion(
+							&mock.RequestAssertion{
+								Header: api.DefaultWriteMockHeaders,
+								Method: "POST",
+								Path:   "/api/v1/deployments/_search",
+								Host:   api.DefaultMockHost,
+								Body: mock.NewStructBody(models.SearchRequest{
+									Cursor: "cursor2",
+									Query: &models.QueryContainer{
+										MatchAll: struct{}{},
+									},
+									Size: 250,
+									Sort: []interface{}{"id"},
+								}),
+							},
+							mock.NewStructBody(models.DeploymentsSearchResponse{
+								Cursor:          "cursor3",
+								Deployments:     []*models.DeploymentSearchResponse{},
+								MatchCount:      3,
+								MinimalMetadata: nil,
+								ReturnCount:     ec.Int32(0),
+							}),
+						),
+					},
+				},
+			},
+			want: testutils.Assertion{
+				Stdout: string(expectedOutput) + "\n",
+			},
+		},
+		{
+			name: "all-matches requires a query with a sort",
+			args: testutils.Args{
+				Cmd: searchCmd,
+				Args: []string{
+					"search",
+					"-f",
+					"testdata/search_query_no_sort.json",
+					"--all-matches",
+				},
+				Cfg: testutils.MockCfg{
+					OutputFormat: "json",
+					Responses:    []mock.Response{},
+				},
+			},
+			want: testutils.Assertion{
+				Err: "The query must include a sort-field when using --all-matches. Example: \"sort\": [\"id\"]",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testutils.RunCmdAssertion(t, tt.args, tt.want)
+		})
+	}
+}
+
+var expectedOutput = `{
+  "deployments": [
+    {
+      "healthy": null,
+      "id": "d1",
+      "name": null,
+      "resources": null
+    },
+    {
+      "healthy": null,
+      "id": "d2",
+      "name": null,
+      "resources": null
+    },
+    {
+      "healthy": null,
+      "id": "d3",
+      "name": null,
+      "resources": null
+    }
+  ],
+  "match_count": 3,
+  "minimal_metadata": null,
+  "return_count": 3
+}`

cmd/deployment/testdata/search_query.json

@@ -0,0 +1,6 @@
+{
+  "query": {
+    "match_all": {}
+  },
+  "sort": ["id"]
+}

cmd/deployment/testdata/search_query_no_sort.json

@@ -0,0 +1,5 @@
+{
+  "query": {
+    "match_all": {}
+  }
+}

docs/ecctl_deployment_search.adoc

@@ -30,8 +30,10 @@ ecctl deployment search -f <query file.json> [flags]
 === Options
 
 ----
+  -a, --all-matches   Uses a cursor to return all matches of the query (ignoring the size in the query). This can be used to query more than 10k results.
   -f, --file string   JSON file that contains JSON-style domain-specific language query
   -h, --help          help for search
+      --size int32    Defines the size per request when using the --all-matches option. (default 500)
 ----
 
 [float]

docs/ecctl_deployment_search.md

@@ -28,8 +28,10 @@ $ ecctl deployment search -f query_string_query.json
 ### Options
 
 ```
+  -a, --all-matches   Uses a cursor to return all matches of the query (ignoring the size in the query). This can be used to query more than 10k results.
   -f, --file string   JSON file that contains JSON-style domain-specific language query
   -h, --help          help for search
+      --size int32    Defines the size per request when using the --all-matches option. (default 500)
 ```
 
 ### Options inherited from parent commands

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
 	github.com/blang/semver/v4 v4.0.0
-	github.com/elastic/cloud-sdk-go v1.20.0
+	github.com/elastic/cloud-sdk-go v1.22.0
 	github.com/go-openapi/runtime v0.23.0
 	github.com/go-openapi/strfmt v0.21.2
 	github.com/pkg/errors v0.9.1

go.sum

@@ -23,8 +23,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/elastic/cloud-sdk-go v1.20.0 h1:OkGG0CRXSZbntNwoKATbqO8SQoaBZAfAcXavdi8sA5Y=
-github.com/elastic/cloud-sdk-go v1.20.0/go.mod h1:k0ZebhZKX22l6Ysl5Zbpc8VLF54hfwDtHppEEEVUJ04=
+github.com/elastic/cloud-sdk-go v1.22.0 h1:sPjvu7zZeDbgl6eufy41VH0TjWbaMgDS+Cy9qIvdFZ4=
+github.com/elastic/cloud-sdk-go v1.22.0/go.mod h1:k0ZebhZKX22l6Ysl5Zbpc8VLF54hfwDtHppEEEVUJ04=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=