From 505b5e70980623d988a0b7507c851b5832e03099 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 21 Dec 2023 16:05:30 +0100 Subject: [PATCH 1/5] [Refactor] Refactor common code between all benchmark commands There is quite a bit of duplicated code across all the benchmark commands. This is a first quick take on bringing it together. Initially I wanted to only refactor 1-2 functions but one lead to the next things. I am pushing this now to get early review from others, this is not complete / not fully tested. Also potentially now that the refactoring is bigger, likeyl the best place of the common files has become a different place. --- .../benchrunner/runners/common/scenario.go | 114 ++++++++++++++++++ internal/benchrunner/runners/rally/metrics.go | 5 +- internal/benchrunner/runners/rally/report.go | 9 +- internal/benchrunner/runners/rally/runner.go | 13 +- .../benchrunner/runners/rally/scenario.go | 71 ----------- internal/benchrunner/runners/stream/runner.go | 19 +-- .../benchrunner/runners/stream/scenario.go | 109 ----------------- .../benchrunner/runners/system/scenario.go | 21 +--- 8 files changed, 145 insertions(+), 216 deletions(-) create mode 100644 internal/benchrunner/runners/common/scenario.go diff --git a/internal/benchrunner/runners/common/scenario.go b/internal/benchrunner/runners/common/scenario.go new file mode 100644 index 000000000..7be4098e8 --- /dev/null +++ b/internal/benchrunner/runners/common/scenario.go @@ -0,0 +1,114 @@ +package common + +import ( + "errors" + "fmt" + "github.com/elastic/go-ucfg/yaml" + "os" + "path/filepath" + "strings" +) + +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +const DevPath = "_dev/benchmark/rally" + +type Scenario struct { + Package string `config:"package" json:"package"` + Description string `config:"description" json:"description"` + Version string `config:"version" json:"version"` + DataStream DataStream `config:"data_stream" json:"data_stream"` + Corpora Corpora `config:"corpora" json:"corpora"` +} + +type DataStream struct { + Name string `config:"name" json:"name"` +} + +type Corpora struct { + Generator *Generator `config:"generator" json:"generator"` +} + +type Generator struct { + TotalEvents uint64 `config:"total_events" json:"total_events"` + Template CorporaTemplate `config:"template" json:"template"` + Config CorporaAsset `config:"config" json:"config"` + Fields CorporaAsset `config:"fields" json:"fields"` +} + +type CorporaAsset struct { + Raw map[string]interface{} `config:"raw" json:"raw"` + Path string `config:"path" json:"path"` +} +type CorporaTemplate struct { + Raw string `config:"raw" json:"raw"` + Path string `config:"path" json:"path"` + Type string `config:"type" json:"type"` +} + +func DefaultConfig() *Scenario { + return &Scenario{} +} + +func ReadConfig(path, scenario, packageName, packageVersion string) (*Scenario, error) { + configPath := filepath.Join(path, DevPath, fmt.Sprintf("%s.yml", scenario)) + c := DefaultConfig() + cfg, err := yaml.NewConfigWithFile(configPath) + if err != nil { + return nil, fmt.Errorf("can't load benchmark configuration: %s: %w", configPath, err) + } + + if err == nil { + if err := cfg.Unpack(c); err != nil { + return nil, fmt.Errorf("can't unpack benchmark configuration: %s: %w", configPath, err) + } + } + + c.Package = packageName + c.Version = packageVersion + + if c.DataStream.Name == "" { + return nil, errors.New("can't read data stream name from benchmark configuration: empty") + } + + return c, nil +} + +func ReadScenarios(path, scenarioName, packageName, packageVersion string) (map[string]*Scenario, error) { + scenarios := make(map[string]*Scenario) + if len(scenarioName) > 0 { + scenario, err := ReadConfig(path, scenarioName, packageName, packageVersion) + if err != nil { + return nil, fmt.Errorf("error loading scenario: %w", err) + } + scenarios[scenarioName] = scenario + } else { + err := filepath.Walk(filepath.Join(path, DevPath), func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + if strings.HasSuffix(info.Name(), "-benchmark.yml") { + scenarioName = strings.TrimSuffix(info.Name(), ".yml") + scenario, err := ReadConfig(path, scenarioName, packageName, packageVersion) + if err != nil { + return err + } + scenarios[scenarioName] = scenario + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("error loading scenario: %w", err) + } + } + + return scenarios, nil +} diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 8cac07cae..01527fdd1 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -9,6 +9,7 @@ import ( _ "embed" "encoding/json" "fmt" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "io" "sync" "sync/atomic" @@ -23,7 +24,7 @@ import ( type collector struct { ctxt servicedeployer.ServiceContext metadata benchMeta - scenario scenario + scenario common.Scenario interval time.Duration esAPI *elasticsearch.API @@ -66,7 +67,7 @@ type metricsSummary struct { func newCollector( ctxt servicedeployer.ServiceContext, benchName string, - scenario scenario, + scenario common.Scenario, esAPI, metricsAPI *elasticsearch.API, interval time.Duration, datastream, pipelinePrefix string, diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index 3f6e46e23..2958fc24a 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -7,6 +7,7 @@ package rally import ( "encoding/json" "fmt" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "strings" "time" @@ -31,8 +32,8 @@ type report struct { } Parameters struct { PackageVersion string - DataStream dataStream - Corpora corpora + DataStream common.DataStream + Corpora common.Corpora } ClusterName string Nodes int @@ -43,7 +44,7 @@ type report struct { RallyStats []rallyStat } -func createReport(benchName, corporaFile string, s *scenario, sum *metricsSummary, stats []rallyStat) (reporters.Reportable, error) { +func createReport(benchName, corporaFile string, s *common.Scenario, sum *metricsSummary, stats []rallyStat) (reporters.Reportable, error) { r := newReport(benchName, corporaFile, s, sum, stats) human := reporters.NewReport(s.Package, reportHumanFormat(r)) @@ -59,7 +60,7 @@ func createReport(benchName, corporaFile string, s *scenario, sum *metricsSummar return mr, nil } -func newReport(benchName, corporaFile string, s *scenario, sum *metricsSummary, stats []rallyStat) *report { +func newReport(benchName, corporaFile string, s *common.Scenario, sum *metricsSummary, stats []rallyStat) *report { var report report report.Info.Benchmark = benchName report.Info.Description = s.Description diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index 3b9cbb506..ddcbb4da5 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -12,6 +12,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "io" "net/http" "os" @@ -150,7 +151,7 @@ type rallyStat struct { type runner struct { options Options - scenario *scenario + scenario *common.Scenario ctxt servicedeployer.ServiceContext runtimeDataStream string @@ -262,7 +263,7 @@ func (r *runner) setUp() error { return fmt.Errorf("reading package manifest failed: %w", err) } - scenario, err := readConfig(r.options.PackageRootPath, r.options.BenchName, pkgManifest.Name, pkgManifest.Version) + scenario, err := common.ReadConfig(r.options.PackageRootPath, r.options.BenchName, pkgManifest.Name, pkgManifest.Version) if err != nil { return err } @@ -596,7 +597,7 @@ func (r *runner) getGeneratorConfig() (*config.Config, error) { ) if r.scenario.Corpora.Generator.Config.Path != "" { - configPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Config.Path)) + configPath := filepath.Clean(filepath.Join(common.DevPath, r.scenario.Corpora.Generator.Config.Path)) configPath = os.ExpandEnv(configPath) if _, err := os.Stat(configPath); err != nil { return nil, fmt.Errorf("can't find config file %s: %w", configPath, err) @@ -627,7 +628,7 @@ func (r *runner) getGeneratorFields() (fields.Fields, error) { ) if r.scenario.Corpora.Generator.Fields.Path != "" { - fieldsPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Fields.Path)) + fieldsPath := filepath.Clean(filepath.Join(common.DevPath, r.scenario.Corpora.Generator.Fields.Path)) fieldsPath = os.ExpandEnv(fieldsPath) if _, err := os.Stat(fieldsPath); err != nil { return nil, fmt.Errorf("can't find fields file %s: %w", fieldsPath, err) @@ -659,7 +660,7 @@ func (r *runner) getGeneratorTemplate() ([]byte, error) { ) if r.scenario.Corpora.Generator.Template.Path != "" { - tplPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Template.Path)) + tplPath := filepath.Clean(filepath.Join(common.DevPath, r.scenario.Corpora.Generator.Template.Path)) tplPath = os.ExpandEnv(tplPath) if _, err := os.Stat(tplPath); err != nil { return nil, fmt.Errorf("can't find template file %s: %w", tplPath, err) @@ -1113,7 +1114,7 @@ type benchMeta struct { Benchmark string `json:"benchmark"` RunID string `json:"run_id"` } `json:"info"` - Parameters scenario `json:"parameter"` + Parameters common.Scenario `json:"parameter"` } func (r *runner) enrichEventWithBenchmarkMetadata(e map[string]interface{}) map[string]interface{} { diff --git a/internal/benchrunner/runners/rally/scenario.go b/internal/benchrunner/runners/rally/scenario.go index 27982bf61..7c788993a 100644 --- a/internal/benchrunner/runners/rally/scenario.go +++ b/internal/benchrunner/runners/rally/scenario.go @@ -3,74 +3,3 @@ // you may not use this file except in compliance with the Elastic License. package rally - -import ( - "errors" - "fmt" - "path/filepath" - - "github.com/elastic/go-ucfg/yaml" -) - -const devPath = "_dev/benchmark/rally" - -type scenario struct { - Package string `config:"package" json:"package"` - Description string `config:"description" json:"description"` - Version string `config:"version" json:"version"` - DataStream dataStream `config:"data_stream" json:"data_stream"` - Corpora corpora `config:"corpora" json:"corpora"` -} - -type dataStream struct { - Name string `config:"name" json:"name"` -} - -type corpora struct { - Generator *generator `config:"generator" json:"generator"` -} - -type generator struct { - TotalEvents uint64 `config:"total_events" json:"total_events"` - Template corporaTemplate `config:"template" json:"template"` - Config corporaAsset `config:"config" json:"config"` - Fields corporaAsset `config:"fields" json:"fields"` -} - -type corporaAsset struct { - Raw map[string]interface{} `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` -} -type corporaTemplate struct { - Raw string `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` - Type string `config:"type" json:"type"` -} - -func defaultConfig() *scenario { - return &scenario{} -} - -func readConfig(path, scenario, packageName, packageVersion string) (*scenario, error) { - configPath := filepath.Join(path, devPath, fmt.Sprintf("%s.yml", scenario)) - c := defaultConfig() - cfg, err := yaml.NewConfigWithFile(configPath) - if err != nil { - return nil, fmt.Errorf("can't load benchmark configuration: %s: %w", configPath, err) - } - - if err == nil { - if err := cfg.Unpack(c); err != nil { - return nil, fmt.Errorf("can't unpack benchmark configuration: %s: %w", configPath, err) - } - } - - c.Package = packageName - c.Version = packageVersion - - if c.DataStream.Name == "" { - return nil, errors.New("can't read data stream name from benchmark configuration: empty") - } - - return c, nil -} diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index ef50cbe90..d6e4e44fc 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "io" "net/http" "os" @@ -39,7 +40,7 @@ import ( type runner struct { options Options - scenarios map[string]*scenario + scenarios map[string]*common.Scenario ctxt servicedeployer.ServiceContext runtimeDataStreams map[string]string @@ -114,7 +115,7 @@ func (r *runner) setUp() error { return fmt.Errorf("reading package manifest failed: %w", err) } - scenarios, err := readScenarios(r.options.PackageRootPath, r.options.BenchName, pkgManifest.Name, pkgManifest.Version) + scenarios, err := common.ReadScenarios(r.options.PackageRootPath, r.options.BenchName, pkgManifest.Name, pkgManifest.Version) if err != nil { return err } @@ -271,7 +272,7 @@ func (r *runner) deleteDataStreamDocs(dataStream string) error { return nil } -func (r *runner) initializeGenerator(tpl []byte, config genlib.Config, fields genlib.Fields, scenario *scenario, backFill time.Duration, totEvents uint64) (genlib.Generator, error) { +func (r *runner) initializeGenerator(tpl []byte, config genlib.Config, fields genlib.Fields, scenario *common.Scenario, backFill time.Duration, totEvents uint64) (genlib.Generator, error) { timestampConfig := genlib.ConfigField{Name: r.options.TimestampField} if backFill < 0 { timestampConfig.Period = backFill @@ -334,14 +335,14 @@ func (r *runner) collectGenerators() error { return nil } -func (r *runner) getGeneratorConfig(scenario *scenario) (*config.Config, error) { +func (r *runner) getGeneratorConfig(scenario *common.Scenario) (*config.Config, error) { var ( data []byte err error ) if scenario.Corpora.Generator.Config.Path != "" { - configPath := filepath.Clean(filepath.Join(devPath, scenario.Corpora.Generator.Config.Path)) + configPath := filepath.Clean(filepath.Join(common.DevPath, scenario.Corpora.Generator.Config.Path)) configPath = os.ExpandEnv(configPath) if _, err := os.Stat(configPath); err != nil { return nil, fmt.Errorf("can't find config file %s: %w", configPath, err) @@ -365,14 +366,14 @@ func (r *runner) getGeneratorConfig(scenario *scenario) (*config.Config, error) return &cfg, nil } -func (r *runner) getGeneratorFields(scenario *scenario) (fields.Fields, error) { +func (r *runner) getGeneratorFields(scenario *common.Scenario) (fields.Fields, error) { var ( data []byte err error ) if scenario.Corpora.Generator.Fields.Path != "" { - fieldsPath := filepath.Clean(filepath.Join(devPath, scenario.Corpora.Generator.Fields.Path)) + fieldsPath := filepath.Clean(filepath.Join(common.DevPath, scenario.Corpora.Generator.Fields.Path)) fieldsPath = os.ExpandEnv(fieldsPath) if _, err := os.Stat(fieldsPath); err != nil { return nil, fmt.Errorf("can't find fields file %s: %w", fieldsPath, err) @@ -397,14 +398,14 @@ func (r *runner) getGeneratorFields(scenario *scenario) (fields.Fields, error) { return fields, nil } -func (r *runner) getGeneratorTemplate(scenario *scenario) ([]byte, error) { +func (r *runner) getGeneratorTemplate(scenario *common.Scenario) ([]byte, error) { var ( data []byte err error ) if scenario.Corpora.Generator.Template.Path != "" { - tplPath := filepath.Clean(filepath.Join(devPath, scenario.Corpora.Generator.Template.Path)) + tplPath := filepath.Clean(filepath.Join(common.DevPath, scenario.Corpora.Generator.Template.Path)) tplPath = os.ExpandEnv(tplPath) if _, err := os.Stat(tplPath); err != nil { return nil, fmt.Errorf("can't find template file %s: %w", tplPath, err) diff --git a/internal/benchrunner/runners/stream/scenario.go b/internal/benchrunner/runners/stream/scenario.go index 36abf5f84..745c47818 100644 --- a/internal/benchrunner/runners/stream/scenario.go +++ b/internal/benchrunner/runners/stream/scenario.go @@ -3,112 +3,3 @@ // you may not use this file except in compliance with the Elastic License. package stream - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/elastic/go-ucfg/yaml" -) - -const devPath = "_dev/benchmark/rally" - -type scenario struct { - Package string `config:"package" json:"package"` - Description string `config:"description" json:"description"` - Version string `config:"version" json:"version"` - DataStream dataStream `config:"data_stream" json:"data_stream"` - Corpora corpora `config:"corpora" json:"corpora"` -} - -type dataStream struct { - Name string `config:"name" json:"name"` -} - -type corpora struct { - Generator *generator `config:"generator" json:"generator"` -} - -type generator struct { - TotalEvents uint64 `config:"total_events" json:"total_events"` - Template corporaTemplate `config:"template" json:"template"` - Config corporaAsset `config:"config" json:"config"` - Fields corporaAsset `config:"fields" json:"fields"` -} - -type corporaAsset struct { - Raw map[string]interface{} `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` -} -type corporaTemplate struct { - Raw string `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` - Type string `config:"type" json:"type"` -} - -func defaultConfig() *scenario { - return &scenario{} -} - -func readConfig(path, scenarioName, packageName, packageVersion string) (*scenario, error) { - configPath := filepath.Join(path, devPath, fmt.Sprintf("%s.yml", scenarioName)) - c := defaultConfig() - cfg, err := yaml.NewConfigWithFile(configPath) - if err != nil { - return nil, fmt.Errorf("can't load benchmark configuration: %s: %w", configPath, err) - } - - if err == nil { - if err := cfg.Unpack(c); err != nil { - return nil, fmt.Errorf("can't unpack benchmark configuration: %s: %w", configPath, err) - } - } - - c.Package = packageName - c.Version = packageVersion - - if c.DataStream.Name == "" { - return nil, errors.New("can't read data stream name from benchmark configuration: empty") - } - - return c, nil -} -func readScenarios(path, scenarioName, packageName, packageVersion string) (map[string]*scenario, error) { - scenarios := make(map[string]*scenario) - if len(scenarioName) > 0 { - scenario, err := readConfig(path, scenarioName, packageName, packageVersion) - if err != nil { - return nil, fmt.Errorf("error loading scenario: %w", err) - } - scenarios[scenarioName] = scenario - } else { - err := filepath.Walk(filepath.Join(path, devPath), func(_ string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - return nil - } - - if strings.HasSuffix(info.Name(), "-benchmark.yml") { - scenarioName = strings.TrimSuffix(info.Name(), ".yml") - scenario, err := readConfig(path, scenarioName, packageName, packageVersion) - if err != nil { - return err - } - scenarios[scenarioName] = scenario - } - - return nil - }) - if err != nil { - return nil, fmt.Errorf("error loading scenario: %w", err) - } - } - - return scenarios, nil -} diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go index 00a7909f0..c1cc80e08 100644 --- a/internal/benchrunner/runners/system/scenario.go +++ b/internal/benchrunner/runners/system/scenario.go @@ -7,6 +7,7 @@ package system import ( "errors" "fmt" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "os" "path/filepath" "time" @@ -32,14 +33,15 @@ type scenario struct { Corpora corpora `config:"corpora" json:"corpora"` } +// TODO: Why is this and next one slightly different from the common fields? type dataStream struct { Name string `config:"name" json:"name"` Vars map[string]interface{} `config:"vars" json:"vars"` } type corpora struct { - Generator *generator `config:"generator" json:"generator"` - InputService *inputService `config:"input_service" json:"input_service"` + Generator *common.Generator `config:"generator" json:"generator"` + InputService *inputService `config:"input_service" json:"input_service"` } type inputService struct { @@ -47,24 +49,13 @@ type inputService struct { Signal string `config:"signal" json:"signal"` } -type generator struct { - TotalEvents uint64 `config:"total_events" json:"total_events"` - Template corporaTemplate `config:"template" json:"template"` - Config corporaConfig `config:"config" json:"config"` - Fields corporaFields `config:"fields" json:"fields"` -} - -type corporaTemplate struct { - Raw string `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` - Type string `config:"type" json:"type"` -} - +// TODO: This seems to be identical to CorporaAsset type corporaConfig struct { Raw map[string]interface{} `config:"raw" json:"raw"` Path string `config:"path" json:"path"` } +// TODO: This seems to be identical to CorporaAsset type corporaFields struct { Raw map[string]interface{} `config:"raw" json:"raw"` Path string `config:"path" json:"path"` From 3de300c35318323c09cbb11f8dbe5c5a05380477 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 21 Dec 2023 16:10:22 +0100 Subject: [PATCH 2/5] remove obsolete methods --- internal/benchrunner/runners/system/scenario.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go index c1cc80e08..bbccfcc63 100644 --- a/internal/benchrunner/runners/system/scenario.go +++ b/internal/benchrunner/runners/system/scenario.go @@ -49,18 +49,6 @@ type inputService struct { Signal string `config:"signal" json:"signal"` } -// TODO: This seems to be identical to CorporaAsset -type corporaConfig struct { - Raw map[string]interface{} `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` -} - -// TODO: This seems to be identical to CorporaAsset -type corporaFields struct { - Raw map[string]interface{} `config:"raw" json:"raw"` - Path string `config:"path" json:"path"` -} - func defaultConfig() *scenario { timeout := 10 * time.Minute return &scenario{ From 7bea775e6e12dd2a5a419c66d8b634f32825ba3d Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 21 Dec 2023 16:11:46 +0100 Subject: [PATCH 3/5] add note about more common fields that colud use refactoring --- internal/benchrunner/runners/system/scenario.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go index bbccfcc63..027e92999 100644 --- a/internal/benchrunner/runners/system/scenario.go +++ b/internal/benchrunner/runners/system/scenario.go @@ -33,12 +33,13 @@ type scenario struct { Corpora corpora `config:"corpora" json:"corpora"` } -// TODO: Why is this and next one slightly different from the common fields? +// TODO: Why is this slightly different from the common fields? type dataStream struct { Name string `config:"name" json:"name"` Vars map[string]interface{} `config:"vars" json:"vars"` } +// TODO: Why is this slightly different from the common fields? type corpora struct { Generator *common.Generator `config:"generator" json:"generator"` InputService *inputService `config:"input_service" json:"input_service"` From e5d444aabc815951e06e948052b49a6730448581 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 15 Jan 2024 16:49:35 +0100 Subject: [PATCH 4/5] cleanup imporots --- internal/benchrunner/runners/common/scenario.go | 7 ++++++- internal/benchrunner/runners/rally/metrics.go | 3 ++- internal/benchrunner/runners/rally/report.go | 3 ++- internal/benchrunner/runners/rally/runner.go | 3 ++- internal/benchrunner/runners/stream/runner.go | 3 ++- internal/benchrunner/runners/system/scenario.go | 3 ++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/benchrunner/runners/common/scenario.go b/internal/benchrunner/runners/common/scenario.go index 7be4098e8..05c95d449 100644 --- a/internal/benchrunner/runners/common/scenario.go +++ b/internal/benchrunner/runners/common/scenario.go @@ -1,12 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package common import ( "errors" "fmt" - "github.com/elastic/go-ucfg/yaml" "os" "path/filepath" "strings" + + "github.com/elastic/go-ucfg/yaml" ) // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/internal/benchrunner/runners/rally/metrics.go b/internal/benchrunner/runners/rally/metrics.go index 2d73d55c6..a6a965e32 100644 --- a/internal/benchrunner/runners/rally/metrics.go +++ b/internal/benchrunner/runners/rally/metrics.go @@ -9,12 +9,13 @@ import ( _ "embed" "encoding/json" "fmt" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "io" "sync" "sync/atomic" "time" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/elasticsearch/ingest" "github.com/elastic/elastic-package/internal/logger" diff --git a/internal/benchrunner/runners/rally/report.go b/internal/benchrunner/runners/rally/report.go index 2958fc24a..728e73f21 100644 --- a/internal/benchrunner/runners/rally/report.go +++ b/internal/benchrunner/runners/rally/report.go @@ -7,10 +7,11 @@ package rally import ( "encoding/json" "fmt" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "strings" "time" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/dustin/go-humanize" "github.com/jedib0t/go-pretty/table" "github.com/jedib0t/go-pretty/text" diff --git a/internal/benchrunner/runners/rally/runner.go b/internal/benchrunner/runners/rally/runner.go index ddcbb4da5..34a2f3b5a 100644 --- a/internal/benchrunner/runners/rally/runner.go +++ b/internal/benchrunner/runners/rally/runner.go @@ -12,7 +12,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "io" "net/http" "os" @@ -22,6 +21,8 @@ import ( "text/template" "time" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/elastic/elastic-package/internal/packages/installer" "github.com/magefile/mage/sh" diff --git a/internal/benchrunner/runners/stream/runner.go b/internal/benchrunner/runners/stream/runner.go index d6e4e44fc..bcf97a866 100644 --- a/internal/benchrunner/runners/stream/runner.go +++ b/internal/benchrunner/runners/stream/runner.go @@ -10,7 +10,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "io" "net/http" "os" @@ -19,6 +18,8 @@ import ( "sync" "time" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/elastic/elastic-package/internal/packages/installer" "github.com/google/uuid" diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go index 027e92999..460872ab4 100644 --- a/internal/benchrunner/runners/system/scenario.go +++ b/internal/benchrunner/runners/system/scenario.go @@ -7,11 +7,12 @@ package system import ( "errors" "fmt" - "github.com/elastic/elastic-package/internal/benchrunner/runners/common" "os" "path/filepath" "time" + "github.com/elastic/elastic-package/internal/benchrunner/runners/common" + "github.com/aymerick/raymond" "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" From dab5303877c8cf229df1c29fd2686131202746c5 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 23 Jan 2024 09:10:24 +0100 Subject: [PATCH 5/5] remove todo --- internal/benchrunner/runners/system/scenario.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go index 460872ab4..d775e4778 100644 --- a/internal/benchrunner/runners/system/scenario.go +++ b/internal/benchrunner/runners/system/scenario.go @@ -34,13 +34,11 @@ type scenario struct { Corpora corpora `config:"corpora" json:"corpora"` } -// TODO: Why is this slightly different from the common fields? type dataStream struct { Name string `config:"name" json:"name"` Vars map[string]interface{} `config:"vars" json:"vars"` } -// TODO: Why is this slightly different from the common fields? type corpora struct { Generator *common.Generator `config:"generator" json:"generator"` InputService *inputService `config:"input_service" json:"input_service"`