Skip to content

Commit

Permalink
fix: Remove validation on parse target_field (#114)
Browse files Browse the repository at this point in the history 
The target_field of parse and parse sequential processors are optional.
Remove the validation and add tests

Ref: LOG-18805
  • Loading branch information
@biblicalph
biblicalph authored Dec 8, 2023
1 parent 93b693c commit 2f4f708
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 3 deletions.
3 changes: 0 additions & 3 deletions internal/provider/models/processors/base_parse_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,6 @@ var base_schema = SchemaAttributes{
Computed: true,
Description: "The field into which the parsed value should be inserted. Leave blank to " +
"insert the parsed data into the original field.",
Validators: []validator.String{
stringvalidator.LengthAtLeast(1),
},
},
}

Expand Down
85 changes: 85 additions & 0 deletions internal/provider/models/processors/test/parse_sequentially_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,88 @@ func TestParseSequentiallyProcessor(t *testing.T) {
ExpectError: regexp.MustCompile(`(?s)Attribute parsers\[0\].timestamp_parser_options.custom_format string length.*must be at least 1, got: 0`),
},

// parser with empty target
{
Config: GetCachedConfig(cacheKey) + `
resource "mezmo_parse_sequentially_processor" "with_empty_target" {
title = "custom regex parser title"
description = "custom regex parser desc"
pipeline_id = mezmo_pipeline.test_parent.id
field = ".something"
target_field = ""
parsers = [
{
parser = "regex_parser"
regex_parser_options = {
pattern = "\\d{3}-\\d{2}-\\d{3}"
}
}
]
}`,
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr(
"mezmo_parse_sequentially_processor.with_empty_target", "id", regexp.MustCompile(`[\w-]{36}`)),

StateHasExpectedValues("mezmo_parse_sequentially_processor.with_empty_target", map[string]any{
"pipeline_id": "#mezmo_pipeline.test_parent.id",
"title": "custom regex parser title",
"description": "custom regex parser desc",
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parsers.#": "1",
"parsers.0.parser": "regex_parser",
"parsers.0.regex_parser_options.pattern": "\\d{3}-\\d{2}-\\d{3}",
"parsers.0.regex_parser_options.case_sensitive": "true",
"parsers.0.regex_parser_options.multiline": "false",
"parsers.0.regex_parser_options.match_newline": "false",
"parsers.0.regex_parser_options.crlf_newline": "false",
"parsers.0.output_name": regexp.MustCompile(".+"),
}),
),
},
// with explicit target
{
Config: GetCachedConfig(cacheKey) + `
resource "mezmo_parse_sequentially_processor" "with_target" {
title = "custom regex parser title"
description = "custom regex parser desc"
pipeline_id = mezmo_pipeline.test_parent.id
field = ".something"
target_field = ".data_parsed"
parsers = [
{
parser = "regex_parser"
regex_parser_options = {
pattern = "\\d{3}-\\d{2}-\\d{3}"
}
}
]
}`,
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr(
"mezmo_parse_sequentially_processor.with_target", "id", regexp.MustCompile(`[\w-]{36}`)),

StateHasExpectedValues("mezmo_parse_sequentially_processor.with_target", map[string]any{
"pipeline_id": "#mezmo_pipeline.test_parent.id",
"title": "custom regex parser title",
"description": "custom regex parser desc",
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": ".data_parsed",
"parsers.#": "1",
"parsers.0.parser": "regex_parser",
"parsers.0.regex_parser_options.pattern": "\\d{3}-\\d{2}-\\d{3}",
"parsers.0.regex_parser_options.case_sensitive": "true",
"parsers.0.regex_parser_options.multiline": "false",
"parsers.0.regex_parser_options.match_newline": "false",
"parsers.0.regex_parser_options.crlf_newline": "false",
"parsers.0.output_name": regexp.MustCompile(".+"),
}),
),
},
// Create regex parser - default options for regex
{
Config: GetCachedConfig(cacheKey) + `
Expand Down Expand Up @@ -478,6 +560,7 @@ func TestParseSequentiallyProcessor(t *testing.T) {
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parsers.#": "1",
"parsers.0.parser": "regex_parser",
"parsers.0.regex_parser_options.pattern": "\\d{3}-\\d{2}-\\d{3}",
Expand Down Expand Up @@ -521,6 +604,7 @@ func TestParseSequentiallyProcessor(t *testing.T) {
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parsers.#": "1",
"parsers.0.parser": "regex_parser",
"parsers.0.regex_parser_options.pattern": "\\d{3}-\\d{2}-\\d{3}",
Expand Down Expand Up @@ -560,6 +644,7 @@ func TestParseSequentiallyProcessor(t *testing.T) {
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parsers.#": "1",
"parsers.0.parser": "csv_row",
"parsers.0.csv_row_options.field_names.#": "2",
Expand Down
74 changes: 74 additions & 0 deletions internal/provider/models/processors/test/parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,77 @@ func TestParseProcessor(t *testing.T) {
}),
),
},

// Parser with empty target_field
{
Config: GetCachedConfig(cacheKey) + `
resource "mezmo_parse_processor" "with_empty_target" {
title = "custom apache parser title"
description = "custom apache parser desc"
pipeline_id = mezmo_pipeline.test_parent.id
field = ".something"
target_field = ""
parser = "apache_log"
apache_log_options = {
format = "common"
timestamp_format = "Custom"
custom_timestamp_format = "%Y/%m/%d %H:%M:%S"
}
}`,
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr(
"mezmo_parse_processor.with_empty_target", "id", regexp.MustCompile(`[\w-]{36}`)),

StateHasExpectedValues("mezmo_parse_processor.with_empty_target", map[string]any{
"pipeline_id": "#mezmo_pipeline.test_parent.id",
"title": "custom apache parser title",
"description": "custom apache parser desc",
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parser": "apache_log",
"apache_log_options.format": "common",
"apache_log_options.timestamp_format": "Custom",
"apache_log_options.custom_timestamp_format": "%Y/%m/%d %H:%M:%S",
}),
),
},
// with non empty parser
{
Config: GetCachedConfig(cacheKey) + `
resource "mezmo_parse_processor" "with_target" {
title = "custom apache parser title"
description = "custom apache parser desc"
pipeline_id = mezmo_pipeline.test_parent.id
field = ".something"
target_field = ".parsed"
parser = "apache_log"
apache_log_options = {
format = "common"
timestamp_format = "Custom"
custom_timestamp_format = "%Y/%m/%d %H:%M:%S"
}
}`,
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr(
"mezmo_parse_processor.with_target", "id", regexp.MustCompile(`[\w-]{36}`)),

StateHasExpectedValues("mezmo_parse_processor.with_target", map[string]any{
"pipeline_id": "#mezmo_pipeline.test_parent.id",
"title": "custom apache parser title",
"description": "custom apache parser desc",
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": ".parsed",
"parser": "apache_log",
"apache_log_options.format": "common",
"apache_log_options.timestamp_format": "Custom",
"apache_log_options.custom_timestamp_format": "%Y/%m/%d %H:%M:%S",
}),
),
},
// Create apache parser with default timestamp
{
Config: GetCachedConfig(cacheKey) + `
Expand All @@ -458,6 +529,7 @@ func TestParseProcessor(t *testing.T) {
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parser": "apache_log",
"apache_log_options.format": "common",
"apache_log_options.timestamp_format": "%d/%b/%Y:%T %z",
Expand Down Expand Up @@ -490,6 +562,7 @@ func TestParseProcessor(t *testing.T) {
"generation_id": "0",
"inputs.#": "0",
"field": ".something",
"target_field": "",
"parser": "apache_log",
"apache_log_options.format": "common",
"apache_log_options.timestamp_format": "Custom",
Expand Down Expand Up @@ -547,6 +620,7 @@ func TestParseProcessor(t *testing.T) {
"inputs.#": "1",
"inputs.0": "#mezmo_http_source.my_source.id",
"field": ".something",
"target_field": "",
"parser": "nginx_log",
"nginx_log_options.format": "combined",
"nginx_log_options.timestamp_format": "Custom",
Expand Down

0 comments on commit 2f4f708

Please sign in to comment.