From df7ebb450f80c4ec104c61c98856fe5920a4ae32 Mon Sep 17 00:00:00 2001 From: mwdomino Date: Wed, 31 Jul 2024 14:22:36 -0400 Subject: [PATCH 1/6] add support for rotating file outputs --- docs/user_guide/outputs/file_output.md | 18 ++++++-- pkg/outputs/file/file_output.go | 50 +++++++++++++------- pkg/outputs/file/rotating_file.go | 64 ++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 pkg/outputs/file/rotating_file.go diff --git a/docs/user_guide/outputs/file_output.md b/docs/user_guide/outputs/file_output.md index 0c35a7b4..d03e3080 100644 --- a/docs/user_guide/outputs/file_output.md +++ b/docs/user_guide/outputs/file_output.md @@ -10,9 +10,9 @@ outputs: # filename to write telemetry data to. # will be ignored if `file-type` is set filename: /path/to/filename - # file-type, stdout or stderr. - # overwrites `filename` - file-type: # stdout or stderr + # file-type, stdout, stderr, or rotating. + # overwrites `filename` if stdout or stderr + file-type: # stdout, stderr, or rotating # string, message formatting, json, protojson, prototext, event format: # string, one of `overwrite`, `if-not-present`, `` @@ -54,10 +54,20 @@ outputs: enable-metrics: false # list of processors to apply on the message before writing event-processors: + # file rotation configuration + # all fields are required if enabling file rotation + rotation: + max-size: 100 # size in megabytes + max-age: 30 # max age in days + max-backups: 3 # maximum number of old files to store, not counting the current file + compress: false # whether or not to enable compression + ``` -The file output can be used to write to file on the disk, to stdout or to stderr. +The file output can be used to write to file on the disk, to stdout or to stderr. Also includes support for rotating files to control disk utilization and maximum age. For a disk file, a file name is required. For stdout or stderr, only file-type is required. + +For rotation, all fields (other than `compress`) are required. diff --git a/pkg/outputs/file/file_output.go b/pkg/outputs/file/file_output.go index 1452bbae..1c7d9fd4 100644 --- a/pkg/outputs/file/file_output.go +++ b/pkg/outputs/file/file_output.go @@ -50,7 +50,7 @@ func init() { // File // type File struct { cfg *Config - file *os.File + file file logger *log.Logger mo *formatters.MarshalOptions sem *semaphore.Weighted @@ -62,22 +62,29 @@ type File struct { // Config // type Config struct { - FileName string `mapstructure:"filename,omitempty"` - FileType string `mapstructure:"file-type,omitempty"` - Format string `mapstructure:"format,omitempty"` - Multiline bool `mapstructure:"multiline,omitempty"` - Indent string `mapstructure:"indent,omitempty"` - Separator string `mapstructure:"separator,omitempty"` - SplitEvents bool `mapstructure:"split-events,omitempty"` - OverrideTimestamps bool `mapstructure:"override-timestamps,omitempty"` - AddTarget string `mapstructure:"add-target,omitempty"` - TargetTemplate string `mapstructure:"target-template,omitempty"` - EventProcessors []string `mapstructure:"event-processors,omitempty"` - MsgTemplate string `mapstructure:"msg-template,omitempty"` - ConcurrencyLimit int `mapstructure:"concurrency-limit,omitempty"` - EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` - Debug bool `mapstructure:"debug,omitempty"` - CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` + FileName string `mapstructure:"filename,omitempty"` + FileType string `mapstructure:"file-type,omitempty"` + Format string `mapstructure:"format,omitempty"` + Multiline bool `mapstructure:"multiline,omitempty"` + Indent string `mapstructure:"indent,omitempty"` + Separator string `mapstructure:"separator,omitempty"` + SplitEvents bool `mapstructure:"split-events,omitempty"` + OverrideTimestamps bool `mapstructure:"override-timestamps,omitempty"` + AddTarget string `mapstructure:"add-target,omitempty"` + TargetTemplate string `mapstructure:"target-template,omitempty"` + EventProcessors []string `mapstructure:"event-processors,omitempty"` + MsgTemplate string `mapstructure:"msg-template,omitempty"` + ConcurrencyLimit int `mapstructure:"concurrency-limit,omitempty"` + EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` + Debug bool `mapstructure:"debug,omitempty"` + CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` + Rotation RotationConfig `mapstructure:"rotation,omitempty"` +} + +type file interface { + Close() error + Name() string + Write([]byte) (int, error) } func (f *File) String() string { @@ -142,6 +149,15 @@ func (f *File) Init(ctx context.Context, name string, cfg map[string]interface{} f.file = os.Stdout case "stderr": f.file = os.Stderr + case "rotating": + err := f.cfg.Rotation.validateConfig() + if err != nil { + f.logger.Printf("failed to validate rotating configuration: %v", err) + + return err + } + + f.file = newRotatingFile(f.cfg.FileName, f.cfg.Rotation.Compress, f.cfg.Rotation.MaxSize, f.cfg.Rotation.MaxBackups, f.cfg.Rotation.MaxAge) default: CRFILE: f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) diff --git a/pkg/outputs/file/rotating_file.go b/pkg/outputs/file/rotating_file.go new file mode 100644 index 00000000..5838d511 --- /dev/null +++ b/pkg/outputs/file/rotating_file.go @@ -0,0 +1,64 @@ +package file + +import ( + "fmt" + + "gopkg.in/natefinch/lumberjack.v2" +) + +// RotationConfig manages configuration around file rotation +type RotationConfig struct { + MaxSize int `mapstructure:"max-size,omitempty"` + MaxBackups int `mapstructure:"max-backups,omitempty"` + MaxAge int `mapstructure:"max-age,omitempty"` + Compress bool `mapstructure:"compress,omitempty"` +} + +// validateConfig ensures all parameters are supplied +func (rc *RotationConfig) validateConfig() error { + if rc.MaxSize == 0 { + return fmt.Errorf("Rotation.MaxSize is required if using type 'rotating'") + } + + if rc.MaxBackups == 0 { + return fmt.Errorf("Rotation.MaxBackups is required if using type 'rotating'") + } + + if rc.MaxAge == 0 { + return fmt.Errorf("Rotation.MaxAge is required if using type 'rotating'") + } + + return nil +} + +type rotatingFile struct { + l *lumberjack.Logger +} + +// newRotatingFile initialize the lumberjack instance +func newRotatingFile(filename string, compress bool, maxSize, maxBackups, maxAge int) *rotatingFile { + lj := lumberjack.Logger{ + Filename: filename, + MaxSize: maxSize, + MaxBackups: maxBackups, + MaxAge: maxAge, + Compress: compress, + } + + return &rotatingFile{l: &lj} +} + +// Close closes the file +func (r *rotatingFile) Close() error { + return r.l.Close() +} + +// Name returns the name of the file +func (r *rotatingFile) Name() string { + return r.l.Filename +} + +// Write implements io.Writer +func (r *rotatingFile) Write(b []byte) (int, error) { + return r.l.Write(b) +} From 3677b9ba07333f66ef6b5630eba03f0b21b728ed Mon Sep 17 00:00:00 2001 From: mwdomino Date: Wed, 31 Jul 2024 14:32:18 -0400 Subject: [PATCH 2/6] clean up file output --- pkg/outputs/file/file_output.go | 2 +- pkg/outputs/file/rotating_file.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/outputs/file/file_output.go b/pkg/outputs/file/file_output.go index 1c7d9fd4..8a77690a 100644 --- a/pkg/outputs/file/file_output.go +++ b/pkg/outputs/file/file_output.go @@ -157,7 +157,7 @@ func (f *File) Init(ctx context.Context, name string, cfg map[string]interface{} return err } - f.file = newRotatingFile(f.cfg.FileName, f.cfg.Rotation.Compress, f.cfg.Rotation.MaxSize, f.cfg.Rotation.MaxBackups, f.cfg.Rotation.MaxAge) + f.file = newRotatingFile(f.cfg) default: CRFILE: f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) diff --git a/pkg/outputs/file/rotating_file.go b/pkg/outputs/file/rotating_file.go index 5838d511..e6973545 100644 --- a/pkg/outputs/file/rotating_file.go +++ b/pkg/outputs/file/rotating_file.go @@ -36,13 +36,13 @@ type rotatingFile struct { } // newRotatingFile initialize the lumberjack instance -func newRotatingFile(filename string, compress bool, maxSize, maxBackups, maxAge int) *rotatingFile { +func newRotatingFile(cfg *Config) *rotatingFile { lj := lumberjack.Logger{ - Filename: filename, - MaxSize: maxSize, - MaxBackups: maxBackups, - MaxAge: maxAge, - Compress: compress, + Filename: cfg.FileName, + MaxSize: cfg.Rotation.MaxSize, + MaxBackups: cfg.Rotation.MaxBackups, + MaxAge: cfg.Rotation.MaxAge, + Compress: cfg.Rotation.Compress, } return &rotatingFile{l: &lj} From f1186c1ee0e1a2de64149e95beebebdaf6f03637 Mon Sep 17 00:00:00 2001 From: mwdomino Date: Wed, 31 Jul 2024 16:24:27 -0400 Subject: [PATCH 3/6] review fixes --- docs/user_guide/outputs/file_output.md | 3 --- pkg/outputs/file/file_output.go | 9 +-------- pkg/outputs/file/rotating_file.go | 24 ++++++++++-------------- 3 files changed, 11 insertions(+), 25 deletions(-) diff --git a/docs/user_guide/outputs/file_output.md b/docs/user_guide/outputs/file_output.md index d03e3080..6d7f4b1f 100644 --- a/docs/user_guide/outputs/file_output.md +++ b/docs/user_guide/outputs/file_output.md @@ -55,7 +55,6 @@ outputs: # list of processors to apply on the message before writing event-processors: # file rotation configuration - # all fields are required if enabling file rotation rotation: max-size: 100 # size in megabytes max-age: 30 # max age in days @@ -69,5 +68,3 @@ The file output can be used to write to file on the disk, to stdout or to stderr For a disk file, a file name is required. For stdout or stderr, only file-type is required. - -For rotation, all fields (other than `compress`) are required. diff --git a/pkg/outputs/file/file_output.go b/pkg/outputs/file/file_output.go index 8a77690a..bac20212 100644 --- a/pkg/outputs/file/file_output.go +++ b/pkg/outputs/file/file_output.go @@ -78,7 +78,7 @@ type Config struct { EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` Debug bool `mapstructure:"debug,omitempty"` CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` - Rotation RotationConfig `mapstructure:"rotation,omitempty"` + Rotation rotationConfig `mapstructure:"rotation,omitempty"` } type file interface { @@ -150,13 +150,6 @@ func (f *File) Init(ctx context.Context, name string, cfg map[string]interface{} case "stderr": f.file = os.Stderr case "rotating": - err := f.cfg.Rotation.validateConfig() - if err != nil { - f.logger.Printf("failed to validate rotating configuration: %v", err) - - return err - } - f.file = newRotatingFile(f.cfg) default: CRFILE: diff --git a/pkg/outputs/file/rotating_file.go b/pkg/outputs/file/rotating_file.go index e6973545..6b5098db 100644 --- a/pkg/outputs/file/rotating_file.go +++ b/pkg/outputs/file/rotating_file.go @@ -1,34 +1,28 @@ package file import ( - "fmt" - "gopkg.in/natefinch/lumberjack.v2" ) // RotationConfig manages configuration around file rotation -type RotationConfig struct { +type rotationConfig struct { MaxSize int `mapstructure:"max-size,omitempty"` MaxBackups int `mapstructure:"max-backups,omitempty"` MaxAge int `mapstructure:"max-age,omitempty"` Compress bool `mapstructure:"compress,omitempty"` } -// validateConfig ensures all parameters are supplied -func (rc *RotationConfig) validateConfig() error { - if rc.MaxSize == 0 { - return fmt.Errorf("Rotation.MaxSize is required if using type 'rotating'") +func (r *rotationConfig) SetDefaults() { + if r.MaxSize == 0 { + r.MaxSize = 100 } - - if rc.MaxBackups == 0 { - return fmt.Errorf("Rotation.MaxBackups is required if using type 'rotating'") + if r.MaxBackups == 0 { + r.MaxBackups = 3 } - if rc.MaxAge == 0 { - return fmt.Errorf("Rotation.MaxAge is required if using type 'rotating'") + if r.MaxAge == 0 { + r.MaxAge = 30 } - - return nil } type rotatingFile struct { @@ -37,6 +31,8 @@ type rotatingFile struct { // newRotatingFile initialize the lumberjack instance func newRotatingFile(cfg *Config) *rotatingFile { + cfg.Rotation.SetDefaults() + lj := lumberjack.Logger{ Filename: cfg.FileName, MaxSize: cfg.Rotation.MaxSize, From 7662a34323c531ea6b84bc7bbb423af3357f2243 Mon Sep 17 00:00:00 2001 From: mwdomino Date: Fri, 2 Aug 2024 09:16:22 -0400 Subject: [PATCH 4/6] review fix --- pkg/outputs/file/file_output.go | 50 +++++++++++++++++---------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/pkg/outputs/file/file_output.go b/pkg/outputs/file/file_output.go index bac20212..aed8bbd8 100644 --- a/pkg/outputs/file/file_output.go +++ b/pkg/outputs/file/file_output.go @@ -62,23 +62,23 @@ type File struct { // Config // type Config struct { - FileName string `mapstructure:"filename,omitempty"` - FileType string `mapstructure:"file-type,omitempty"` - Format string `mapstructure:"format,omitempty"` - Multiline bool `mapstructure:"multiline,omitempty"` - Indent string `mapstructure:"indent,omitempty"` - Separator string `mapstructure:"separator,omitempty"` - SplitEvents bool `mapstructure:"split-events,omitempty"` - OverrideTimestamps bool `mapstructure:"override-timestamps,omitempty"` - AddTarget string `mapstructure:"add-target,omitempty"` - TargetTemplate string `mapstructure:"target-template,omitempty"` - EventProcessors []string `mapstructure:"event-processors,omitempty"` - MsgTemplate string `mapstructure:"msg-template,omitempty"` - ConcurrencyLimit int `mapstructure:"concurrency-limit,omitempty"` - EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` - Debug bool `mapstructure:"debug,omitempty"` - CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` - Rotation rotationConfig `mapstructure:"rotation,omitempty"` + FileName string `mapstructure:"filename,omitempty"` + FileType string `mapstructure:"file-type,omitempty"` + Format string `mapstructure:"format,omitempty"` + Multiline bool `mapstructure:"multiline,omitempty"` + Indent string `mapstructure:"indent,omitempty"` + Separator string `mapstructure:"separator,omitempty"` + SplitEvents bool `mapstructure:"split-events,omitempty"` + OverrideTimestamps bool `mapstructure:"override-timestamps,omitempty"` + AddTarget string `mapstructure:"add-target,omitempty"` + TargetTemplate string `mapstructure:"target-template,omitempty"` + EventProcessors []string `mapstructure:"event-processors,omitempty"` + MsgTemplate string `mapstructure:"msg-template,omitempty"` + ConcurrencyLimit int `mapstructure:"concurrency-limit,omitempty"` + EnableMetrics bool `mapstructure:"enable-metrics,omitempty"` + Debug bool `mapstructure:"debug,omitempty"` + CalculateLatency bool `mapstructure:"calculate-latency,omitempty"` + Rotation *rotationConfig `mapstructure:"rotation,omitempty"` } type file interface { @@ -149,15 +149,17 @@ func (f *File) Init(ctx context.Context, name string, cfg map[string]interface{} f.file = os.Stdout case "stderr": f.file = os.Stderr - case "rotating": - f.file = newRotatingFile(f.cfg) default: CRFILE: - f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - f.logger.Printf("failed to create file: %v", err) - time.Sleep(10 * time.Second) - goto CRFILE + if f.cfg.Rotation != nil { + f.file = newRotatingFile(f.cfg) + } else { + f.file, err = os.OpenFile(f.cfg.FileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + f.logger.Printf("failed to create file: %v", err) + time.Sleep(10 * time.Second) + goto CRFILE + } } } From acd47141f33a50def243240a23fcc3a9b3335fea Mon Sep 17 00:00:00 2001 From: mwdomino Date: Fri, 2 Aug 2024 09:17:17 -0400 Subject: [PATCH 5/6] update docs --- docs/user_guide/outputs/file_output.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user_guide/outputs/file_output.md b/docs/user_guide/outputs/file_output.md index 6d7f4b1f..2b9d4c70 100644 --- a/docs/user_guide/outputs/file_output.md +++ b/docs/user_guide/outputs/file_output.md @@ -10,9 +10,9 @@ outputs: # filename to write telemetry data to. # will be ignored if `file-type` is set filename: /path/to/filename - # file-type, stdout, stderr, or rotating. + # file-type, stdout or stderr # overwrites `filename` if stdout or stderr - file-type: # stdout, stderr, or rotating + file-type: # stdout or stderr # string, message formatting, json, protojson, prototext, event format: # string, one of `overwrite`, `if-not-present`, `` @@ -63,7 +63,7 @@ outputs: ``` -The file output can be used to write to file on the disk, to stdout or to stderr. Also includes support for rotating files to control disk utilization and maximum age. +The file output can be used to write to file on the disk, to stdout or to stderr. Also includes support for rotating files to control disk utilization and maximum age using the `rotation` configuration section. For a disk file, a file name is required. From 3a77d398bd77bf2343afe47caead68a044db33d7 Mon Sep 17 00:00:00 2001 From: mwdomino Date: Fri, 2 Aug 2024 09:18:07 -0400 Subject: [PATCH 6/6] update docs --- docs/user_guide/outputs/file_output.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/outputs/file_output.md b/docs/user_guide/outputs/file_output.md index 2b9d4c70..737a507d 100644 --- a/docs/user_guide/outputs/file_output.md +++ b/docs/user_guide/outputs/file_output.md @@ -10,8 +10,8 @@ outputs: # filename to write telemetry data to. # will be ignored if `file-type` is set filename: /path/to/filename - # file-type, stdout or stderr - # overwrites `filename` if stdout or stderr + # file-type, stdout or stderr. + # overwrites `filename` file-type: # stdout or stderr # string, message formatting, json, protojson, prototext, event format: