Redact utility statements by default (#588)

pganalyze · Aug 29, 2024 · 6ea5a56 · 6ea5a56
1 parent 44486e8
commit 6ea5a56
Show file tree

Hide file tree

Showing 13 changed files with 123 additions and 11 deletions.
diff --git a/go.mod b/go.mod
@@ -44,7 +44,7 @@ require (
 	github.com/fatih/color v1.16.0
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.1
-	github.com/pganalyze/pg_query_go/v5 v5.1.0
+	github.com/pganalyze/pg_query_go/v5 v5.1.1-0.20240829182208-c3a818d346a9
 	github.com/prometheus/procfs v0.7.3
 	go.opentelemetry.io/otel v1.19.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0

diff --git a/go.sum b/go.sum
@@ -156,8 +156,8 @@ github.com/ogier/pflag v0.0.0-20160129220114-45c278ab3607 h1:db+rES1EpSjP45xOU3h
 github.com/ogier/pflag v0.0.0-20160129220114-45c278ab3607/go.mod h1:zkFki7tvTa0tafRvTBIZTvzYyAu6kQhPZFnshFFPE+g=
 github.com/papertrail/go-tail v0.0.0-20180509224916-973c153b0431 h1:i1egM7gz4bPxLCIwBJOkpk6TqHpjTnL4dE1xdN/4dcs=
 github.com/papertrail/go-tail v0.0.0-20180509224916-973c153b0431/go.mod h1:dMID0RaS2a5rhpOjC4RsAKitU6WGgkFBZnPVffL69b8=
-github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg=
-github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug=
+github.com/pganalyze/pg_query_go/v5 v5.1.1-0.20240829182208-c3a818d346a9 h1:9aymnVwCh0j7i+Myb1x3g+Xm1JtAk4TrTAyqBwCWhN8=
+github.com/pganalyze/pg_query_go/v5 v5.1.1-0.20240829182208-c3a818d346a9/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

diff --git a/logs/analyze.go b/logs/analyze.go
@@ -11,6 +11,7 @@ import (
 	"github.com/pganalyze/collector/logs/util"
 	"github.com/pganalyze/collector/output/pganalyze_collector"
 	"github.com/pganalyze/collector/state"
+	pg_query "github.com/pganalyze/pg_query_go/v5"
 )
 
 type match struct {
@@ -2246,6 +2247,9 @@ func AnalyzeBackendLogLines(logLines []state.LogLine) (logLinesOut []state.LogLi
 
 		logLine, statementLine, detailLine, contextLine, hintLine, samples = classifyAndSetDetails(logLine, statementLine, detailLine, contextLine, hintLine, samples)
 
+		markUtilitySecret(&logLine)
+		markUtilitySecret(&statementLine)
+
 		if statementLineIdx != 0 {
 			logLines[statementLineIdx] = statementLine
 		}
@@ -2270,3 +2274,20 @@ func AnalyzeBackendLogLines(logLines []state.LogLine) (logLinesOut []state.LogLi
 
 	return
 }
+
+func markUtilitySecret(line *state.LogLine) {
+	for _, m := range line.SecretMarkers {
+		if m.Kind == state.StatementTextLogSecret {
+			query := line.Content[m.ByteStart:m.ByteEnd]
+			normalized, err := pg_query.NormalizeUtility(query)
+			if err == nil && query != normalized {
+				line.SecretMarkers = append(line.SecretMarkers, state.LogSecretMarker{
+					ByteStart: m.ByteStart,
+					ByteEnd:   m.ByteEnd,
+					Kind:      state.CredentialLogSecret,
+				})
+				return
+			}
+		}
+	}
+}
diff --git a/logs/analyze_test.go b/logs/analyze_test.go
@@ -155,6 +155,34 @@ var tests = []testpair{
 		}},
 		nil,
 	},
+	{
+		[]state.LogLine{{
+			Content:  "duration: 3205.800 ms  execute a2: CREATE ROLE postgres PASSWORD 'xyz'\n",
+			LogLevel: pganalyze_collector.LogLineInformation_LOG,
+		}},
+		[]state.LogLine{{
+			Query:              "CREATE ROLE postgres PASSWORD 'xyz'",
+			LogLevel:           pganalyze_collector.LogLineInformation_LOG,
+			Classification:     pganalyze_collector.LogLineInformation_STATEMENT_DURATION,
+			ReviewedForSecrets: true,
+			SecretMarkers: []state.LogSecretMarker{
+				{
+					ByteStart: 35,
+					ByteEnd:   70,
+					Kind:      state.StatementTextLogSecret,
+				},
+				{
+					ByteStart: 35,
+					ByteEnd:   70,
+					Kind:      state.CredentialLogSecret,
+				},
+			},
+		}},
+		[]state.PostgresQuerySample{{
+			Query:     "CREATE ROLE postgres PASSWORD 'xyz'",
+			RuntimeMs: 3205.8,
+		}},
+	},
 	// Statement log
 	{
 		[]state.LogLine{{

diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/CHANGELOG.md b/vendor/github.com/pganalyze/pg_query_go/v5/CHANGELOG.md
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/Makefile b/vendor/github.com/pganalyze/pg_query_go/v5/Makefile
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/README.md b/vendor/github.com/pganalyze/pg_query_go/v5/README.md
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/parser/include/pg_query.h b/vendor/github.com/pganalyze/pg_query_go/v5/parser/include/pg_query.h
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/parser/parser.go b/vendor/github.com/pganalyze/pg_query_go/v5/parser/parser.go
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/parser/pg_query_normalize.c b/vendor/github.com/pganalyze/pg_query_go/v5/parser/pg_query_normalize.c
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/parser/pg_query_scan.c b/vendor/github.com/pganalyze/pg_query_go/v5/parser/pg_query_scan.c
diff --git a/vendor/github.com/pganalyze/pg_query_go/v5/pg_query.go b/vendor/github.com/pganalyze/pg_query_go/v5/pg_query.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -318,7 +318,7 @@ github.com/ogier/pflag
 # github.com/papertrail/go-tail v0.0.0-20180509224916-973c153b0431
 ## explicit
 github.com/papertrail/go-tail/follower
-# github.com/pganalyze/pg_query_go/v5 v5.1.0
+# github.com/pganalyze/pg_query_go/v5 v5.1.1-0.20240829182208-c3a818d346a9
 ## explicit; go 1.14
 github.com/pganalyze/pg_query_go/v5
 github.com/pganalyze/pg_query_go/v5/parser