Skip to content

Commit

Permalink
PII Filtering: Detect bind parameters in CONTEXT as statement_text
Browse files Browse the repository at this point in the history
This can occur in certain cases, for example auto_explain output
on some Postgres versions. We previously would have not detected
this CONTEXT line, but are now correctly detecting it as containing
statement_parameter log secrets.

In passing add a test case for the new "Query Parameters" field in
auto_explain which gets redacted based on the statement_text filter that
filters out the whole plan text (we normalize the individual fields in
resulting query samples, but the log text for auto_explain is redacted
altogether).
  • Loading branch information
lfittl committed Jan 7, 2025
1 parent 2c37620 commit 7499212
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 1 deletion.
7 changes: 6 additions & 1 deletion logs/analyze.go
Original file line number Diff line number Diff line change
Expand Up @@ -1153,6 +1153,11 @@ var otherContextPatterns = []match{
regexp: regexp.MustCompile(`^JSON data, line (\d+): (.+)`),
secrets: []state.LogSecretKind{0, state.TableDataLogSecret},
},
{
prefixes: []string{"portal \"", "unnamed portal "},
regexp: regexp.MustCompile(`(?:(?:unnamed portal|portal \"(.+)\") with parameters: |, )\$\d+ = (?:(NULL)|'((?:[^']|'')*)')`),
secrets: []state.LogSecretKind{0, state.StatementParameterLogSecret, state.StatementParameterLogSecret},
},
}

var autoVacuumIndexRegexp = regexp.MustCompile(`index "(.+?)": pages: (\d+) in total, (\d+) newly deleted, (\d+) currently deleted, (\d+) reusable,?\s*`)
Expand Down Expand Up @@ -2170,7 +2175,7 @@ func matchOtherContextLogLine(logLine state.LogLine) state.LogLine {
return logLine
}
for _, match := range otherContextPatterns {
logLine, parts := matchLogLine(logLine, match)
logLine, parts := matchLogLineAll(logLine, match)
if parts != nil {
return logLine
}
Expand Down
10 changes: 10 additions & 0 deletions logs/replace_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,21 @@ var replaceTests = []replaceTestpair{
input: "2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:LOG: duration: 1242.570 ms statement: SELECT 1\n",
output: "duration: 1242.570 ms statement: [redacted]\n",
},
{
filterLogSecret: "statement_text",
input: "2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:LOG: duration: 2007.111 ms plan:\n{\"Query Text\": \"SELECT pg_sleep($1)\", \"Query Parameters\": \"$1 = '2'\", \"Plan\": { } }\n",
output: "duration: 2007.111 ms plan:\n[redacted]\n",
},
{
filterLogSecret: "statement_parameter",
input: "2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:LOG: duration: 4079.697 ms execute <unnamed>: \nSELECT * FROM x WHERE y = $1 LIMIT $2\n2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:DETAIL: parameters: $1 = 'long string', $2 = '1', $3 = 'long string'\n",
output: "duration: 4079.697 ms execute <unnamed>: \nSELECT * FROM x WHERE y = $1 LIMIT $2\nparameters: $1 = '[redacted]', $2 = '[redacted]', $3 = '[redacted]'\n",
},
{
filterLogSecret: "statement_parameter",
input: "2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:LOG: duration: 2007.111 ms plan:\n{\"Query Text\": \"SELECT * FROM x WHERE y = $1 LIMIT $2\", \"Plan\": { } }\n2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:CONTEXT: unnamed portal with parameters: $1 = 'long string', $2 = '1', $3 = 'long string'\n",
output: "duration: 2007.111 ms plan:\n{\"Query Text\": \"SELECT * FROM x WHERE y = $1 LIMIT $2\", \"Plan\": { } }\nunnamed portal with parameters: $1 = '[redacted]', $2 = '[redacted]', $3 = '[redacted]'\n",
},
{
filterLogSecret: "none",
input: "2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:ERROR: division by zero\n2018-03-11 20:00:02 UTC:1.1.1.1(2):a@b:[3]:ERROR: Unknown Data\n",
Expand Down

0 comments on commit 7499212

Please sign in to comment.