From 697899b48bdf805aa717641e7a435a0b74d650d4 Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Wed, 19 Jul 2023 22:33:10 +0200 Subject: [PATCH] Add support for sending payload to event channel Closes: #1594 --- src/cfgutils.c | 11 ++-- src/ctl.c | 36 +++++----- src/ctl.h | 8 ++- src/ipc_resp.c | 4 +- src/report.c | 99 +++++++++++++++------------- src/scopetypes.h | 1 - src/state.c | 2 +- src/wrap.c | 2 +- test/integration/cli/test_inspect.sh | 4 +- test/integration/payload/scope-test | 13 ++-- 10 files changed, 101 insertions(+), 79 deletions(-) diff --git a/src/cfgutils.c b/src/cfgutils.c index de644dce7..2601842d3 100644 --- a/src/cfgutils.c +++ b/src/cfgutils.c @@ -2863,10 +2863,13 @@ initCtl(config_t *cfg) */ payload_status_t payloadStatus = PAYLOAD_STATUS_DISABLE; if (cfgPayEnable(cfg) || protocolDefinitionsUsePayloads()) { - if (cfgLogStreamEnable(cfg) && !payloadToDiskForced() ) { - payloadStatus = PAYLOAD_STATUS_CRIBL; - } else { - payloadStatus = PAYLOAD_STATUS_DISK; + payloadStatus = PAYLOAD_STATUS_DISK; + if (!payloadToDiskForced()) { + if (cfgLogStreamEnable(cfg)) { + payloadStatus = PAYLOAD_STATUS_CRIBL; + } else if (cfgEvtEnable(cfg)) { + payloadStatus = PAYLOAD_STATUS_CTL; + } } } if (payloadStatus == PAYLOAD_STATUS_CRIBL) { diff --git a/src/ctl.c b/src/ctl.c index bd15be49b..b3e10b225 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1232,33 +1232,35 @@ ctlEvtGet(ctl_t *ctl) return ctl ? ctl->evt : NULL; } -static transport_status_t -ctlPayConnectionStatus(ctl_t * ctl, payload_status_t payStatus) { +transport_status_t +ctlPayloadConnectionStatus(ctl_t *ctl) { + // retrieve the information about source + payload_status_t payStatus = ctlPayStatus(ctl); transport_status_t status = { .configString = NULL, .isConnected = FALSE, .connectAttemptCount = 0, .failureString = NULL}; - if ((!ctl) || (payStatus != PAYLOAD_STATUS_DISK)) { - return status; + switch (payStatus) { + case PAYLOAD_STATUS_DISABLE: + return status; + case PAYLOAD_STATUS_CRIBL: + return transportConnectionStatus(ctl->paytrans); + case PAYLOAD_STATUS_CTL: + return transportConnectionStatus(ctl->transport); + case PAYLOAD_STATUS_DISK: + status.configString = ctl->payload.dirRepr; + status.isConnected = TRUE; + return status; + default: + DBG(NULL); + return status; } - - status.configString = ctl->payload.dirRepr; - status.isConnected = TRUE; - - return status; } transport_status_t -ctlConnectionStatus(ctl_t *ctl, which_transport_t who) { - if (who == CFG_LS) { - payload_status_t status = ctlPayStatus(ctl); - if (status == PAYLOAD_STATUS_CRIBL) { - return transportConnectionStatus(ctl->paytrans); - } - return ctlPayConnectionStatus(ctl, status); - } +ctlConnectionStatus(ctl_t *ctl) { return transportConnectionStatus(ctl->transport); } diff --git a/src/ctl.h b/src/ctl.h index ed66e4803..fce3bc9aa 100644 --- a/src/ctl.h +++ b/src/ctl.h @@ -36,8 +36,9 @@ typedef enum { typedef enum { PAYLOAD_STATUS_DISABLE = 0, // payloads are disabled - PAYLOAD_STATUS_CRIBL = 1, // payloads are enabled and will go to cribl - PAYLOAD_STATUS_DISK = 2, // payloads are enabled and will go on disk + PAYLOAD_STATUS_CRIBL = 1, // payloads are enabled and will use cribl transport + PAYLOAD_STATUS_CTL = 2, // payloads are enabled and will use event transport + PAYLOAD_STATUS_DISK = 3, // payloads are enabled and will go on disk } payload_status_t; /** @@ -133,7 +134,8 @@ void ctlTransportSet(ctl_t *, transport_t *, which_transport_t); transport_t * ctlTransport(ctl_t *, which_transport_t); evt_fmt_t * ctlEvtGet(ctl_t *); void ctlEvtSet(ctl_t *, evt_fmt_t *); -transport_status_t ctlConnectionStatus(ctl_t *, which_transport_t); +transport_status_t ctlConnectionStatus(ctl_t *); +transport_status_t ctlPayloadConnectionStatus(ctl_t *); // Accessor for performance bool ctlEvtSourceEnabled(ctl_t *, watch_t); diff --git a/src/ipc_resp.c b/src/ipc_resp.c index 8dad11d61..91b5f5635 100644 --- a/src/ipc_resp.c +++ b/src/ipc_resp.c @@ -327,7 +327,7 @@ payloadTransportEnabled(void) { */ static transport_status_t payloadTransportStatus(void) { - return ctlConnectionStatus(g_ctl, CFG_LS); + return ctlPayloadConnectionStatus(g_ctl); } /* @@ -351,7 +351,7 @@ eventsTransportEnabled(void) { */ static transport_status_t eventsTransportStatus(void) { - return ctlConnectionStatus(g_ctl, CFG_CTL); + return ctlConnectionStatus(g_ctl); } /* diff --git a/src/report.c b/src/report.c index 8917fd8f0..c5cb7180d 100644 --- a/src/report.c +++ b/src/report.c @@ -3599,60 +3599,71 @@ doPayload() char *bdata = NULL; payload_status_t payStatus = ctlPayStatus(g_ctl); - if (payStatus == PAYLOAD_STATUS_CRIBL) { - bdata = scope_calloc(1, hlen + pinfo->len); - if (bdata) { - scope_memmove(bdata, pay, hlen); - scope_strncat(bdata, "\n", hlen); - scope_memmove(&bdata[hlen], pinfo->data, pinfo->len); - cmdSendPayload(g_ctl, bdata, hlen + pinfo->len); - } - } else if (payStatus == PAYLOAD_STATUS_DISK) { - int fd; - char path[PATH_MAX]; - - ///tmp//.in - switch (pinfo->src) { - case NETTX: - case TLSTX: - scope_snprintf(path, PATH_MAX, "%s/%d_%s:%s_%s:%s.out", - ctlPayDir(g_ctl), g_proc.pid, rip, rport, lip, lport); - break; - - case NETRX: - case TLSRX: - scope_snprintf(path, PATH_MAX, "%s/%d_%s:%s_%s:%s.in", - ctlPayDir(g_ctl), g_proc.pid, rip, rport, lip, lport); - break; - - default: - scope_snprintf(path, PATH_MAX, "%s/%d.na", - ctlPayDir(g_ctl), g_proc.pid); + switch (payStatus) { + case PAYLOAD_STATUS_CRIBL: + case PAYLOAD_STATUS_CTL: { + bdata = scope_calloc(1, hlen + pinfo->len); + if (bdata) { + scope_memmove(bdata, pay, hlen); + scope_strncat(bdata, "\n", hlen); + scope_memmove(&bdata[hlen], pinfo->data, pinfo->len); + cmdSendPayload(g_ctl, bdata, hlen + pinfo->len); + } break; } + case PAYLOAD_STATUS_DISK: { + int fd; + char path[PATH_MAX]; + + ///tmp//.in + switch (pinfo->src) { + case NETTX: + case TLSTX: + scope_snprintf(path, PATH_MAX, "%s/%d_%s:%s_%s:%s.out", + ctlPayDir(g_ctl), g_proc.pid, rip, rport, lip, lport); + break; - if ((fd = scope_open(path, O_WRONLY | O_CREAT | O_APPEND, 0666)) != -1) { - if (checkEnv("SCOPE_PAYLOAD_HEADER", "true")) { - scope_write(fd, pay, rc); + case NETRX: + case TLSRX: + scope_snprintf(path, PATH_MAX, "%s/%d_%s:%s_%s:%s.in", + ctlPayDir(g_ctl), g_proc.pid, rip, rport, lip, lport); + break; + + default: + scope_snprintf(path, PATH_MAX, "%s/%d.na", + ctlPayDir(g_ctl), g_proc.pid); + break; } - size_t to_write = pinfo->len; - size_t written = 0; - int rc; + if ((fd = scope_open(path, O_WRONLY | O_CREAT | O_APPEND, 0666)) != -1) { + if (checkEnv("SCOPE_PAYLOAD_HEADER", "true")) { + scope_write(fd, pay, rc); + } - while (to_write > 0) { - rc = scope_write(fd, &pinfo->data[written], to_write); - if (rc <= 0) { - DBG(NULL); - break; + size_t to_write = pinfo->len; + size_t written = 0; + int rc; + + while (to_write > 0) { + rc = scope_write(fd, &pinfo->data[written], to_write); + if (rc <= 0) { + DBG(NULL); + break; + } + + written += rc; + to_write -= rc; } - written += rc; - to_write -= rc; + scope_close(fd); } - - scope_close(fd); + break; } + case PAYLOAD_STATUS_DISABLE: + break; + default: + DBG(NULL); + break; } if (bdata) scope_free(bdata); diff --git a/src/scopetypes.h b/src/scopetypes.h index c808aa9e2..b5463c724 100644 --- a/src/scopetypes.h +++ b/src/scopetypes.h @@ -229,7 +229,6 @@ typedef struct // CRIBL_EDGE_FS_ROOT define the location of the host root path inside the Cribl Edge container #define SCOPE_PID_ENV "SCOPE_PID" #define PRESERVE_PERF_REPORTING "SCOPE_PERF_PRESERVE" -#define SCOPE_PAYLOAD_TO_DISK_ENV "SCOPE_PAYLOAD_TO_DISK" // TLS protocol refs that have been useful: // https://tools.ietf.org/html/rfc5246 diff --git a/src/state.c b/src/state.c index 6062ddc77..e86094bb2 100644 --- a/src/state.c +++ b/src/state.c @@ -295,7 +295,7 @@ initState(void) initMetricCapture(); // Some environment variables we don't want to continuously check - setPayloadToDiskForced(checkEnv(SCOPE_PAYLOAD_TO_DISK_ENV, "true")); + setPayloadToDiskForced(checkEnv("SCOPE_PAYLOAD_TO_DISK", "true")); // the http guard array is static while the net fs array is dynamically allocated // will need to change if we want to re-size at runtime diff --git a/src/wrap.c b/src/wrap.c index bb962aa40..5b14f2863 100644 --- a/src/wrap.c +++ b/src/wrap.c @@ -1256,7 +1256,7 @@ periodic(void *arg) summaryTime = tv.tv_sec + g_thread.interval; if (tv.tv_sec >= logReportTime) { - transport_status_t ctlStatus = ctlConnectionStatus(g_ctl, CFG_CTL); + transport_status_t ctlStatus = ctlConnectionStatus(g_ctl); logOurConnectionStatus(ctlStatus, "event"); transport_status_t mtcStatus = mtcConnectionStatus(g_mtc); logOurConnectionStatus(mtcStatus, "metric"); diff --git a/test/integration/cli/test_inspect.sh b/test/integration/cli/test_inspect.sh index 9d6d36423..fad405394 100755 --- a/test/integration/cli/test_inspect.sh +++ b/test/integration/cli/test_inspect.sh @@ -214,7 +214,7 @@ endtest # disabled cribl, enabled payloads enabled (via env var) expected to see log, events, metrics, payload starttest test_inspect_cribl_disable_payload_enable_via_env -SCOPE_CRIBL_ENABLE=false SCOPE_PAYLOAD_ENABLE=true LD_PRELOAD=/usr/local/scope/lib/libscope.so python3 -m http.server 1> /dev/null 2> /dev/null & +SCOPE_CRIBL_ENABLE=false SCOPE_PAYLOAD_TO_DISK=true SCOPE_PAYLOAD_ENABLE=true LD_PRELOAD=/usr/local/scope/lib/libscope.so python3 -m http.server 1> /dev/null 2> /dev/null & sleep 2 PYTHON_PID=`pidof python3` @@ -236,7 +236,7 @@ endtest # in config disabled cribl payloads enabled (via protocol list) expected to see log, events, metrics, payload starttest test_inspect_cribl_disable_payload_enable_via_config -SCOPE_CONF_PATH=/opt/test/bin/payload_conf.yml LD_PRELOAD=/usr/local/scope/lib/libscope.so python3 -m http.server 1> /dev/null 2> /dev/null & +SCOPE_PAYLOAD_TO_DISK=true SCOPE_CONF_PATH=/opt/test/bin/payload_conf.yml LD_PRELOAD=/usr/local/scope/lib/libscope.so python3 -m http.server 1> /dev/null 2> /dev/null & sleep 2 PYTHON_PID=`pidof python3` diff --git a/test/integration/payload/scope-test b/test/integration/payload/scope-test index 7f173e071..fb65b006d 100755 --- a/test/integration/payload/scope-test +++ b/test/integration/payload/scope-test @@ -124,27 +124,31 @@ FILE_COUNT=$(compgen -G "$SCOPE_PAYLOAD_DIR/*\.out" | wc -l) if (( $FILE_COUNT == 0 )); then fail "FAILED - expected payload files, but found $FILE_COUNT" fi +cleanup endtest starttest payload_on_but_not_to_cribl_1 -echo "When the payload feature is on, and SCOPE_CRIBL_ENABLE=false, verify" +echo "When the payload feature is on, SCOPE_EVENT_ENABLE=false and SCOPE_CRIBL_ENABLE=false, verify" echo " payloads are written to disk. Payloads always go to disk if" -echo " SCOPE_CRIBL_ENABLE=false" +echo " SCOPE_CRIBL_ENABLE=false and SCOPE_EVENT_ENABLE=false" SCOPE_PAYLOAD_ENABLE=true \ SCOPE_CRIBL_ENABLE=false \ +SCOPE_EVENT_ENABLE=false \ scope run -- curl -Lso /dev/null https://cribl.io FILE_COUNT=$(compgen -G "$SCOPE_PAYLOAD_DIR/*\.out" | wc -l) if (( $FILE_COUNT == 0 )); then fail "FAILED - expected payload files, but found $FILE_COUNT" fi +cleanup endtest starttest payload_on_but_not_to_cribl_2 -echo "When the payload feature is on, and SCOPE_CRIBL_ENABLE=false, verify" +echo "When the payload feature is on, and SCOPE_EVENT_ENABLE=false and SCOPE_CRIBL_ENABLE=false, verify" echo " payloads are written to disk even if SCOPE_PAYLOAD_TO_DISK=false." -echo " Payloads always go to disk if SCOPE_CRIBL_ENABLE=false" +echo " Payloads always go to disk if SCOPE_CRIBL_ENABLE=false and SCOPE_EVENT_ENABLE=false" SCOPE_PAYLOAD_ENABLE=true \ SCOPE_CRIBL_ENABLE=false \ +SCOPE_EVENT_ENABLE=false \ SCOPE_PAYLOAD_TO_DISK=false \ scope run -- curl -Lso /dev/null https://cribl.io FILE_COUNT=$(compgen -G "$SCOPE_PAYLOAD_DIR/*\.out" | wc -l) @@ -159,6 +163,7 @@ endtest starttest payload_host_app SCOPE_PAYLOAD_ENABLE=true \ SCOPE_CRIBL_ENABLE=false \ +SCOPE_PAYLOAD_TO_DISK=true \ scope run -- host -t a cribl.io in_count=$(get_file_count_with_extension $SCOPE_PAYLOAD_DIR "in") out_count=$(get_file_count_with_extension $SCOPE_PAYLOAD_DIR "out")