diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 0a2057875b..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -default: - image: docker-hub.repo.splunkdev.net/python:3.11 - -variables: - EXTRACTO_VERSION: - value: "1.0.60" - description: "The version of extracto to use" - SKIP_DOWNSTREAM_TESTING: - value: "False" - description: "If true, downstream testing will be suppressed (useful for debugging or forcing a release in an emergency)." - ENABLE_INTEGRATION_TESTING: - value: "True" - description: "Flag indicating that integration testing should be performed. Defaults to True, may be suppressed in some workflows." - -stages: - - build - - app_inspect - - test - - release - -include: - - local: "pipeline/.build.yml" - - local: "pipeline/.app-inspect.yml" - - local: "pipeline/.test.yml" - - local: "pipeline/.release.yml" - - local: "pipeline/.post.yml" - -workflow: - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - - if: $CI_COMMIT_TAG - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_COMMIT_BRANCH =~ /^release_v[0-9]+\.[0-9]+\.[0-9]+$/ diff --git a/contentctl.yml b/contentctl.yml index 93279c5acc..4a3adfa19e 100644 --- a/contentctl.yml +++ b/contentctl.yml @@ -3,7 +3,7 @@ app: uid: 3449 title: ES Content Updates appid: DA-ESS-ContentUpdate - version: 4.35.0 + version: 4.36.0 description: Explore the Analytic Stories included with ES Content Updates. prefix: ESCU label: ESCU diff --git a/docs/spec/README.md b/docs/spec/README.md deleted file mode 100644 index 9f2e056264..0000000000 --- a/docs/spec/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# README - -## Top-level Schemas - -* [Analytics Story Schema](./stories.md "schema analytics story") – `http://example.com/example.json` - -* [Baseline Schema](./baselines.md "schema for baselines") – `http://example.com/example.json` - -* [Deployment Schema](./deployments.md "schema for deployment") – `http://example.com/example.json` - -* [Detection Schema](./detections.md "schema for detections") – `http://example.com/example.json` - -* [Lookup Manifest](./lookups.md "A object that defines a lookup file and its properties") – `https://api.splunkresearch.com/schemas/lookups.json` - -* [Macro Manifest](./macros.md "An object that defines the parameters for a Splunk Macro") – `https://api.splunkresearch.com/schemas/macros.json` - -* [Response Schema](./response_tasks.md "schema for response task") – `https://raw.githubusercontent.com/splunk/security_content/develop/docs/spec/response_tasks.spec.json` - -* [Response Schema](./responses.md "schema for response") – `https://raw.githubusercontent.com/splunk/security_content/develop/docs/spec/response.spec.json` - -* [Response Schema](./responses_phase.md "schema for phase") – `http://example.com/example.json` - -## Other Schemas - -### Objects - -* [Untitled object in Baseline Schema](./baselines-properties-tags.md "An array of key value pairs for tagging") – `#/properties/tags#/properties/tags` - -* [Untitled object in Deployment Schema](./deployments-properties-alert_action.md "Set alert action parameter for search") – `#/properties/alert_action#/properties/alert_action` - -* [Untitled object in Deployment Schema](./deployments-properties-alert_action-properties-email.md "By enabling it, an email is sent with the results") – `#/properties/alert_action/properties/email#/properties/alert_action/properties/email` - -* [Untitled object in Deployment Schema](./deployments-properties-alert_action-properties-index.md "By enabling it, the results are stored in another index") – `#/properties/alert_action/properties/index#/properties/alert_action/properties/index` - -* [Untitled object in Deployment Schema](./deployments-properties-alert_action-properties-notable.md "By enabling it, a notable is generated") – `#/properties/alert_action/properties/notable#/properties/alert_action/properties/notable` - -* [Untitled object in Deployment Schema](./deployments-properties-scheduling.md "allows to set scheduling parameter") – `#/properties/scheduling#/properties/scheduling` - -* [Untitled object in Response Schema](./response_tasks-properties-automation.md "An array of key value pairs for defining actions and playbooks") – `#/properties/automation#/properties/automation` - -### Arrays - -* [Untitled array in Baseline Schema](./baselines-properties-datamodel.md "datamodel used in the search") – `#/properties/datamodel#/properties/datamodel` - -* [Untitled array in Detection Schema](./detections-properties-references.md "A list of references for this detection") – `#/properties/references#/properties/references` - -* [Untitled array in Macro Manifest](./macros-properties-arguments.md "A list of the arguments being passed to this macro") – `https://api.splunkresearch.com/schemas/macros.json#/properties/arguments` - -* [Untitled array in Response Schema](./responses-properties-response_phase.md "Response divided into phases") – `#/properties/response_phases#/properties/response_phase` - -* [Untitled array in Response Schema](./responses_phase-properties-response_task.md "Response phase is divided into task(s) to be completed") – `#/properties/response_task#/properties/response_task` diff --git a/docs/spec/deployments.md b/docs/spec/deployments.md deleted file mode 100644 index d77cb1e62f..0000000000 --- a/docs/spec/deployments.md +++ /dev/null @@ -1,258 +0,0 @@ -# Deployment Schema Schema - -```txt -http://example.com/example.json -``` - -schema for deployment - -| Abstract | Extensible | Status | Identifiable | Custom Properties | Additional Properties | Access Restrictions | Defined In | -| :------------------ | :--------- | :------------- | :----------- | :---------------- | :-------------------- | :------------------ | :------------------------------------------------------------------------------- | -| Can be instantiated | No | Unknown status | No | Forbidden | Allowed | none | [deployments.spec.json](../../spec/deployments.spec.json "open original schema") | - -## Deployment Schema Type - -`object` ([Deployment Schema](deployments.md)) - -## Deployment Schema Default Value - -The default value is: - -```json -{} -``` - -# Deployment Schema Properties - -| Property | Type | Required | Nullable | Defined by | -| :---------------------------- | :------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------------- | -| [alert_action](#alert_action) | `object` | Optional | cannot be null | [Deployment Schema](deployments-properties-alert_action.md "#/properties/alert_action#/properties/alert_action") | -| [date](#date) | `string` | Required | cannot be null | [Deployment Schema](deployments-properties-date.md "#/properties/date#/properties/date") | -| [description](#description) | `string` | Required | cannot be null | [Deployment Schema](deployments-properties-description.md "#/properties/description#/properties/description") | -| [id](#id) | `string` | Required | cannot be null | [Deployment Schema](deployments-properties-id.md "#/properties/id#/properties/id") | -| [name](#name) | `string` | Required | cannot be null | [Deployment Schema](deployments-properties-name.md "#/properties/name#/properties/name") | -| [scheduling](#scheduling) | `object` | Required | cannot be null | [Deployment Schema](deployments-properties-scheduling.md "#/properties/scheduling#/properties/scheduling") | -| [tags](#tags) | `object` | Required | cannot be null | [Deployment Schema](deployments-properties-tags.md "#/properties/tags#/properties/tags") | -| Additional Properties | Any | Optional | can be null | | - -## alert_action - -Set alert action parameter for search - -`alert_action` - -* is optional - -* Type: `object` ([Details](deployments-properties-alert_action.md)) - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-alert_action.md "#/properties/alert_action#/properties/alert_action") - -### alert_action Type - -`object` ([Details](deployments-properties-alert_action.md)) - -### alert_action Default Value - -The default value is: - -```json -{} -``` - -### alert_action Examples - -```yaml -email: - message: Splunk Alert $name$ triggered %fields% - subject: Splunk Alert $name$ - to: test@test.com -index: - name: asx -notable: - rule_description: '%description%' - rule_title: '%name%' - -``` - -## date - -date of creation or modification, format yyyy-mm-dd - -`date` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-date.md "#/properties/date#/properties/date") - -### date Type - -`string` - -### date Examples - -```yaml -'2019-12-06' - -``` - -## description - -description of the deployment configuration - -`description` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-description.md "#/properties/description#/properties/description") - -### description Type - -`string` - -### description Examples - -```yaml ->- - This deployment configuration provides a standard scheduling policy over all - rules. - -``` - -## id - -uuid as unique identifier - -`id` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-id.md "#/properties/id#/properties/id") - -### id Type - -`string` - -### id Examples - -```yaml -fb4c31b0-13e8-4155-8aa5-24de4b8d6717 - -``` - -## name - -Name of deployment configuration - -`name` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-name.md "#/properties/name#/properties/name") - -### name Type - -`string` - -### name Examples - -```yaml -Deployment Configuration all Detections - -``` - -## scheduling - -allows to set scheduling parameter - -`scheduling` - -* is required - -* Type: `object` ([Details](deployments-properties-scheduling.md)) - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-scheduling.md "#/properties/scheduling#/properties/scheduling") - -### scheduling Type - -`object` ([Details](deployments-properties-scheduling.md)) - -### scheduling Default Value - -The default value is: - -```json -{} -``` - -### scheduling Examples - -```yaml -cron_schedule: '*/10 * * * *' -earliest_time: '-10m' -latest_time: now -schedule_window: auto - -``` - -## tags - -An array of key value pairs for tagging - -`tags` - -* is required - -* Type: `object` ([Details](deployments-properties-tags.md)) - -* cannot be null - -* defined in: [Deployment Schema](deployments-properties-tags.md "#/properties/tags#/properties/tags") - -### tags Type - -`object` ([Details](deployments-properties-tags.md)) - -### tags Constraints - -**minimum number of items**: the minimum number of items for this array is: `1` - -**unique items**: all items in this array must be unique. Duplicates are not allowed. - -### tags Default Value - -The default value is: - -```json -{} -``` - -### tags Examples - -```yaml -analytic_story: credential_dumping - -``` - -## Additional Properties - -Additional properties are allowed and do not have to follow a specific schema diff --git a/docs/spec/detections.md b/docs/spec/detections.md deleted file mode 100644 index e37aa83875..0000000000 --- a/docs/spec/detections.md +++ /dev/null @@ -1,410 +0,0 @@ -# Detection Schema Schema - -```txt -http://example.com/example.json -``` - -schema for detections - -| Abstract | Extensible | Status | Identifiable | Custom Properties | Additional Properties | Access Restrictions | Defined In | -| :------------------ | :--------- | :------------- | :----------- | :---------------- | :-------------------- | :------------------ | :----------------------------------------------------------------------------- | -| Can be instantiated | No | Unknown status | No | Forbidden | Allowed | none | [detections.spec.json](../../spec/detections.spec.json "open original schema") | - -## Detection Schema Type - -`object` ([Detection Schema](detections.md)) - -# Detection Schema Properties - -| Property | Type | Required | Nullable | Defined by | -| :---------------------------------------------- | :-------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | -| [author](#author) | `string` | Required | cannot be null | [Detection Schema](detections-properties-author.md "#/properties/author#/properties/author") | -| [date](#date) | `string` | Required | cannot be null | [Detection Schema](detections-properties-date.md "#/properties/date#/properties/date") | -| [description](#description) | `string` | Required | cannot be null | [Detection Schema](detections-properties-description.md "#/properties/description#/properties/description") | -| [how_to_implement](#how_to_implement) | `string` | Optional | cannot be null | [Detection Schema](detections-properties-how_to_implement.md "#/properties/how_to_implement#/properties/how_to_implement") | -| [id](#id) | `string` | Required | cannot be null | [Detection Schema](detections-properties-id.md "#/properties/id#/properties/id") | -| [known_false_positives](#known_false_positives) | `string` | Required | cannot be null | [Detection Schema](detections-properties-known_false_positives.md "#/properties/knwon_false_positives#/properties/known_false_positives") | -| [name](#name) | `string` | Required | cannot be null | [Detection Schema](detections-properties-name-of-detection.md "#/properties/name#/properties/name") | -| [references](#references) | `array` | Optional | cannot be null | [Detection Schema](detections-properties-references.md "#/properties/references#/properties/references") | -| [search](#search) | `string` | Required | cannot be null | [Detection Schema](detections-properties-search.md "#/properties/search#/properties/search") | -| [tags](#tags) | `object` | Required | cannot be null | [Detection Schema](detections-properties-tags.md "#/properties/tags#/properties/tags") | -| [type](#type) | `string` | Required | cannot be null | [Detection Schema](detections-properties-type.md "#/properties/type#/properties/type") | -| [datamodel](#datamodel) | `array` | Optional | cannot be null | [Detection Schema](detections-properties-datamodel.md "#/properties/datamodel#/properties/datamodel") | -| [version](#version) | `integer` | Required | cannot be null | [Detection Schema](detections-properties-version.md "#/properties/version#/properties/version") | -| Additional Properties | Any | Optional | can be null | | - -## author - -Author of the detection - -`author` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-author.md "#/properties/author#/properties/author") - -### author Type - -`string` - -### author Examples - -```yaml -Patrick Bareiss, Splunk - -``` - -## date - -date of creation or modification, format yyyy-mm-dd - -`date` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-date.md "#/properties/date#/properties/date") - -### date Type - -`string` - -### date Examples - -```yaml -'2019-12-06' - -``` - -## description - -A detailed description of the detection - -`description` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-description.md "#/properties/description#/properties/description") - -### description Type - -`string` - -### description Examples - -```yaml ->- - dbgcore.dll is a specifc DLL for Windows core debugging. It is used to obtain - a memory dump of a process. This search detects the usage of this DLL for - creating a memory dump of LSASS process. Memory dumps of the LSASS process can - be created with tools such as Windows Task Manager or procdump. - -``` - -## how_to_implement - -information about how to implement. Only needed for non standard implementations. - -`how_to_implement` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-how_to_implement.md "#/properties/how_to_implement#/properties/how_to_implement") - -### how_to_implement Type - -`string` - -### how_to_implement Examples - -```yaml ->- - This search requires Sysmon Logs and a Sysmon configuration, which includes - EventCode 10 for lsass.exe. - -``` - -## id - -UUID as unique identifier - -`id` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-id.md "#/properties/id#/properties/id") - -### id Type - -`string` - -### id Examples - -```yaml -fb4c31b0-13e8-4155-8aa5-24de4b8d6717 - -``` - -## known_false_positives - -known false postives - -`known_false_positives` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-known_false_positives.md "#/properties/knwon_false_positives#/properties/known_false_positives") - -### known_false_positives Type - -`string` - -### known_false_positives Examples - -```yaml ->- - Administrators can create memory dumps for debugging purposes, but memory - dumps of the LSASS process would be unusual. - -``` - -## name - - - -`name` - -* is required - -* Type: `string` ([Name of detection](detections-properties-name-of-detection.md)) - -* cannot be null - -* defined in: [Detection Schema](detections-properties-name-of-detection.md "#/properties/name#/properties/name") - -### name Type - -`string` ([Name of detection](detections-properties-name-of-detection.md)) - -### name Examples - -```yaml -Access LSASS Memory for Dump Creation - -``` - -## references - -A list of references for this detection - -`references` - -* is optional - -* Type: `string[]` ([The Items Schema](detections-properties-references-the-items-schema.md)) - -* cannot be null - -* defined in: [Detection Schema](detections-properties-references.md "#/properties/references#/properties/references") - -### references Type - -`string[]` ([The Items Schema](detections-properties-references-the-items-schema.md)) - -### references Default Value - -The default value is: - -```json -[] -``` - -### references Examples - -```yaml -- >- - https://2017.zeronights.org/wp-content/uploads/materials/ZN17_Kheirkhabarov_Hunting_for_Credentials_Dumping_in_Windows_Environment.pdf - -``` - -## search - -The Splunk search for the detection - -`search` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-search.md "#/properties/search#/properties/search") - -### search Type - -`string` - -### search Examples - -```yaml ->- - `sysmon` EventCode=10 TargetImage=*lsass.exe CallTrace=*dbgcore.dll* OR - CallTrace=*dbghelp.dll* | stats count min(_time) as firstTime max(_time) as - lastTime by Computer, TargetImage, TargetProcessId, SourceImage, - SourceProcessId | rename Computer as dest | - `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | - `access_lsass_memory_for_dump_creation_filter` - -``` - -## tags - -An array of key value pairs for tagging - -`tags` - -* is required - -* Type: `object` ([Details](detections-properties-tags.md)) - -* cannot be null - -* defined in: [Detection Schema](detections-properties-tags.md "#/properties/tags#/properties/tags") - -### tags Type - -`object` ([Details](detections-properties-tags.md)) - -### tags Constraints - -**minimum number of items**: the minimum number of items for this array is: `1` - -**unique items**: all items in this array must be unique. Duplicates are not allowed. - -### tags Default Value - -The default value is: - -```json -{} -``` - -### tags Examples - -```yaml -analytic_story: credential_dumping -kill_chain_phases: Action on Objectives -mitre_attack_id: T1078.004 -cis20: CIS 13 -nist: DE.DP -security domain: network -asset_type: AWS Instance -risk_object: user -risk_object_type: network_artifacts -risk score: '60' -custom_key: custom_value - -``` - -## type - -type of detection - -`type` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-type.md "#/properties/type#/properties/type") - -### type Type - -`string` - -### type Examples - -```yaml -streaming - -``` - -## datamodel - -datamodel used in the search - -`datamodel` - -* is optional - -* Type: `string[]` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-datamodel.md "#/properties/datamodel#/properties/datamodel") - -### datamodel Type - -`string[]` - -### datamodel Examples - -```yaml -Endpoint - -``` - -## version - -version of detection, e.g. 1 or 2 ... - -`version` - -* is required - -* Type: `integer` - -* cannot be null - -* defined in: [Detection Schema](detections-properties-version.md "#/properties/version#/properties/version") - -### version Type - -`integer` - -### version Examples - -```yaml -2 - -``` - -## Additional Properties - -Additional properties are allowed and do not have to follow a specific schema diff --git a/docs/spec/lookups.md b/docs/spec/lookups.md deleted file mode 100644 index d6a8ec5141..0000000000 --- a/docs/spec/lookups.md +++ /dev/null @@ -1,321 +0,0 @@ -# Lookup Manifest Schema - -```txt -https://api.splunkresearch.com/schemas/lookups.json -``` - -A object that defines a lookup file and its properties. - -| Abstract | Extensible | Status | Identifiable | Custom Properties | Additional Properties | Access Restrictions | Defined In | -| :------------------ | :--------- | :------------- | :----------- | :---------------- | :-------------------- | :------------------ | :----------------------------------------------------------------------- | -| Can be instantiated | No | Unknown status | No | Forbidden | Allowed | none | [lookups.spec.json](../../spec/lookups.spec.json "open original schema") | - -## Lookup Manifest Type - -`object` ([Lookup Manifest](lookups.md)) - -one (and only one) of - -* [Untitled undefined type in Lookup Manifest](lookups-oneof-0.md "check type definition") - -* [Untitled undefined type in Lookup Manifest](lookups-oneof-1.md "check type definition") - -# Lookup Manifest Properties - -| Property | Type | Required | Nullable | Defined by | -| :-------------------------------------------- | :-------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | -| [case_sensitive_match](#case_sensitive_match) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-case_sensitive_match.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/case_sensitive_match") | -| [collection](#collection) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-collection.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/collection") | -| [default_match](#default_match) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-default_match.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/default_match") | -| [description](#description) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-description.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/description") | -| [fields_list](#fields_list) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-fields_list.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/fields_list") | -| [filename](#filename) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-filename.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/filename") | -| [filter](#filter) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-filter.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/filter") | -| [match_type](#match_type) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-match_type.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/match_type") | -| [max_matches](#max_matches) | `integer` | Optional | cannot be null | [Lookup Manifest](lookups-properties-max_matches.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/max_matches") | -| [min_matches](#min_matches) | `integer` | Optional | cannot be null | [Lookup Manifest](lookups-properties-min_matches.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/min_matches") | -| [name](#name) | `string` | Optional | cannot be null | [Lookup Manifest](lookups-properties-name.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/name") | - -## case_sensitive_match - -What the macro is intended to filter - -`case_sensitive_match` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-case_sensitive_match.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/case_sensitive_match") - -### case_sensitive_match Type - -`string` - -### case_sensitive_match Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :-------- | :---------- | -| `"true"` | | -| `"false"` | | - -### case_sensitive_match Examples - -```yaml -'true' - -``` - -## collection - -Name of the collection to use for this lookup - -`collection` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-collection.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/collection") - -### collection Type - -`string` - -### collection Examples - -```yaml -prohibited_apps_launching_cmd - -``` - -## default_match - -The default value if no match is found - -`default_match` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-default_match.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/default_match") - -### default_match Type - -`string` - -### default_match Examples - -```yaml -'true' - -``` - -## description - -The description of this lookup - -`description` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-description.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/description") - -### description Type - -`string` - -### description Examples - -```yaml -This lookup contains file names that exist in the Windows\System32 directory - -``` - -## fields_list - -A comma and space separated list of field names - -`fields_list` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-fields_list.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/fields_list") - -### fields_list Type - -`string` - -### fields_list Examples - -```yaml -_key, dest, process_name - -``` - -## filename - -The name of the file to use for this lookup - -`filename` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-filename.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/filename") - -### filename Type - -`string` - -### filename Examples - -```yaml -prohibited_apps_launching_cmd.csv - -``` - -## filter - -Use this attribute to improve search performance when working with significantly large KV - -`filter` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-filter.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/filter") - -### filter Type - -`string` - -### filter Examples - -```yaml -dest="SPLK_*" - -``` - -## match_type - -A comma and space-delimited list of \(\) specification to allow for non-exact matching - -`match_type` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-match_type.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/match_type") - -### match_type Type - -`string` - -### match_type Examples - -```yaml -WILDCARD(process) - -``` - -## max_matches - -The maximum number of possible matches for each input lookup value - -`max_matches` - -* is optional - -* Type: `integer` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-max_matches.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/max_matches") - -### max_matches Type - -`integer` - -### max_matches Examples - -```yaml -'100' - -``` - -## min_matches - -Minimum number of possible matches for each input lookup value - -`min_matches` - -* is optional - -* Type: `integer` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-min_matches.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/min_matches") - -### min_matches Type - -`integer` - -### min_matches Examples - -```yaml -'1' - -``` - -## name - -The name of the lookup to be used in searches - -`name` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Lookup Manifest](lookups-properties-name.md "https://api.splunkresearch.com/schemas/lookups.json#/properties/name") - -### name Type - -`string` - -### name Examples - -```yaml -isWindowsSystemFile_lookup - -``` diff --git a/docs/spec/macros.md b/docs/spec/macros.md deleted file mode 100644 index 3047f816f2..0000000000 --- a/docs/spec/macros.md +++ /dev/null @@ -1,123 +0,0 @@ -# Macro Manifest Schema - -```txt -https://api.splunkresearch.com/schemas/macros.json -``` - -An object that defines the parameters for a Splunk Macro - -| Abstract | Extensible | Status | Identifiable | Custom Properties | Additional Properties | Access Restrictions | Defined In | -| :------------------ | :--------- | :------------- | :----------- | :---------------- | :-------------------- | :------------------ | :--------------------------------------------------------------------- | -| Can be instantiated | No | Unknown status | No | Forbidden | Allowed | none | [macros.spec.json](../../spec/macros.spec.json "open original schema") | - -## Macro Manifest Type - -`object` ([Macro Manifest](macros.md)) - -# Macro Manifest Properties - -| Property | Type | Required | Nullable | Defined by | -| :-------------------------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------ | -| [arguments](#arguments) | `array` | Optional | cannot be null | [Macro Manifest](macros-properties-arguments.md "https://api.splunkresearch.com/schemas/macros.json#/properties/arguments") | -| [definition](#definition) | `string` | Optional | cannot be null | [Macro Manifest](macros-properties-definition.md "https://api.splunkresearch.com/schemas/macros.json#/properties/definition") | -| [description](#description) | `string` | Required | cannot be null | [Macro Manifest](macros-properties-description.md "https://api.splunkresearch.com/schemas/macros.json#/properties/description") | -| [name](#name) | `string` | Required | cannot be null | [Macro Manifest](macros-properties-name.md "https://api.splunkresearch.com/schemas/macros.json#/properties/name") | - -## arguments - -A list of the arguments being passed to this macro - -`arguments` - -* is optional - -* Type: `string[]` - -* cannot be null - -* defined in: [Macro Manifest](macros-properties-arguments.md "https://api.splunkresearch.com/schemas/macros.json#/properties/arguments") - -### arguments Type - -`string[]` - -### arguments Constraints - -**minimum number of items**: the minimum number of items for this array is: `0` - -**unique items**: all items in this array must be unique. Duplicates are not allowed. - -## definition - -The macro definition - -`definition` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Macro Manifest](macros-properties-definition.md "https://api.splunkresearch.com/schemas/macros.json#/properties/definition") - -### definition Type - -`string` - -### definition Examples - -```yaml -(query=fls-na* AND query = www* AND query=images*) - -``` - -## description - -What the macro is intended to filter - -`description` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Macro Manifest](macros-properties-description.md "https://api.splunkresearch.com/schemas/macros.json#/properties/description") - -### description Type - -`string` - -### description Examples - -```yaml -Use this macro to filter out known good objects - -``` - -## name - -The name of the macro - -`name` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Macro Manifest](macros-properties-name.md "https://api.splunkresearch.com/schemas/macros.json#/properties/name") - -### name Type - -`string` - -### name Examples - -```yaml -detection_search_output_filter - -``` diff --git a/docs/spec/stories.md b/docs/spec/stories.md deleted file mode 100644 index 9e37724776..0000000000 --- a/docs/spec/stories.md +++ /dev/null @@ -1,287 +0,0 @@ -# Analytics Story Schema Schema - -```txt -http://example.com/example.json -``` - -schema analytics story - -| Abstract | Extensible | Status | Identifiable | Custom Properties | Additional Properties | Access Restrictions | Defined In | -| :------------------ | :--------- | :------------- | :----------- | :---------------- | :-------------------- | :------------------ | :----------------------------------------------------------------------- | -| Can be instantiated | No | Unknown status | No | Forbidden | Allowed | none | [stories.spec.json](../../spec/stories.spec.json "open original schema") | - -## Analytics Story Schema Type - -`object` ([Analytics Story Schema](stories.md)) - -## Analytics Story Schema Default Value - -The default value is: - -```json -{} -``` - -# Analytics Story Schema Properties - -| Property | Type | Required | Nullable | Defined by | -| :-------------------------- | :-------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------- | -| [author](#author) | `string` | Required | cannot be null | [Analytics Story Schema](stories-properties-author.md "#/properties/author#/properties/author") | -| [date](#date) | `string` | Required | cannot be null | [Analytics Story Schema](stories-properties-date.md "#/properties/date#/properties/date") | -| [description](#description) | `string` | Required | cannot be null | [Analytics Story Schema](stories-properties-description.md "#/properties/description#/properties/description") | -| [id](#id) | `string` | Required | cannot be null | [Analytics Story Schema](stories-properties-id.md "#/properties/id#/properties/id") | -| [name](#name) | `string` | Required | cannot be null | [Analytics Story Schema](stories-properties-name.md "#/properties/name#/properties/name") | -| [narrative](#narrative) | `string` | Required | cannot be null | [Analytics Story Schema](stories-properties-narrative.md "#/properties/narrative#/properties/narrative") | -| [search](#search) | `string` | Optional | cannot be null | [Analytics Story Schema](stories-properties-search.md "#/properties/search#/properties/search") | -| [tags](#tags) | `object` | Required | cannot be null | [Analytics Story Schema](stories-properties-tags.md "#/properties/tags#/properties/tags") | -| [version](#version) | `integer` | Required | cannot be null | [Analytics Story Schema](stories-properties-version.md "#/properties/version#/properties/version") | -| Additional Properties | Any | Optional | can be null | | - -## author - -Author of the analytics story - -`author` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-author.md "#/properties/author#/properties/author") - -### author Type - -`string` - -### author Examples - -```yaml -Rico Valdez, Patrick Bareiß, Splunk - -``` - -## date - -date of creation or modification, format yyyy-mm-dd - -`date` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-date.md "#/properties/date#/properties/date") - -### date Type - -`string` - -### date Examples - -```yaml -'2019-12-06' - -``` - -## description - -description of the analytics story - -`description` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-description.md "#/properties/description#/properties/description") - -### description Type - -`string` - -### description Examples - -```yaml ->- - Uncover activity consistent with credential dumping, a technique where - attackers compromise systems and attempt to obtain and exfiltrate passwords. - -``` - -## id - -UUID as unique identifier - -`id` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-id.md "#/properties/id#/properties/id") - -### id Type - -`string` - -### id Examples - -```yaml -fb4c31b0-13e8-4155-8aa5-24de4b8d6717 - -``` - -## name - -Name of the Analytics Story - -`name` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-name.md "#/properties/name#/properties/name") - -### name Type - -`string` - -### name Examples - -```yaml -Credential Dumping - -``` - -## narrative - -narrative of the analytics story - -`narrative` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-narrative.md "#/properties/narrative#/properties/narrative") - -### narrative Type - -`string` - -### narrative Examples - -```yaml ->- - gathering credentials from a target system, often hashed or encrypted, is a - common attack technique. Even though the credentials may not be in plain text, - an attacker can still exfiltrate the data and set to cracking it offline, on - their own systems. - -``` - -## search - -An additional Splunk search, which uses the result of the detections - -`search` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-search.md "#/properties/search#/properties/search") - -### search Type - -`string` - -### search Examples - -```yaml ->- - index=asx mitre_id=t1003 | stats values(source) as detections values(process) - as processes values(user) as users values(_time) as time count by dest - -``` - -## tags - -An explanation about the purpose of this instance. - -`tags` - -* is required - -* Type: `object` ([Details](stories-properties-tags.md)) - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-tags.md "#/properties/tags#/properties/tags") - -### tags Type - -`object` ([Details](stories-properties-tags.md)) - -### tags Constraints - -**minimum number of items**: the minimum number of items for this array is: `1` - -### tags Default Value - -The default value is: - -```json -{} -``` - -### tags Examples - -```yaml -analytic_story: credential_dumping - -``` - -## version - -version of analytics story, e.g. 1 or 2 ... - -`version` - -* is required - -* Type: `integer` - -* cannot be null - -* defined in: [Analytics Story Schema](stories-properties-version.md "#/properties/version#/properties/version") - -### version Type - -`integer` - -### version Examples - -```yaml -1 - -``` - -## Additional Properties - -Additional properties are allowed and do not have to follow a specific schema diff --git a/pipeline/.app-inspect.yml b/pipeline/.app-inspect.yml deleted file mode 100644 index e3a60dd382..0000000000 --- a/pipeline/.app-inspect.yml +++ /dev/null @@ -1,56 +0,0 @@ -include: - - local: "pipeline/.install-contentctl.yml" - -# This CI job run appinspect using the latest version of contentctl -app_inspect: - stage: app_inspect - needs: - - build - artifacts: - when: always - paths: - - artifacts/* - expire_in: 60 days - before_script: - - !reference [.install-contentctl, before_script] - script: - - 'NEW_VERSION=$(echo $CI_COMMIT_TAG | sed "s/^v//")' - - 'echo "Updating contentctl.yml version: $NEW_VERSION"' - - 'sed -i "/app:/,/apps:/s/version: [0-9]*\.[0-9]*\.[0-9]*/version: $NEW_VERSION/" contentctl.yml' - - cat contentctl.yml - - contentctl inspect --splunk-api-username $APPINSPECT_USERNAME --splunk-api-password $APPINSPECT_PASSWORD --stack_type victoria --enrichments - - after_script: - - mkdir -p artifacts/app_inspect_report - - cp -r dist/*.{json,html} artifacts/app_inspect_report - - cp -r dist/*.tar.gz artifacts/ - - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success - -# This CI job uploads DA-ESS-ContentUpdate-latest.tar.gz to Splunkbase -upload_to_splunkbase: - stage: app_inspect - needs: - - app_inspect - artifacts: - when: always - paths: - - artifacts/* - expire_in: 60 days - script: - - echo "Start upload_to_splunkbase" - - chmod +x pipeline/upload_to_splunkbase.sh - - HTTP_STATUS=$(./pipeline/upload_to_splunkbase.sh) - - | - if [ "$HTTP_STATUS" -eq 200 ]; then - echo "Upload to Splunkbase successful, pipeline will pass." - else - echo "Upload failed with status $HTTP_STATUS, failing the pipeline." - exit 1 - fi - - echo "Finished uploading to splunkbase" - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success diff --git a/pipeline/.build.yml b/pipeline/.build.yml deleted file mode 100644 index 6357e4b9de..0000000000 --- a/pipeline/.build.yml +++ /dev/null @@ -1,26 +0,0 @@ -include: - - local: "pipeline/.install-contentctl.yml" - -# This CI job validates and builds the ESCU/BA/API package -build: - stage: build - artifacts: - when: always - paths: - - artifacts/* - expire_in: 14 days - before_script: - - !reference [.install-contentctl, before_script] - script: - - echo "Validating and Building all files in the dist/ directory" - - pip list - - echo "ContentCTL build starting..." - - contentctl build --enrichments - - echo "ContentCTL build finished" - - after_script: - - mkdir -p artifacts - - cp dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/ - - tar -czf artifacts/SSA-Content-latest.tar.gz -C dist/ssa . - - tar -czf artifacts/SSE-API-latest.tar.gz -C dist/api . - diff --git a/pipeline/.ephemeral-credentials.yml b/pipeline/.ephemeral-credentials.yml deleted file mode 100644 index 72a1467377..0000000000 --- a/pipeline/.ephemeral-credentials.yml +++ /dev/null @@ -1,17 +0,0 @@ -# init creds-helper -.creds-init: - before_script: | - eval $(creds-helper init) - id_tokens: - CI_JOB_JWT: - aud: $CICD_VAULT_ADDR - -# acquire reader role -.generic-read: - before_script: | - if [ "${CI_COMMIT_REF_PROTECTED}" == true ] ; then - export GENERIC_READ_ROLE="artifactory:v2/cloud/role/generic-reader-threat-research-security-content-prod-role" - else - export GENERIC_READ_ROLE="artifactory:v2/cloud/role/generic-reader-threat-research-security-content-test-role" - fi - eval $(creds-helper artifactory --eval ${GENERIC_READ_ROLE}) diff --git a/pipeline/.install-contentctl.yml b/pipeline/.install-contentctl.yml deleted file mode 100644 index 0d1582800f..0000000000 --- a/pipeline/.install-contentctl.yml +++ /dev/null @@ -1,7 +0,0 @@ -# Install contentctl w/ pip -.install-contentctl: - before_script: | - python3.11 -m venv .venv - source .venv/bin/activate - pip install contentctl==4.0.5 - git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git diff --git a/pipeline/.post.yml b/pipeline/.post.yml deleted file mode 100644 index 3d1ca7998a..0000000000 --- a/pipeline/.post.yml +++ /dev/null @@ -1,59 +0,0 @@ -include: - - local: "pipeline/.ephemeral-credentials.yml" - -# set ESCU_VERSION -.set_version: &set_version - - | - if [[ $CI_COMMIT_REF_NAME =~ "^v[0-9]+\.[0-9]+\.[0-9]+$" ]]; then - export ESCU_VERSION=$CI_COMMIT_REF_NAME - elif [[ $CI_COMMIT_REF_NAME =~ "^release_v[0-9]+\.[0-9]+\.[0-9]+$" ]]; then - export ESCU_VERSION=${CI_COMMIT_REF_NAME:8}-${CI_COMMIT_SHA:0:7} - else - export ESCU_VERSION=${CI_COMMIT_SHA:0:7} - fi - -# download and configure extracto -.setup_extracto: &setup_extracto - - | - curl -u $ARTIFACTORY_AUTHORIZATION -L $ARTIFACTORY_BASE_URL/generic/extracto/$EXTRACTO_VERSION/linux_amd64/extracto -o extracto - chmod +x ./extracto - cat > extracto_config.yml << EOF - enrichers: - - gitlab - forwarders: - - splunk - saga-name: ${CI_JOB_NAME}-pipeline_status - test-package-name: ${CI_JOB_NAME}-pipeline_status - version: $ESCU_VERSION - - EOF - - export EXTRACTO_SPLUNK_INDEX="gitlab_extracto" - - export EXTRACTO_CONFIG=./extracto_config.yml - -# send pipeline metadata to extracto -send_pipeline_results: - stage: .post - artifacts: - when: always - paths: - - pipeline_report.json - - extracto_config.json - dependencies: [] - when: always - id_tokens: - !reference [.creds-init, id_tokens] - before_script: - - !reference [.creds-init, before_script] - - !reference [.generic-read, before_script] - script: - - *set_version - - *setup_extracto - - echo "Sending pipline results for CI_PIPELINE_ID=${CI_PIPELINE_ID} to Skynet via extracto" - - if [ -n "$CI_MERGE_REQUEST_IID" ]; then - echo "Extracting status for a merge pipeline"; - ./extracto fetch pipeline_status --ci-job-token ${GITLAB_PIPELINE_ACCESS_TOKEN_READ_ONLY} --ci-pipeline-id ${CI_PIPELINE_ID} --ci-project-id ${CI_PROJECT_ID} --ci-mr-id ${CI_MERGE_REQUEST_IID} --output-file pipeline_report.json; - else - echo "Extracting status for a non merge pipeline"; - ./extracto fetch pipeline_status --ci-job-token ${GITLAB_PIPELINE_ACCESS_TOKEN_READ_ONLY} --ci-pipeline-id ${CI_PIPELINE_ID} --ci-project-id ${CI_PROJECT_ID} --output-file pipeline_report.json; - fi; - - ./extracto extract pipeline_status pipeline_report.json diff --git a/pipeline/.release.yml b/pipeline/.release.yml deleted file mode 100644 index 042005cb79..0000000000 --- a/pipeline/.release.yml +++ /dev/null @@ -1,178 +0,0 @@ -include: - - local: "pipeline/.install-contentctl.yml" -# Run reporting -reporting: - stage: release - needs: - - app_inspect - variables: - BUCKET: "security-content" - before_script: - - !reference [.install-contentctl, before_script] - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - script: - - contentctl report - - aws s3 cp reporting s3://$BUCKET/reporting --recursive --exclude "*" --include "*.svg" - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success - -# Update Attack Range ESCU App -attack_range_escu_app: - stage: release - needs: - - app_inspect - artifacts: - when: always - paths: - - artifacts/* - expire_in: 14 days - variables: - BUCKET: "attack-range-appbinaries" - before_script: - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - script: - - aws s3 cp artifacts/DA-ESS-ContentUpdate-latest.tar.gz s3://$BUCKET/ - - aws s3api put-object-acl --bucket $BUCKET --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success - -# Push SSE security content Json files to Artifactory -push_sse_json_to_artifactory: - stage: release - needs: - - github_push_develop - before_script: - - !reference [.install-contentctl, before_script] - # Update contentctl.yml version key for ESCU using CI_COMMIT_TAG and then build the app using sed - - 'NEW_VERSION=$(echo $CI_COMMIT_TAG | sed "s/^v//")' - - 'echo "Updating contentctl.yml version: $NEW_VERSION"' - - 'sed -i "/app:/,/apps:/s/version: [0-9]*\.[0-9]*\.[0-9]*/version: $NEW_VERSION/" contentctl.yml' - - contentctl build --enrichments - - tar -czvf sse.tar.gz dist/api/* - script: - - creds-helper init - - eval $(creds-helper artifactory --eval $ARTIFACTORY_GENERIC_WRITE_ROLE) - - curl -u $ARTIFACTORY_AUTHORIZATION -X PUT $ARTIFACTORY_BASE_URL/generic/threat-research-security-content/$CI_COMMIT_TAG/$CI_PIPELINE_ID/sse.tar.gz -T sse.tar.gz - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - id_tokens: - CI_JOB_JWT: - aud: $CICD_VAULT_ADDR - -trigger_downstream_sse_json_release: - stage: release - needs: - - push_sse_json_to_artifactory - inherit: - variables: false - variables: - UPSTREAM_ESCU_VERSION: $CI_COMMIT_TAG - IS_DOWNSTREAM_SSE_JSON_RELEASE: "True" - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - trigger: - project: securitycontent/security-content-automation - branch: main - strategy: depend - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - -# Create Package GitLab -release_job: - stage: release - needs: - - app_inspect - image: registry.gitlab.com/gitlab-org/release-cli:latest - script: - - echo "running release_job" - # See https://docs.gitlab.com/ee/ci/yaml/#release for available properties - release: - tag_name: '$CI_COMMIT_TAG' - description: '$CI_COMMIT_TAG' - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success - -# Updating Gitlab dist -update_gitlab_dist: - stage: release - needs: - - release_job - artifacts: - when: always - paths: - - artifacts/* - expire_in: 14 days - before_script: - - !reference [.install-contentctl, before_script] - # Update contentctl.yml version key for ESCU using CI_COMMIT_TAG and then build the app using sed - - 'echo "Current CI_COMMIT_TAG: $CI_COMMIT_TAG"' - - 'NEW_VERSION=$(echo $CI_COMMIT_TAG | sed "s/^v//")' - - 'echo "Updating contentctl.yml version: $NEW_VERSION"' - - 'sed -i "/app:/,/apps:/s/version: [0-9]*\.[0-9]*\.[0-9]*/version: $NEW_VERSION/" contentctl.yml' - - script: - - contentctl build --enrichments - - echo "Updating dist build scripts in a branch" - - git config --global user.email "research@splunk.com" - - git config --global user.name "research-bot" - - git remote set-url origin https://PUSH_DIST_VIA_CI:$PUSH_DIST_VIA_CI@cd.splunkdev.com/threat-research/security_content.git - - git fetch --all - - git checkout develop - - git add contentctl.yml - - git add -f dist/api/* - - git add -f dist/DA-ESS-ContentUpdate/* - - git add -f dist/ssa/* - - git commit -m "Auto-update dist/* files for ESCU,BA,API via release job for tag $CI_COMMIT_TAG [skip ci]" - - git push origin develop - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success - -# Sync changes to GitHub, by creating a branch and push the content and create a PR -github_push_develop: - stage: release - needs: - - update_gitlab_dist - variables: - BRANCH: "gitlab_release_$CI_COMMIT_TAG" - before_script: - - !reference [.install-contentctl, before_script] - - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )' - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - script: - - git config user.email "research@splunk.com" - - git config user.name "research bot" - - git remote add github_origin git@github.com:splunk/security_content.git - - git fetch github_origin - - git checkout -b $BRANCH - - git config merge.submodule ignore - - git merge github_origin/develop -X theirs || true - # Update contentctl.yml version key for ESCU using CI_COMMIT_TAG and then build the app using sed - - 'NEW_VERSION=$(echo $CI_COMMIT_TAG | sed "s/^v//")' - - 'echo "Updating contentctl.yml version: $NEW_VERSION"' - - 'sed -i "/app:/,/apps:/s/version: [0-9]*\.[0-9]*\.[0-9]*/version: $NEW_VERSION/" contentctl.yml' - - contentctl build --enrichments - - git add . - - git add -f dist/api/* - - git add -f dist/DA-ESS-ContentUpdate/* - - git add -f dist/ssa/* - - git status - - git commit -m "Updating Github with Content from ESCU - $CI_COMMIT_TAG" - - git push -u github_origin $BRANCH - # Automatically create a Github PR - - 'curl -X POST -H "Authorization: token $CREATE_GH_RELEASE_PR" -H "Accept: application/vnd.github.v3+json" -d "{\"title\":\"Release $CI_COMMIT_TAG\", \"body\":\"This PR contains content for ESCU - $CI_COMMIT_TAG\", \"head\":\"$BRANCH\", \"base\":\"develop\"}" https://api.github.com/repos/splunk/security_content/pulls' - rules: - - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]$/' - when: on_success \ No newline at end of file diff --git a/pipeline/.test.yml b/pipeline/.test.yml deleted file mode 100644 index c196272fad..0000000000 --- a/pipeline/.test.yml +++ /dev/null @@ -1,206 +0,0 @@ -include: - - local: "pipeline/.install-contentctl.yml" - -# GitLab checks out the repo in a HEAD-less state, so we must checkout the source ref for dry-run -# to work -.checkout_source_branch: &checkout_source_branch - - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME - - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - git checkout $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME - - GIT_HEAD=$(git rev-parse HEAD) - - > - if [ $GIT_HEAD = $CI_COMMIT_SHA ]; then - echo "HEAD ${GIT_HEAD} matches pipline commit sha ${CI_COMMIT_SHA}"; - else - echo "HEAD ${GIT_HEAD} DOES NOT match pipline commit sha ${CI_COMMIT_SHA}"; - exit 1; - fi - -# Save pipeline ID to artifacts so the downstream can do a sanity check -save_pipeline_id: - stage: test - rules: - - if: '$SKIP_DOWNSTREAM_TESTING != "True"' - when: on_success - needs: - - build - artifacts: - when: always - paths: - - artifacts/* - expire_in: 14 days - script: - - echo $CI_PIPELINE_ID > artifacts/security_content_pipeline_id.txt - - -# Generate the list of detections changed in an MR for the downstream; we disable shallow cloning -# here to support diff operations -get_detections_changed: - stage: test - variables: - GIT_DEPTH: 0 - rules: - - if: '$SKIP_DOWNSTREAM_TESTING == "True"' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: on_success - needs: - - build - artifacts: - when: always - paths: - - artifacts/* - reports: - dotenv: build.env - expire_in: 14 days - before_script: - - *checkout_source_branch - - !reference [.install-contentctl, before_script] - script: - - contentctl test --plan-only mode:changes --mode.target-branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - deactivate - - pip3 install virtualenv - - virtualenv .venv - - source .venv/bin/activate - - pip3 install PyYAML==6.0.1 - - python pipeline/tools/check_num_detections.py -c test_plan.yml -d build.env - after_script: - - cp test_plan.yml artifacts/mr_contentctl_test.yml - -# Trigger a downstream test (non-MR) -trigger_escu_test: - stage: test - rules: - - if: '$SKIP_DOWNSTREAM_TESTING == "True"' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^release_v[0-9]+\.[0-9]+\.[0-9]+$/' - when: manual - - if: '$CI_PIPELINE_SOURCE != "merge_request_event"' - when: on_success - inherit: - variables: false - variables: - SECURITY_CONTENT_PIPELINE_ID: $CI_PIPELINE_ID - SECURITY_CONTENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - SECURITY_CONTENT_GIT_REF: $CI_COMMIT_REF_NAME - SECURITY_CONTENT_GIT_COMMIT_SHA: $CI_COMMIT_SHA - SECURITY_CONTENT_GIT_COMMIT_TAG: $CI_COMMIT_TAG - ENABLE_INTEGRATION_TESTING: $ENABLE_INTEGRATION_TESTING - ESCU_BUILD_PATH: artifacts/DA-ESS-ContentUpdate-latest.tar.gz - BUILD_NAME: DA-ESS-ContentUpdate - needs: - - save_pipeline_id - trigger: - project: securitycontent/security-content-automation - branch: main - strategy: depend - -# Trigger a downstream test (MR) -> extends trigger_escu_test -trigger_escu_test_mr: - extends: trigger_escu_test - rules: - - if: '$SKIP_DOWNSTREAM_TESTING == "True"' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: on_success - variables: - SECURITY_CONTENT_PIPELINE_ID: $CI_PIPELINE_ID - SECURITY_CONTENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - SECURITY_CONTENT_GIT_REF: $CI_COMMIT_REF_NAME - SECURITY_CONTENT_GIT_COMMIT_SHA: $CI_COMMIT_SHA - SECURITY_CONTENT_MR_ID: $CI_MERGE_REQUEST_ID - SECURITY_CONTENT_MR_IID: $CI_MERGE_REQUEST_IID - SECURITY_CONTENT_MR_REF_PATH: $CI_MERGE_REQUEST_REF_PATH - SECURITY_CONTENT_MR_EVENT_TYPE: $CI_MERGE_REQUEST_EVENT_TYPE - SECURITY_CONTENT_SOURCE_BRANCH: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME - SECURITY_CONTENT_TARGET_BRANCH: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - SECURITY_CONTENT_SOURCE_COMMIT: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - SECURITY_CONTENT_TARGET_COMMIT: $CI_MERGE_REQUEST_TARGET_BRANCH_SHA - SECURITY_CONTENT_MR_TEST_CONFIG_PATH: artifacts/mr_contentctl_test.yml - ESCU_BUILD_PATH: artifacts/DA-ESS-ContentUpdate-latest.tar.gz - BUILD_NAME: DA-ESS-ContentUpdate - NUM_INSTANCES: "-1" - needs: - - get_detections_changed - - save_pipeline_id - -# Trigger SSA test -trigger_ssa_test: - stage: test - rules: - - if: '$SKIP_DOWNSTREAM_TESTING == "True"' - when: never - - if: '$CI_COMMIT_BRANCH =~ /^release_v[0-9]+\.[0-9]+\.[0-9]+$/' - when: manual - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: - paths: - - 'ssa_detections/**/*' - when: on_success - - if: '$CI_PIPELINE_SOURCE != "merge_request_event"' - when: on_success - inherit: - variables: false - variables: - SECURITY_CONTENT_PIPELINE_ID: $CI_PIPELINE_ID - SECURITY_CONTENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - SECURITY_CONTENT_GIT_REF: $CI_COMMIT_REF_NAME - SECURITY_CONTENT_GIT_COMMIT_SHA: $CI_COMMIT_SHA - SECURITY_CONTENT_GIT_COMMIT_TAG: $CI_COMMIT_TAG - SECURITY_CONTENT_MR_REF_PATH: $CI_MERGE_REQUEST_REF_PATH - SSA_BUILD_PATH: artifacts/SSA-Content-latest.tar.gz - BUILD_NAME: SSA_Content - needs: - - save_pipeline_id - trigger: - project: securitycontent/security-content-automation - branch: main - strategy: depend - -# Retrieve downstream results (non-MR) -get_artifacts: - stage: test - rules: - - if: '$SKIP_DOWNSTREAM_TESTING == "True"' - when: never - - if: '$CI_PIPELINE_SOURCE != "merge_request_event"' - when: always - needs: - - trigger_escu_test - artifacts: - when: always - paths: - - downstream_artifacts/* - reports: - junit: downstream_artifacts/contentctl_summary.xml - expire_in: 14 days - before_script: - - mkdir downstream_artifacts - - mkdir downstream_artifacts_tmp - - pip3 install requests==2.31.0 - script: - - python3 pipeline/tools/get_artifacts.py -t trigger_escu_test --local-token $PIPELINE_READ_TOKEN --down-token $SCA_READ_TOKEN --down-job execute-testcases -o artifacts.zip - after_script: - - unzip artifacts.zip -d downstream_artifacts_tmp - - mv downstream_artifacts_tmp/artifacts/results/contentctl_summary.xml downstream_artifacts/contentctl_summary.xml - - mv downstream_artifacts_tmp/artifacts/results/contentctl_summary.yml downstream_artifacts/contentctl_summary.yml - -# Retrieve downstream results (MR) -> extends get_artifacts -get_artifacts_mr: - extends: get_artifacts - rules: - - if: '$SKIP_DOWNSTREAM_TESTING == "True"' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: always - needs: - - trigger_escu_test_mr - - get_detections_changed - script: - - | - if [ $SKIP_TESTING == "True" ]; then - echo "Nothing was tested, skipping artifact retrieval. after_script commands will fail, but that's ok" - else - python3 pipeline/tools/get_artifacts.py -t trigger_escu_test_mr --local-token $PIPELINE_READ_TOKEN --down-token $SCA_READ_TOKEN --down-job execute-testcases-mr -o artifacts.zip - fi diff --git a/pipeline/tools/check_num_detections.py b/pipeline/tools/check_num_detections.py deleted file mode 100644 index 9771589299..0000000000 --- a/pipeline/tools/check_num_detections.py +++ /dev/null @@ -1,204 +0,0 @@ -""" -A simple script for determining if there are detections to test when doing a diff-level pipeline for -MRs, and enabling/disabling downstream testing as appropriate. If no detections changed, testing is -disabled. If 1+ detections changed, it's enabled. -""" - -import os -import logging -import argparse -from typing import Optional - -import yaml - -# Setup logging defaults -DEFAULT_LOG_LEVEL = logging.INFO -DEFAULT_LOG_PATH = "check_num_detections.log" - -# Create the share logging reference -global_logger: Optional[logging.Logger] = None - - -def setup_logging( - log_path: str = DEFAULT_LOG_PATH, - log_level: int = DEFAULT_LOG_LEVEL -) -> logging.Logger: - """ - Creates a shared logging object for the script - :param log_path: log file path - :param log_level: log level - """ - # create logging object - logger = logging.getLogger(__name__) - logger.setLevel(log_level) - - # create a file and console handler - file_handler = logging.FileHandler(log_path) - file_handler.setLevel(log_level) - console_handler = logging.StreamHandler() - console_handler.setLevel(log_level) - - # create a logging format - formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d - %(message)s") - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - - # add the handlers to the logger - logger.addHandler(file_handler) - logger.addHandler(console_handler) - - return logger - - -def get_logger() -> logging.Logger: - """ - Get logger object (instantiate if not yet setup) - :return logging.Logger: logger object - """ - global global_logger - if global_logger is None: - global_logger = setup_logging() - - global_logger.propagate = False - - return global_logger - - -class ContentctlConfig: - """Base class for manipulating contenctl existing YAML configs""" - - class InitializationError(Exception): - """Class initialization error""" - pass - - class ConfigKeyError(Exception): - """Bad key access in the config""" - pass - - def __init__(self, path: str) -> None: - self.logger = get_logger() - - # raise if the path does not exist - if not os.path.exists(path): - message = f"Path '{path}' does not exist; cannot initialize" - self.logger.error(message) - raise ContentctlConfig.InitializationError(message) - - # raise if the given path is not a file - if not os.path.isfile(path): - message = f"Path '{path}' is not a file; cannot initialize" - self.logger.error(message) - raise ContentctlConfig.InitializationError(message) - - self.path = path - self.config: dict - self.__open() - - def __open(self) -> None: - """Open the config and parse the YAML""" - try: - with open(self.path, "r", encoding="utf-8") as f: - config = yaml.safe_load(f) - except OSError as e: - self.logger.error(f"Failed to open file '{self.path}': {e}") - raise e - except yaml.YAMLError as e: - self.logger.error(f"Failed to parse YAML: {e}") - raise e - - if not isinstance(config, dict): - msg = f"YAML config was loaded as something other than a dict: {type(config)}" - self.logger.error(msg) - raise ValueError(msg) - - self.config = config - - -def parse_args() -> argparse.Namespace: - """ - Parse CLI args - :returns: a Namespace object of the parsed arguments - """ - parser = argparse.ArgumentParser( - prog="check_num_detections", - description=( - "Triggers a pipeline downstream in securitycontent/security_content_automation (for the" - " MR diff-level testing flow)." - ) - ) - parser.add_argument( - "-c", - "--config", - required=True, - help="The contentctl test config (YAML) filepath.", - dest="config_path" - ) - parser.add_argument( - "-d", - "--dotenv", - required=True, - help="The output filepath for the generated dotenv file (e.g. build.env).", - dest="dotenv_path" - ) - return parser.parse_args() - - -def set_skip_testing(value: bool, path: str) -> None: - """ - Writes a dotenv file at the provided path (or appends to the file at the provided path) with - the env var SKIP_TESTING set to the provided value (True/False) - :param value: a bool, the value to set the env var to - :param path: str, the path for the dotenv file - """ - with open(path, "a") as f: - f.write(f"SKIP_TESTING={value}\n") - - -def disable_testing(path: str): - """ - Convenience wrapper around set_skip_testing, disabling downstream testing - :param path: str, the path for the dotenv file - """ - set_skip_testing(value=True, path=path) - - -def enable_testing(path: str): - """ - Convenience wrapper around set_skip_testing, enabling downstream testing - :param path: str, the path for the dotenv file - """ - set_skip_testing(value=False, path=path) - - -def main() -> None: - # Setup logging - logger = get_logger() - - # Parse the CLI args - args = parse_args() - - # Load the test config - contentctl_test = ContentctlConfig(args.config_path) - - # Ensure a list of detections was present - if ("mode" not in contentctl_test.config) or ("files" not in contentctl_test.config["mode"]): - msg = ( - "Either `mode` or `mode.files` is not specified in the provided config: " - f"{args.config_path}" - ) - logger.error(msg) - raise ValueError(msg) - - # If the list of detections is empty, exit with an error - num_detections = len(contentctl_test.config["mode"]["files"]) - if num_detections == 0: - logger.info("No detections to test; telling downstream to skip testing.") - disable_testing(args.dotenv_path) - else: - # If not empty, trigger testing - logger.info(f"Found {num_detections} to test; telling downtream to proceed with testing.") - enable_testing(args.dotenv_path) - - -if __name__ == "__main__": - main() diff --git a/pipeline/tools/get_artifacts.py b/pipeline/tools/get_artifacts.py deleted file mode 100644 index 99f94b8581..0000000000 --- a/pipeline/tools/get_artifacts.py +++ /dev/null @@ -1,418 +0,0 @@ -""" -Simple script for pulling artifacts from the downstream job triggered by a local job. Needs the -following inputs to retrieve artifacts: -- Project Access Token for the local repo -- Project Access Token for the downstream repo -- Local trigger job name -- Downstream job name to fetch artifacts from -- Local pipeline ID (will attempt to get from CI_PIPELINE_ID if not provided) -- Local repo project ID (will attempt to get from CI_PROJECT_ID if not provided) -- Job Token for the current job (will attempt to get from CI_JOB_TOKEN if not provided) -- Output path for the artifacts zip file (will write to ./artifacts.zip if not provided) -""" - -import os -import logging -import argparse -import json -from urllib import parse -from typing import Optional, Union - -import requests - -# Setup logging defaults -DEFAULT_LOG_LEVEL = logging.INFO -DEFAULT_LOG_PATH = "get_artifacts.log" - -# Create the shared logging reference -global_logger: Optional[logging.Logger] = None - - -def setup_logging( - log_path: str = DEFAULT_LOG_PATH, - log_level: int = DEFAULT_LOG_LEVEL -) -> logging.Logger: - """ - Creates a shared logging object for the script - :param log_path: log file path - :param log_level: log level - """ - # create logging object - logger = logging.getLogger(__name__) - logger.setLevel(log_level) - - # create a file and console handler - file_handler = logging.FileHandler(log_path) - file_handler.setLevel(log_level) - console_handler = logging.StreamHandler() - console_handler.setLevel(log_level) - - # create a logging format - formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d - %(message)s") - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - - # add the handlers to the logger - logger.addHandler(file_handler) - logger.addHandler(console_handler) - - return logger - - -def get_logger() -> logging.Logger: - """ - Get logger object (instantiate if not yet setup) - :return logging.Logger: logger object - """ - global global_logger - if global_logger is None: - global_logger = setup_logging() - - global_logger.propagate = False - - return global_logger - - -def parse_args() -> argparse.Namespace: - """ - Parse CLI args - :returns: a Namespace object of the parsed arguments - """ - parser = argparse.ArgumentParser( - prog="get_artifacts", - description=( - "Pulls artifacts from a downstream job" - ) - ) - parser.add_argument( - "-t", - "--trigger-job", - required=True, - type=str, - help="The name of the local pipeline's trigger job.", - dest="trigger_job_name" - ) - parser.add_argument( - "--pipeline-id", - type=str, - help=( - "The local pipeline's ID (attempts to pull CI_PIPELINE_ID from the environment if " - "unspecified)." - ), - default=os.environ.get("CI_PIPELINE_ID"), - dest="local_pipeline_id" - ) - parser.add_argument( - "--project-id", - type=str, - help=( - "The local pipeline's project ID (attempts to pull CI_PROJECT_ID from the environment " - "if unspecified)." - ), - default=os.environ.get("CI_PROJECT_ID"), - dest="local_project_id" - ) - parser.add_argument( - "--job-token", - type=str, - help=( - "The local CI job's token (attempts to pull CI_JOB_TOKEN from the environment if " - "unspecified)." - ), - default=os.environ.get("CI_JOB_TOKEN"), - dest="job_token" - ) - parser.add_argument( - "--local-token", - required=True, - type=str, - help="A project access token with read permissions for the local repo.", - dest="local_token" - ) - parser.add_argument( - "--down-token", - required=True, - type=str, - help="A project access token with read permissions for the target downstream repo.", - dest="down_token" - ) - parser.add_argument( - "--down-job", - required=True, - type=str, - help="The name of the job downstream you want to fetch artifacts from.", - dest="down_job_name" - ) - parser.add_argument( - "-o", - "--out", - type=str, - help="The filepath to write the retrieved artifact bundle to (default: artifacts.zip).", - default="artifacts.zip", - dest="out" - ) - - # Validate and return the arguments - args = parser.parse_args() - validate_args(args) - return args - - -def validate_args(args: argparse.Namespace) -> None: - """ - Validates the given arguments - :param args: a Namespace representing the CLI args - """ - # Check if we failed to pull any of the env var value from the environment - if args.job_token is None: - raise ValueError( - "Could not find CI_JOB_TOKEN in the environment; please provide explicitly via " - "--job-token" - ) - if args.local_pipeline_id is None: - raise ValueError( - "Could not find CI_PIPELINE_ID in the environment; please provide explicitly via " - "--pipeline-id" - ) - if args.local_project_id is None: - raise ValueError( - "Could not find CI_PROJECT_ID in the environment; please provide explicitly via " - "--project-id" - ) - - -class GitLabSession(requests.Session): - """Simple extension of Session that is aware of Splunk's GitLab base URL""" - - def __init__(self): - super().__init__() - self.base_url = "https://cd.splunkdev.com/api/v4/projects/" - self.logger = get_logger() - - def request( - self, - method: Union[str, bytes], - url: Union[str, bytes], - *args, - **kwargs) -> requests.Response: - """ - Extends request, using the base URL with a provided path - :param method: a str for the HTTP method (e.g. "GET") - :param url: a str representing the API path on top of the base URL - :raises HTTPError: if the response code is an error - :returns: a Response object - """ - joined_url = parse.urljoin(self.base_url, url) - self.logger.info(f"Requesting URL: {joined_url}") - response = super().request(method, joined_url, *args, **kwargs) - - try: - response.raise_for_status() - except requests.HTTPError: - self.logger.error(f"Receive HTTP error ({response.status_code}): {response.content!r}") - raise - - return response - - def get_downstream_pipeline( - self, - local_pipeline_id: str, - local_project_id: str, - local_token: str, - trigger_job_name: str - ) -> dict: - """ - Retrieve metadata about the downstream pipeline - :param local_pipeline_id: the local pipeline ID - :param local_project_id: the local project ID - :param local_token: a token for the local project w/ read permissions - :param trigger_job_name: the local trigger job name - :raises HTTPError: if the response code is an error - :returns: a dict representing the downstream pipeline - """ - # Construct API path - api_path = f"{local_project_id}/pipelines/{local_pipeline_id}/bridges" - - # Request the trigger jobs - response = self.get( - api_path, - headers={ - "PRIVATE-TOKEN": local_token - } - ) - - # Try to unpack the response into JSON - try: - trigger_jobs = json.loads(response.content) - except json.JSONDecodeError: - self.logger.error( - "Failed to decode response from bridges API into JSON: {response.content!r}" - ) - raise - - # Look for the specified trigger job name in the trigger jobs - downstream_pipeline = None - for job in trigger_jobs: - if job["name"] == trigger_job_name: - # If we've found the right trigger job, grab the downstream pipeline - downstream_pipeline = job["downstream_pipeline"] - - # If we didn't find it, log and raise - if downstream_pipeline is None: - msg = ( - f"Could not find the specified trigger job in the JSON response: {trigger_job_name}" - ) - self.logger.error(msg) - raise ValueError(msg) - - # If the downstream_pipeline is not a dict, log and raise - if not isinstance(downstream_pipeline, dict): - msg = ( - f"The downstream_pipeline field was of type {type(downstream_pipeline)}; " - "expected dict" - ) - self.logger.error(msg) - raise ValueError(msg) - - self.logger.info( - f"Downstream project: {downstream_pipeline['project_id']} | " - f"Downstream pipeline: {downstream_pipeline['id']}" - ) - return downstream_pipeline - - # NOTE: currently, a project access token is treated like a user of the repo it has access to; - # as a result, it *should* have access to any repos that repo has access to (e.g. - # security_content_automation). If this changes, or if the current private scoping of the SCA - # repo poses issues, I can add another token for the downstream repo. Also, scope tokens for - # read-only (I think they need Developer access for pipeline data) - # This should also be configured w/ a token rotation stage: - # https://docs.gitlab.com/ee/api/project_access_tokens.html#:~:text=Rotate%20a%20project%20access%20token,-History&text=Revokes%20the%20previous%20token%20and,year%20from%20the%20rotation%20date. - # https://docs.gitlab.com/ee/api/project_level_variables.html#update-a-variable - def get_downstream_job( - self, - downstream_project_id: str, - downstream_pipeline_id: str, - downstream_job_name: str, - downstream_token: str - ) -> dict: - """ - Retrieve metadata about a downstream job - :param downstream_project_id: the downstream project ID - :param downstream_pipeline_id: the downstream pipeline ID - :param downstream_job_name: the downstream job name - :param downstream_token: a token for the downstream project w/ read permissions - :raises HTTPError: if the response code is an error - :returns: a dict representing the downstream job - """ - # Construct API path - api_path = f"{downstream_project_id}/pipelines/{downstream_pipeline_id}/jobs" - - # Request the downstream jobs - response = self.get( - api_path, - headers={ - "PRIVATE-TOKEN": downstream_token - } - ) - - # Decode the JSON response; log and raise if there are issues - try: - downstream_jobs = json.loads(response.content) - except json.JSONDecodeError: - self.logger.error( - "Failed to decode response from jobs API into JSON: {response.content!r}" - ) - raise - - # Try to find the downstream job with the specified name - downstream_job = None - for job in downstream_jobs: - if job["name"] == downstream_job_name: - downstream_job = job - - # Log and raise if we couldn't find it - if downstream_job is None: - msg = ( - f"Could not find the specified downstream job in the JSON response: {downstream_job_name}" - ) - self.logger.error(msg) - raise ValueError(msg) - - # Log and raise if it's not a dict - if not isinstance(downstream_job, dict): - msg = ( - f"The {downstream_job_name} field was of type {type(downstream_job)}; " - "expected dict" - ) - self.logger.error(msg) - raise ValueError(msg) - - self.logger.info(f"Downstream job: {downstream_job['name']} ({downstream_job['id']})") - return downstream_job - - def get_artifacts( - self, - downstream_project_id: str, - downstream_job_id: str, - job_token: str, - out: str - ) -> None: - """ - Retrieve artifacts from a downstream job and write to disk - :param downstream_project_id: the downstream project ID - :param downstream_job_id: the downstream job ID - :param job_token: a job token for the current job - :param out: the filepath to write the artifacts ZIP to - :raises HTTPError: if the response code is an error - """ - # Construct API path - api_path = f"{downstream_project_id}/jobs/{downstream_job_id}/artifacts" - - # Request the downstream job's artifacts - response = self.get( - api_path, - data={ - "job_token": job_token - } - ) - - # Write the response to disk - with open(out, "wb") as f: - f.write(response.content) - self.logger.info(f"Wrote artifacts to disk at: {out}") - - -def main(): - # Parse the arguments - args = parse_args() - - # Instantiate the session - session = GitLabSession() - - # Get the downstream pipeline - downstream_pipeline = session.get_downstream_pipeline( - args.local_pipeline_id, - args.local_project_id, - args.local_token, - args.trigger_job_name - ) - - # Get the downstream job - downstream_job = session.get_downstream_job( - downstream_pipeline["project_id"], - downstream_pipeline["id"], - args.down_job_name, - args.down_token - ) - - # Get the artifacts - session.get_artifacts( - downstream_pipeline["project_id"], - downstream_job["id"], - args.job_token, - args.out - ) - - -if __name__ == "__main__": - main() diff --git a/pipeline/upload_to_splunkbase.sh b/pipeline/upload_to_splunkbase.sh deleted file mode 100755 index 40ac3bbac6..0000000000 --- a/pipeline/upload_to_splunkbase.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Check if all required variables are set - -FILE_PATH="/builds/threat-research/security_content/artifacts/DA-ESS-ContentUpdate-latest.tar.gz" -FILE_NAME="DA-ESS-ContentUpdate-latest.tar.gz" -SPLUNKBASE_USERNAME=$SPLUNKBASE_USERNAME -SPLUNKBASE_PASSWORD=$SPLUNKBASE_PASSWORD - -if [ -z "$FILE_PATH" ] || [ -z "$FILE_NAME" ] || [ -z "$SPLUNKBASE_USERNAME" ] || [ -z "$SPLUNKBASE_PASSWORD" ]; then - echo "One or more required variables are undefined." - exit 1 -fi - -curl -u "${SPLUNKBASE_USERNAME}:${SPLUNKBASE_PASSWORD}" --request POST https://splunkbase.splunk.com/api/v1/app/3449/new_release/ \ - -F "files[]=@${FILE_PATH}" \ - -F "filename=${FILE_NAME}" \ - -F "cim_versions=5.x,4.x" \ - -F "splunk_versions=9.2,9.1,9.0,8.2,8.1,8.0,7.3" \ - -F "visibility=false" \ - -o /dev/null -s -w "%{http_code}" diff --git a/spec/ba_detections.spec.json b/spec/ba_detections.spec.json deleted file mode 100644 index 8c5a297566..0000000000 --- a/spec/ba_detections.spec.json +++ /dev/null @@ -1,670 +0,0 @@ -{ - "$id": "http://example.com/example.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "description": "schema for ba detections", - "properties": { - "name": { - "$id": "#/properties/name", - "default": "", - "examples": [ - "Anomalous usage of Archive Tools" - ], - "title": "Name of detection", - "type": "string" - }, - "id": { - "$id": "#/properties/id", - "default": "", - "description": "UUID as unique identifier", - "examples": [ - "fb4c31b0-13e8-4155-8aa5-24de4b8d6717" - ], - "type": "string" - }, - "version": { - "$id": "#/properties/version", - "default": 0, - "description": "version of detection, e.g. 1 or 2 ...", - "examples": [ - 1 - ], - "type": "integer" - }, - "status": { - "$id": "#/properties/status", - "default": "production", - "description": "status of detection: production or validation", - "examples": [ - 1 - ], - "type": "string" - }, - "description": { - "$id": "#/properties/description", - "default": "", - "description": "A detailed description of the detection", - "examples": [ - "The following detection identifies the usage of archive tools from the command line." - ], - "type": "string" - }, - "search": { - "$id": "#/properties/search", - "default": "", - "description": "The Splunk search for the detection", - "examples": [ - "| from read_ba_enriched_events()" - ], - "type": "string" - }, - "how_to_implement": { - "$id": "#/properties/how_to_implement", - "default": "", - "description": "information about how to implement.", - "examples": [ - "Ingest Windows Event Code 4688 with Command Line Logging enabled." - ], - "type": "string" - }, - "known_false_positives": { - "$id": "#/properties/knwon_false_positives", - "default": "", - "description": "known false postives", - "examples": [ - "False positives can be ligitmate usage of archive tools from the command line." - ], - "type": "string" - }, - "references": { - "$id": "#/properties/references", - "additionalItems": true, - "default": [], - "description": "A list of references for this detection", - "examples": [ - [ - "https://attack.mitre.org/techniques/T1560/001/" - ] - ], - "items": { - "$id": "#/properties/references/items", - "default": "", - "description": "Link to reference", - "examples": [ - "https://attack.mitre.org/techniques/T1560/001/" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "runtime": { - "$id": "#/properties/runtime", - "default": "", - "description": "runtime for BA detection", - "examples": [ - "SPL2" - ], - "items": { - "enum": [ - "DSP-SPL", - "SPL2", - "Spark" - ], - "type": "string" - }, - "type": "string" - }, - "detection_type": { - "$id": "#/properties/detection_type", - "default": "", - "description": "type of detection", - "examples": [ - "Rule" - ], - "items": { - "enum": [ - "Rule", - "ML" - ], - "type": "string" - }, - "type": "string" - }, - "job_params": { - "$id": "#/properties/job_params", - "additionalProperties": true, - "default": "", - "description": "job parameters", - "examples": [ - { - "entrypointClassName": "com.splunk.ssa.batch.poc.BatchEntryPoint", - "modelsJarFileName": "batch-models.jar" - } - ], - "type": "object" - }, - "model_params": { - "$id": "#/properties/model_params", - "additionalProperties": true, - "default": "", - "description": "model parameters", - "examples": [ - { - "modelClassName": "com.splunk.ssa.batch.poc.models.DeviceAccessRareModel", - "lookBackPeriodInDays": 30 - } - ], - "type": "object" - }, - "tags": { - "$id": "#/properties/tags", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for tagging", - "examples": [ - { - "annotations": {"analytic_story": ["NOBELIUM Group"], "cis20": ["CIS 10"], "kill_chain_phases": ["Exploitation"], "mitre_attack_id": ["T1560.001", "T1560"], "nist": ["DE.AE"]}, - "ocsf_attributes": {"activity_id": 1, "category_uid": 2, "class_uid": 102001, "risk_level_id": 2, "risk_score": 42, "severity_id": 0, "rule": {"name": "Anomalous usage of Archive Tools", "uid": "63614a58-10e2-4c6c-ae81-ea1113681439", "type": "Streaming"}, "metadata": {"product": {"name": "Behavior Analytics", "vendor_name": "Splunk"}, "version": "1.0.0-rc.2"}, "type_uid": 10200101}, - "required_fields": ["process.user.name", "device.hostname"], - "risk_score": 42, - "security_domain": "endpoint", - "risk_severity": "low", - "research_site_url": "https://research.splunk.com/endpoint/63614a58-10e2-4c6c-ae81-ea1113681439/", - "event_schema": "ocsf", - "mappings": [{"ocsf": "process.user.name", "cim": "user"},{"ocsf": "device.hostname", "cim": "dest"}] - } - ], - "type": "object", - "uniqueItems": true, - "required": [ - "annotations", - "risk_score", - "security_domain", - "risk_severity", - "research_site_url", - "event_schema", - "mappings" - ], - "properties": { - "annotations": { - "$id": "#/properties/tags/annotations", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for annotations", - "examples": [ - { - "analytic_story": ["NOBELIUM Group"], - "cis20": ["CIS 10"], - "kill_chain_phases": ["Exploitation"], - "mitre_attack_id": ["T1560.001"], - "nist": ["DE.AE"] - } - ], - "type": "object", - "uniqueItems": true, - "required": [ - "analytic_story", - "cis20", - "kill_chain_phases", - "mitre_attack_id", - "nist" - ], - "properties": { - "analytic_story": { - "$id": "#/properties/tags/annotations/analytic_story", - "additionalItems": true, - "default": [], - "description": "A list of analytic stories for this detection", - "examples": [ - [ - "NOBELIUM Group" - ] - ], - "items": { - "$id": "#/properties/tags/annotations/analytic_story/items", - "default": "", - "description": "analytic story name", - "examples": [ - "NOBELIUM Group" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "cis20": { - "$id": "#/properties/tags/annotations/cis20", - "additionalItems": true, - "default": [], - "description": "cis tags", - "examples": [ - ["CIS 10"] - ], - "items": { - "$id": "#/properties/tags/annotations/cis20/items", - "default": "", - "description": "cis tag", - "examples": [ - "CIS 10" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "kill_chain_phases": { - "$id": "#/properties/tags/annotations/kill_chain_phases", - "additionalItems": true, - "default": [], - "description": "kill_chain_phases", - "examples": [ - ["Exploitation"] - ], - "items": { - "$id": "#/properties/tags/annotations/kill_chain_phases/items", - "default": "", - "description": "kill_chain_phase", - "examples": [ - "Exploitation" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "mitre_attack_id": { - "$id": "#/properties/tags/annotations/mitre_attack_id", - "additionalItems": true, - "default": [], - "description": "mitre_attack_id", - "examples": [ - ["T1560.001", "T1560"] - ], - "items": { - "$id": "#/properties/tags/annotations/mitre_attack_id/items", - "default": "", - "description": "kill_chain_phase", - "examples": [ - "T1560.001" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "nist": { - "$id": "#/properties/tags/annotations/nist", - "additionalItems": true, - "default": [], - "description": "nist array", - "examples": [ - ["DE.AE"] - ], - "items": { - "$id": "#/properties/tags/annotations/nist/items", - "default": "", - "description": "nist", - "examples": [ - "DE.AE" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - } - } - }, - "ocsf_attributes": { - "$id": "#/properties/tags/ocsf_attributes", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for static ocsf attributes", - "examples": [ - { - "activity_id": 1, - "category_uid": 2, - "class_uid": 102001, - "risk_level_id": 2, - "risk_score": 42, - "severity_id": 0, - "rule": { - "name": "Anomalous usage of Archive Tools", - "uid": "63614a58-10e2-4c6c-ae81-ea1113681439", - "type": "Streaming" - }, - "metadata": { - "product": { - "name": "Behavior Analytics", - "vendor_name": "Splunk" - }, - "version": "1.0.0-rc.2" - }, - "type_uid": 10200101 - } - ], - "type": "object", - "uniqueItems": true, - "required": [ - "activity_id", - "category_uid", - "class_uid", - "risk_level_id", - "risk_score", - "severity_id", - "rule", - "metadata", - "type_uid" - ], - "properties": { - "activity_id": { - "$id": "#/properties/tags/ocsf_attributes/activity_id", - "default": 1, - "description": "activity_id", - "examples": [ - 1 - ], - "type": "integer" - }, - "category_uid": { - "$id": "#/properties/tags/ocsf_attributes/category_uid", - "default": 2, - "description": "category_uid", - "examples": [ - 2 - ], - "type": "integer" - }, - "class_uid": { - "$id": "#/properties/tags/ocsf_attributes/class_uid", - "default": 102001, - "description": "class_uid", - "examples": [ - 102001 - ], - "type": "integer" - }, - "risk_level_id": { - "$id": "#/properties/tags/ocsf_attributes/risk_level_id", - "default": 2, - "description": "risk_level_id", - "examples": [ - 2 - ], - "type": "integer" - }, - "risk_score": { - "$id": "#/properties/tags/ocsf_attributes/risk_score", - "default": 42, - "description": "risk_score", - "examples": [ - 42 - ], - "type": "integer" - }, - "severity_id": { - "$id": "#/properties/tags/ocsf_attributes/severity_id", - "default": 0, - "description": "severity_id", - "examples": [ - 0 - ], - "type": "integer" - }, - "rule": { - "$id": "#/properties/tags/ocsf_attributes/rule", - "additionalProperties": true, - "default": {}, - "description": "rule object", - "examples": [ - { - "name": "Anomalous usage of Archive Tools", - "uid": "63614a58-10e2-4c6c-ae81-ea1113681439", - "type": "Streaming" - } - ], - "type": "object", - "uniqueItems": true, - "required": [ - "name", - "uid" - ], - "properties": { - "name": { - "$id": "#/properties/tags/ocsf_attributes/rule/name", - "default": "", - "description": "rule name", - "examples": [ - "Anomalous usage of Archive Tools" - ], - "type": "string" - }, - "uid": { - "$id": "#/properties/tags/ocsf_attributes/rule/uid", - "default": "", - "description": "rule uid", - "examples": [ - "63614a58-10e2-4c6c-ae81-ea1113681439" - ], - "type": "string" - }, - "type": { - "$id": "#/properties/tags/ocsf_attributes/rule/type", - "default": "", - "description": "rule type", - "examples": [ - "Streaming" - ], - "type": "string" - } - } - }, - "metadata": { - "$id": "#/properties/tags/ocsf_attributes/metadata", - "additionalProperties": true, - "default": {}, - "description": "metadata object", - "examples": [ - { - "product": { - "name": "Behavior Analytics", - "vendor_name": "Splunk" - }, - "version": "1.0.0-rc.2" - } - ], - "type": "object", - "uniqueItems": true, - "required": [ - "product", - "version" - ], - "properties": { - "product": { - "$id": "#/properties/tags/ocsf_attributes/metadata/product", - "additionalProperties": true, - "default": {}, - "description": "product object", - "examples": [ - { - "name": "Behavior Analytics", - "vendor_name": "Splunk" - } - ], - "type": "object", - "uniqueItems": true, - "required": [ - "name", - "vendor_name" - ], - "properties": { - "name": { - "$id": "#/properties/tags/ocsf_attributes/metadata/product/name", - "default": "", - "description": "product name", - "examples": [ - "Behavior Analytics" - ], - "type": "string" - }, - "vendor_name": { - "$id": "#/properties/tags/ocsf_attributes/metadata/product/vendor_name", - "default": "", - "description": "vendor product name", - "examples": [ - "Splunk" - ], - "type": "string" - } - } - }, - "version": { - "$id": "#/properties/tags/ocsf_attributes/metadata/version", - "default": "", - "description": "ocsf schema version", - "examples": [ - "1.0.0-rc.2" - ], - "type": "string" - } - } - }, - "type_uid": { - "$id": "#/properties/tags/ocsf_attributes/type_uid", - "default": 10200101, - "description": "type_uid", - "examples": [ - 10200101 - ], - "type": "integer" - } - } - }, - "required_fields": { - "$id": "#/properties/tags/required_fields", - "additionalItems": true, - "default": [], - "description": "required_fields", - "examples": [ - ["process.user.name", "device.hostname"] - ], - "items": { - "$id": "#/properties/tags/required_fields/items", - "default": "", - "description": "required_fields name", - "examples": [ - "process.user.name" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "risk_score": { - "$id": "#/properties/tags/risk_score", - "default": 50, - "description": "risk_score", - "examples": [ - 42 - ], - "type": "integer" - }, - "security_domain": { - "$id": "#/properties/tags/security_domain", - "default": "", - "description": "security_domain", - "examples": [ - "endpoint" - ], - "type": "string" - }, - "risk_severity": { - "$id": "#/properties/tags/risk_severity", - "default": "", - "description": "risk_severity", - "examples": [ - "low" - ], - "type": "string" - }, - "research_site_url": { - "$id": "#/properties/tags/research_site_url", - "default": "", - "description": "research_site_url", - "examples": [ - "https://research.splunk.com/endpoint/63614a58-10e2-4c6c-ae81-ea1113681439/" - ], - "type": "string" - }, - "event_schema": { - "$id": "#/properties/tags/event_schema", - "default": "", - "description": "event_schema", - "examples": [ - "ocsf" - ], - "type": "string" - }, - "mappings": { - "$id": "#/properties/tags/mappings", - "default": [], - "description": "mappings array", - "examples": [ - [{"ocsf": "process.user.name", "cim": "user"},{"ocsf": "device.hostname", "cim": "dest"}] - ], - "items": { - "$id": "#/properties/tags/mappings/items", - "default": {}, - "description": "mappings object", - "examples": [ - { - "ocsf": "process.user.name", - "cim": "user" - } - ], - "type": "object", - "uniqueItems": true, - "properties": { - "ocsf": { - "$id": "#/properties/tags/mappings/items/ocsf", - "default": "", - "description": "ocsf mapping", - "examples": [ - "process.user.name" - ], - "type": "string" - }, - "cim": { - "$id": "#/properties/tags/mappings/items/cim", - "default": "", - "description": "cim mapping", - "examples": [ - "user" - ], - "type": "string" - } - }, - "required": [ - "ocsf", - "cim" - ] - }, - "type": "array" - } - } - } - }, - "required": [ - "name", - "id", - "version", - "description", - "how_to_implement", - "known_false_positives", - "references", - "tags" - ], - "title": "BA Detection Schema", - "type": "object" -} - \ No newline at end of file diff --git a/spec/ba_detections_validator.py b/spec/ba_detections_validator.py deleted file mode 100644 index d4875d47fa..0000000000 --- a/spec/ba_detections_validator.py +++ /dev/null @@ -1,46 +0,0 @@ -import json -import sys -from pathlib import Path -import yaml -from jsonschema import validate, ValidationError - -def read_yaml_file(file_path): - with open(file_path, 'r') as file: - return yaml.safe_load(file) - -def read_json_file(file_path): - with open(file_path, 'r') as file: - return json.load(file) - -def validate_json_against_schema(json_data, schema): - try: - validate(instance=json_data, schema=schema) - return True - except ValidationError as e: - print(f"Validation Error: {e}") - return False - -def main(yaml_file_path, json_schema_file_path): - yaml_data = read_yaml_file(yaml_file_path) - json_schema = read_json_file(json_schema_file_path) - - is_valid = validate_json_against_schema(yaml_data, json_schema) - - if is_valid: - print("The YAML file is valid according to the JSON schema.") - else: - print("The YAML file is not valid according to the JSON schema.") - -if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage: python ba_detections_validator.py ") - sys.exit(1) - - yaml_file_path = Path(sys.argv[1]) - json_schema_file_path = Path(sys.argv[2]) - - if not yaml_file_path.is_file() or not json_schema_file_path.is_file(): - print("Both input files must exist.") - sys.exit(1) - - main(yaml_file_path, json_schema_file_path) \ No newline at end of file diff --git a/spec/deployments.spec.json b/spec/deployments.spec.json deleted file mode 100644 index 1dc2e17652..0000000000 --- a/spec/deployments.spec.json +++ /dev/null @@ -1,369 +0,0 @@ -{ - "$id": "http://example.com/example.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "default": {}, - "description": "schema for deployment", - "properties": { - "alert_action": { - "$id": "#/properties/alert_action", - "additionalProperties": true, - "default": {}, - "description": "Set alert action parameter for search", - "examples": [ - { - "email": { - "message": "Splunk Alert $name$ triggered %fields%", - "subject": "Splunk Alert $name$", - "to": "test@test.com" - }, - "index": { - "name": "asx" - }, - "notable": { - "rule_description": "%description%", - "rule_title": "%name%" - } - } - ], - "properties": { - "email": { - "$id": "#/properties/alert_action/properties/email", - "additionalProperties": true, - "default": {}, - "description": "By enabling it, an email is sent with the results", - "examples": [ - { - "message": "Splunk Alert $name$ triggered %fields%", - "subject": "Splunk Alert $name$", - "to": "test@test.com" - } - ], - "properties": { - "message": { - "$id": "#/properties/alert_action/properties/email/properties/message", - "default": "", - "description": "message of email", - "examples": [ - "Splunk Alert $name$ triggered %fields%" - ], - "type": "string" - }, - "subject": { - "$id": "#/properties/alert_action/properties/email/properties/subject", - "default": "", - "description": "Subject of email", - "examples": [ - "Splunk Alert $name$" - ], - "type": "string" - }, - "to": { - "$id": "#/properties/alert_action/properties/email/properties/to", - "default": "", - "description": "Recipient of email", - "examples": [ - "test@test.com" - ], - "type": "string" - } - }, - "required": [ - "to", - "subject", - "message" - ], - "type": "object" - }, - "index": { - "$id": "#/properties/alert_action/properties/index", - "additionalProperties": true, - "default": {}, - "description": "By enabling it, the results are stored in another index", - "examples": [ - { - "name": "asx" - } - ], - "properties": { - "name": { - "$id": "#/properties/alert_action/properties/index/properties/name", - "default": "", - "description": "Name of the index", - "examples": [ - "asx" - ], - "type": "string" - } - }, - "required": [ - "name" - ], - "type": "object" - }, - "notable": { - "$id": "#/properties/alert_action/properties/notable", - "additionalProperties": true, - "default": {}, - "description": "By enabling it, a notable is generated", - "examples": [ - { - "rule_description": "%description%", - "rule_title": "%name%" - } - ], - "properties": { - "rule_description": { - "$id": "#/properties/alert_action/properties/notable/properties/rule_description", - "default": "", - "description": "Rule description of the notable event", - "examples": [ - "%description%" - ], - "type": "string" - }, - "rule_title": { - "$id": "#/properties/alert_action/properties/notable/properties/rule_title", - "default": "", - "description": "Rule title of the notable event", - "examples": [ - "%name%" - ], - "type": "string" - } - }, - "required": [ - "rule_title", - "rule_description" - ], - "type": "object" - }, - "slack": { - "$id": "#/properties/alert_action/properties/slack", - "additionalProperties": true, - "default": {}, - "description": "By enabling it, a slack message is sent", - "examples": [ - { - "channel": "slack_channel", - "message": "Alert x triggered" - } - ], - "properties": { - "channel": { - "$id": "#/properties/alert_action/properties/slack/properties/channel", - "default": "", - "description": "Slack channel", - "examples": [ - "slack_channel" - ], - "type": "string" - }, - "message": { - "$id": "#/properties/alert_action/properties/slack/properties/message", - "default": "", - "description": "message", - "examples": [ - "Alert x triggered" - ], - "type": "string" - } - }, - "required": [ - "channel", - "message" - ], - "type": "object" - }, - "phantom": { - "$id": "#/properties/alert_action/properties/phantom", - "additionalProperties": true, - "default": {}, - "description": "By enabling it, the event is sent to phantom", - "examples": [ - { - "phantom_server": "phantom", - "label": "events", - "sensitivity": "amber", - "severity": "medium", - "cam_workers": "local" - } - ], - "properties": { - "phantom_server": { - "$id": "#/properties/alert_action/properties/phantom/properties/phantom_server", - "default": "", - "description": "Phantom server", - "examples": [ - "phantom" - ], - "type": "string" - }, - "label": { - "$id": "#/properties/alert_action/properties/phantom/properties/label", - "default": "", - "description": "label", - "examples": [ - "events" - ], - "type": "string" - }, - "sensitivity": { - "$id": "#/properties/alert_action/properties/phantom/properties/sensitivity", - "default": "", - "description": "sensitivity", - "examples": [ - "amber" - ], - "type": "string" - }, - "severity": { - "$id": "#/properties/alert_action/properties/phantom/properties/severity", - "default": "", - "description": "severity", - "examples": [ - "medium" - ], - "type": "string" - }, - "cam_workers": { - "$id": "#/properties/alert_action/properties/phantom/properties/cam_workers", - "default": "", - "description": "adaptive response worker set", - "examples": [ - "local" - ], - "type": "string" - } - }, - "required": [ - "phantom_server", - "label", - "sensitivity", - "severity" - ], - "type": "object" - } - }, - "type": "object" - }, - "date": { - "$id": "#/properties/date", - "default": "", - "description": "date of creation or modification, format yyyy-mm-dd", - "examples": [ - "2019-12-06" - ], - "type": "string" - }, - "description": { - "$id": "#/properties/description", - "default": "", - "description": "description of the deployment configuration", - "examples": [ - "This deployment configuration provides a standard scheduling policy over all rules." - ], - "type": "string" - }, - "id": { - "$id": "#/properties/id", - "default": "", - "description": "uuid as unique identifier", - "examples": [ - "fb4c31b0-13e8-4155-8aa5-24de4b8d6717" - ], - "type": "string" - }, - "name": { - "$id": "#/properties/name", - "default": "", - "description": "Name of deployment configuration", - "examples": [ - "Deployment Configuration all Detections" - ], - "type": "string" - }, - "scheduling": { - "$id": "#/properties/scheduling", - "additionalProperties": true, - "default": {}, - "description": "allows to set scheduling parameter", - "examples": [ - { - "cron_schedule": "*/10 * * * *", - "earliest_time": "-10m", - "latest_time": "now", - "schedule_window": "auto" - } - ], - "properties": { - "cron_schedule": { - "$id": "#/properties/scheduling/properties/cron_schedule", - "default": "", - "description": "Cron schedule to schedule the Splunk searches.", - "examples": [ - "*/10 * * * *" - ], - "type": "string" - }, - "earliest_time": { - "$id": "#/properties/scheduling/properties/earliest_time", - "default": "", - "description": "earliest time of search", - "examples": [ - "-10m" - ], - "type": "string" - }, - "latest_time": { - "$id": "#/properties/scheduling/properties/latest_time", - "default": "", - "description": "latest time of search", - "examples": [ - "now" - ], - "type": "string" - }, - "schedule_window": { - "$id": "#/properties/scheduling/properties/schedule_window", - "default": "", - "description": "schedule window for search", - "examples": [ - "auto" - ], - "type": "string" - } - }, - "required": [ - "cron_schedule", - "earliest_time", - "latest_time" - ], - "type": "object" - }, - "tags": { - "$id": "#/properties/tags", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for tagging", - "examples": [ - { - "analytic_story": "credential_dumping" - } - ], - "minItems": 1, - "type": "object", - "uniqueItems": true - } - }, - "required": [ - "name", - "id", - "date", - "description", - "scheduling", - "tags" - ], - "title": "Deployment Schema", - "type": "object" -} diff --git a/spec/detections.spec.json b/spec/detections.spec.json deleted file mode 100644 index 4d5b2a69ad..0000000000 --- a/spec/detections.spec.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "$id": "http://example.com/example.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "description": "schema for detections", - "properties": { - "author": { - "$id": "#/properties/author", - "default": "", - "description": "Author of the detection", - "examples": [ - "Patrick Bareiss, Splunk" - ], - "type": "string" - }, - "date": { - "$id": "#/properties/date", - "default": "", - "description": "date of creation or modification, format yyyy-mm-dd", - "examples": [ - "2019-12-06" - ], - "type": "string" - }, - "description": { - "$id": "#/properties/description", - "default": "", - "description": "A detailed description of the detection", - "examples": [ - "dbgcore.dll is a specifc DLL for Windows core debugging. It is used to obtain a memory dump of a process. This search detects the usage of this DLL for creating a memory dump of LSASS process. Memory dumps of the LSASS process can be created with tools such as Windows Task Manager or procdump." - ], - "type": "string" - }, - "how_to_implement": { - "$id": "#/properties/how_to_implement", - "default": "", - "description": "information about how to implement. Only needed for non standard implementations.", - "examples": [ - "This search requires Sysmon Logs and a Sysmon configuration, which includes EventCode 10 for lsass.exe." - ], - "type": "string" - }, - "id": { - "$id": "#/properties/id", - "default": "", - "description": "UUID as unique identifier", - "examples": [ - "fb4c31b0-13e8-4155-8aa5-24de4b8d6717" - ], - "type": "string" - }, - "known_false_positives": { - "$id": "#/properties/knwon_false_positives", - "default": "", - "description": "known false postives", - "examples": [ - "Administrators can create memory dumps for debugging purposes, but memory dumps of the LSASS process would be unusual." - ], - "type": "string" - }, - "name": { - "$id": "#/properties/name", - "default": "", - "examples": [ - "Access LSASS Memory for Dump Creation" - ], - "title": "Name of detection", - "type": "string" - }, - "references": { - "$id": "#/properties/references", - "additionalItems": true, - "default": [], - "description": "A list of references for this detection", - "examples": [ - [ - "https://2017.zeronights.org/wp-content/uploads/materials/ZN17_Kheirkhabarov_Hunting_for_Credentials_Dumping_in_Windows_Environment.pdf" - ] - ], - "items": { - "$id": "#/properties/references/items", - "default": "", - "description": "An explanation about the purpose of this instance.", - "examples": [ - "https://2017.zeronights.org/wp-content/uploads/materials/ZN17_Kheirkhabarov_Hunting_for_Credentials_Dumping_in_Windows_Environment.pdf" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "search": { - "$id": "#/properties/search", - "default": "", - "description": "The Splunk search for the detection", - "examples": [ - "`sysmon` EventCode=10 TargetImage=*lsass.exe CallTrace=*dbgcore.dll* OR CallTrace=*dbghelp.dll* | stats count min(_time) as firstTime max(_time) as lastTime by Computer, TargetImage, TargetProcessId, SourceImage, SourceProcessId | rename Computer as dest | `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | `access_lsass_memory_for_dump_creation_filter`" - ], - "type": "string" - }, - "tags": { - "$id": "#/properties/tags", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for tagging", - "examples": [ - { - "analytic_story": "credential_dumping", - "kill_chain_phases": "Action on Objectives", - "mitre_attack_id": "T1078.004", - "cis20": "CIS 13", - "nist": "DE.DP", - "security domain": "network", - "asset_type": "AWS Instance", - "risk_object": "user", - "risk_object_type": "network_artifacts", - "risk score": "60", - "custom_key": "custom_value" - } - ], - "minItems": 1, - "type": "object", - "uniqueItems": true - }, - "type": { - "$id": "#/properties/type", - "default": "", - "description": "type of detection", - "examples": [ - "Anomaly" - ], - "items": { - "enum": [ - "TTP", - "Anomaly", - "Hunting", - "Baseline", - "Investigation", - "Correlation" - ], - "type": "string" - }, - "type": "string" - }, - "datamodel": { - "$id": "#/properties/datamodel", - "default": "", - "description": "datamodel used in the search", - "examples": [ - "Endpoint" - ], - "items": { - "enum": [ - "Endpoint", - "Network_Traffic", - "Authentication", - "Change", - "Change_Analysis", - "Email", - "Endpoint", - "Network_Resolution", - "Network_Sessions", - "Network_Traffic", - "UEBA", - "Updates", - "Vulnerabilities", - "Web", - "Risk" - ], - "type": "string" - }, - "type": "array" - }, - "version": { - "$id": "#/properties/version", - "default": 0, - "description": "version of detection, e.g. 1 or 2 ...", - "examples": [ - 2 - ], - "type": "integer" - } - }, - "required": [ - "name", - "id", - "version", - "date", - "datamodel", - "description", - "type", - "author", - "search", - "known_false_positives", - "tags" - ], - "title": "Detection Schema", - "type": "object" -} diff --git a/spec/lookups.spec.json b/spec/lookups.spec.json deleted file mode 100644 index 4d59673a35..0000000000 --- a/spec/lookups.spec.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "$id": "https://api.splunkresearch.com/schemas/lookups.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "A object that defines a lookup file and its properties.", - "oneOf": [ - { - "required": [ - "collection" - ] - }, - { - "required": [ - "filename" - ] - } - ], - "properties": { - "case_sensitive_match": { - "description": "What the macro is intended to filter", - "enum": [ - "true", - "false" - ], - "examples": [ - "true" - ], - "type": "string" - }, - "collection": { - "description": "Name of the collection to use for this lookup", - "examples": [ - "prohibited_apps_launching_cmd" - ], - "type": "string" - }, - "default_match": { - "description": "The default value if no match is found", - "examples": [ - "true" - ], - "type": "string" - }, - "description": { - "description": "The description of this lookup", - "examples": [ - "This lookup contains file names that exist in the Windows\\System32 directory" - ], - "type": "string" - }, - "fields_list": { - "description": "A comma and space separated list of field names", - "examples": [ - "_key, dest, process_name" - ], - "type": "string" - }, - "filename": { - "description": "The name of the file to use for this lookup", - "examples": [ - "prohibited_apps_launching_cmd.csv" - ], - "type": "string" - }, - "filter": { - "description": "Use this attribute to improve search performance when working with significantly large KV", - "examples": [ - "dest=\"SPLK_*\"" - ], - "type": "string" - }, - "match_type": { - "description": "A comma and space-delimited list of () specification to allow for non-exact matching", - "examples": [ - "WILDCARD(process)" - ], - "type": "string" - }, - "max_matches": { - "description": "The maximum number of possible matches for each input lookup value", - "examples": [ - "100" - ], - "type": "integer" - }, - "min_matches": { - "description": "Minimum number of possible matches for each input lookup value", - "examples": [ - "1" - ], - "type": "integer" - }, - "name": { - "description": "The name of the lookup to be used in searches", - "examples": [ - "isWindowsSystemFile_lookup" - ], - "type": "string" - } - }, - "title": "Lookup Manifest", - "type": "object" -} diff --git a/spec/macros.spec.json b/spec/macros.spec.json deleted file mode 100644 index cd0f5cafe1..0000000000 --- a/spec/macros.spec.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$id": "https://api.splunkresearch.com/schemas/macros.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "An object that defines the parameters for a Splunk Macro", - "properties": { - "arguments": { - "description": "A list of the arguments being passed to this macro", - "items": { - "type": "string" - }, - "minItems": 0, - "type": "array", - "uniqueItems": true - }, - "definition": { - "description": "The macro definition", - "examples": [ - "(query=fls-na* AND query = www* AND query=images*)" - ], - "type": "string" - }, - "description": { - "description": "What the macro is intended to filter", - "examples": [ - "Use this macro to filter out known good objects" - ], - "type": "string" - }, - "name": { - "description": "The name of the macro", - "examples": [ - "detection_search_output_filter" - ], - "type": "string" - } - }, - "required": [ - "name", - "description" - ], - "title": "Macro Manifest", - "type": "object" -} diff --git a/spec/playbooks.spec.json b/spec/playbooks.spec.json deleted file mode 100644 index 33af1eeb25..0000000000 --- a/spec/playbooks.spec.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "$id": "http://example.com/example.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "description": "schema for playbooks", - "properties": { - "author": { - "$id": "#/properties/author", - "default": "", - "description": "Author of the playbook", - "examples": [ - "Lou Stella, Splunk" - ], - "type": "string" - }, - "date":{ - "$id": "#/properties/date", - "default": "", - "description": "date of creation or modification, format yyyy-mm-dd", - "examples": [ - "2021-09-28" - ], - "type": "string" - }, - "description": { - "$id": "#/properties/description", - "default": "", - "description": "A detailed description of the playbook", - "examples": [ - "This playbook investigates and contains ransomware detected on endpoints." - ], - "type": "string" - }, - "how_to_implement": { - "$id": "#/properties/how_to_implement", - "default": "", - "description": "information about how to implement the playbook in Splunk SOAR", - "examples": [ - "This playbook requires the Splunk SOAR apps for Palo Alto Networks Firewalls, Palo Alto Wildfire, LDAP, and Carbon Black Response." - ], - "type": "string" - }, - "references": { - "$id": "#/properties/references", - "additionalItems": true, - "default": [], - "description": "A list of references for this playbook", - "examples": [ - [ - "https://www.splunk.com/en_us/blog/security/splunk-soar-playbooks-gcp-unusual-service-account-usage.html" - ] - ], - "items": { - "$id": "#/properties/references/items", - "default": "", - "description": "An explanation about the purpose of this instance.", - "examples": [ - "https://www.splunk.com/en_us/blog/security/splunk-soar-playbooks-gcp-unusual-service-account-usage.html" - ], - "title": "The Items Schema", - "type": "string" - }, - "type": "array" - }, - "id": { - "$id": "#/properties/id", - "default": "", - "description": "UUID as unique identifier", - "examples":[ - "fb4c31b0-13e8-4155-8aa5-24de4b8d6717" - ], - "type": "string" - }, - "playbook":{ - "$id": "#/properties/playbook", - "default": "", - "description": "name of playbook file within same directory without suffix", - "examples":[ - "ransomware_investigate_and_contain" - ], - "type": "string" - }, - "name": { - "$id": "#/properties/name", - "default": "", - "examples": [ - "Ransomware Investigate and Contain" - ], - "title": "name of playbook", - "type": "string" - }, - "app_list":{ - "$id": "#/properties/app_list", - "default": "", - "examples": [ - "LDAP" - ], - "type": "array" - }, - "tags": { - "$id": "#/properties/tags", - "additionalProperties": true, - "default": {}, - "description": "An array of key value pairs for tagging", - "examples":[ - { - "analytic_story": "Ransomware", - "detections": "Conti Common Exec parameter", - "platform_tags": "Investigate", - "playbook_fields": "Username", - "product": "Splunk SOAR", - "defend_technique_id": "D3-DA" - } - ], - "minItems": 1, - "type": "object", - "uniqueItems": true - }, - "type":{ - "$id": "#/properties/type", - "default": "", - "description": "type of playbook", - "examples": [ - "Investigation" - ], - "items": { - "enum": [ - "Investigation", - "Response" - ], - "type": "string" - }, - "type": "string" - }, - "version": { - "$id": "#/properties/version", - "default": 0, - "description": "version of playbook, e.g. 1 or 2...", - "examples": [ - 2 - ], - "type": "integer" - } - }, - "required": [ - "name", - "id", - "version", - "date", - "description", - "type", - "author", - "playbook", - "tags" - ], - "title": "Playbook schema", - "type": "object" -} diff --git a/spec/stories.spec.json b/spec/stories.spec.json deleted file mode 100644 index f2384cbb81..0000000000 --- a/spec/stories.spec.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "$id": "http://example.com/example.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "default": {}, - "description": "schema analytics story", - "properties": { - "author": { - "$id": "#/properties/author", - "default": "", - "description": "Author of the analytics story", - "examples": [ - "Rico Valdez, Patrick Barei\u00df, Splunk" - ], - "type": "string" - }, - "date": { - "$id": "#/properties/date", - "default": "", - "description": "date of creation or modification, format yyyy-mm-dd", - "examples": [ - "2019-12-06" - ], - "type": "string" - }, - "description": { - "$id": "#/properties/description", - "default": "", - "description": "description of the analytics story", - "examples": [ - "Uncover activity consistent with credential dumping, a technique where attackers compromise systems and attempt to obtain and exfiltrate passwords." - ], - "type": "string" - }, - "id": { - "$id": "#/properties/id", - "default": "", - "description": "UUID as unique identifier", - "examples": [ - "fb4c31b0-13e8-4155-8aa5-24de4b8d6717" - ], - "type": "string" - }, - "name": { - "$id": "#/properties/name", - "default": "", - "description": "Name of the Analytics Story", - "examples": [ - "Credential Dumping" - ], - "type": "string" - }, - "narrative": { - "$id": "#/properties/narrative", - "default": "", - "description": "narrative of the analytics story", - "examples": [ - "gathering credentials from a target system, often hashed or encrypted, is a common attack technique. Even though the credentials may not be in plain text, an attacker can still exfiltrate the data and set to cracking it offline, on their own systems." - ], - "type": "string" - }, - "search": { - "$id": "#/properties/search", - "default": "", - "description": "An additional Splunk search, which uses the result of the detections", - "examples": [ - "index=asx mitre_id=t1003 | stats values(source) as detections values(process) as processes values(user) as users values(_time) as time count by dest" - ], - "type": "string" - }, - "tags": { - "$id": "#/properties/tags", - "additionalProperties": true, - "default": {}, - "description": "An explanation about the purpose of this instance.", - "examples": [ - { - "analytic_story": "credential_dumping" - } - ], - "minItems": 1, - "type": "object" - }, - "version": { - "$id": "#/properties/version", - "default": 0, - "description": "version of analytics story, e.g. 1 or 2 ...", - "examples": [ - 1 - ], - "type": "integer" - } - }, - "required": [ - "name", - "id", - "version", - "date", - "description", - "narrative", - "author", - "tags" - ], - "title": "Analytics Story Schema", - "type": "object" -} diff --git a/spec/tests.spec.json b/spec/tests.spec.json deleted file mode 100644 index 8f7ddf8043..0000000000 --- a/spec/tests.spec.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "$id": "https://raw.githubusercontent.com/splunk/security_content/develop/docs/spec/tests.spec.json", - "$schema": "http://json-schema.org/draft-07/schema", - "additionalProperties": true, - "default": {}, - "description": "schema test file", - "properties": { - "name": { - "$id": "#/properties/name", - "default": "", - "description": "Name of the Analytics Story", - "examples": [ - "Credential Dumping" - ], - "type": "string" - }, - "tests": { - "$id": "#/properties/tests", - "additionalProperties": true, - "default": {}, - "description": "a test description.", - "examples": [ - [ - { - "name": "Detect Activity Related to Pass the Hash Attacks", - "file": "endpoint/detect_activity_related_to_pass_the_hash_attacks.yml", - "pass_condition": "| stats count | where count > 0", - "earliest_time": "-24h", - "latest_time": "now", - "attack_data": [ - { - "file_name": "windows-security.log", - "data": "https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1550.002/atomic_red_team/windows-security.log", - "source": "WinEventLog:Security", - "sourcetype": "WinEventLog", - "update_timestamp": true - } - ] - } - ] - ], - "minItems": 1, - "type": "array", - "items": [ - { - "type": "object" - } - ] - } - }, - "required": [ - "name", - "tests" - ], - "title": "Test file schema", - "type": "object" -}