From ccd65cdb3fb80d11a9cf6768262806ebc7b55065 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 19 Oct 2023 11:48:18 -0700 Subject: [PATCH 1/4] Merge some of the ba updates from security_content into contentctl --- contentctl/input/backend_splunk_ba.py | 24 ++++++++++-- contentctl/objects/ssa_detection.py | 1 + contentctl/objects/ssa_detection_tags.py | 6 ++- contentctl/output/ba_yml_output.py | 4 +- contentctl/output/finding_report_writer.py | 45 ++++++++++++++++++++-- 5 files changed, 70 insertions(+), 10 deletions(-) diff --git a/contentctl/input/backend_splunk_ba.py b/contentctl/input/backend_splunk_ba.py index fd759ce4..a99da0fe 100644 --- a/contentctl/input/backend_splunk_ba.py +++ b/contentctl/input/backend_splunk_ba.py @@ -8,7 +8,7 @@ from sigma.exceptions import SigmaFeatureNotSupportedByBackendError from sigma.pipelines.splunk.splunk import splunk_sysmon_process_creation_cim_mapping, splunk_windows_registry_cim_mapping, splunk_windows_file_event_cim_mapping -from contentctl.objects.detection import Detection +from contentctl.objects.ssa_detection import SSADetection from typing import ClassVar, Dict, List, Optional, Pattern, Tuple @@ -67,7 +67,7 @@ class SplunkBABackend(TextQueryBackend): wildcard_match_expression : ClassVar[Optional[str]] = "{field} LIKE {value}" - def __init__(self, processing_pipeline: Optional["sigma.processing.pipeline.ProcessingPipeline"] = None, collect_errors: bool = False, min_time : str = "-30d", max_time : str = "now", detection : Detection = None, field_mapping: dict = None, **kwargs): + def __init__(self, processing_pipeline: Optional["sigma.processing.pipeline.ProcessingPipeline"] = None, collect_errors: bool = False, min_time : str = "-30d", max_time : str = "now", detection : SSADetection = None, field_mapping: dict = None, **kwargs): super().__init__(processing_pipeline, collect_errors, **kwargs) self.min_time = min_time or "-30d" self.max_time = max_time or "now" @@ -110,13 +110,29 @@ def finalize_query_data_model(self, rule: SigmaRule, query: str, index: int, sta parent = new_val i = i + 1 continue - parser_str = '| eval ' + new_val + ' = ' + parent + '.' + val + ' ' + new_val_spaces = new_val + "=" + if new_val_spaces not in query: + parser_str = '| eval ' + new_val + ' = ' + parent + '.' + val + ' ' + else: + parser_str = '| eval ' + new_val + ' = ' + 'lower(' + parent + '.' + val + ') ' detection_str = detection_str + parser_str parsed_fields.append(new_val) parent = new_val i = i + 1 - detection_str = detection_str + "| where " + query + ### Convert sigma values into lower case + lower_query = "" + in_quotes = False + for char in query: + if char == '"': + in_quotes = not in_quotes + if in_quotes: + lower_query += char.lower() + else: + lower_query += char + + detection_str = detection_str + "| where " + lower_query + detection_str = detection_str.replace("\\\\\\\\", "\\\\") return detection_str diff --git a/contentctl/objects/ssa_detection.py b/contentctl/objects/ssa_detection.py index 7d794f69..fd86aa9d 100644 --- a/contentctl/objects/ssa_detection.py +++ b/contentctl/objects/ssa_detection.py @@ -48,6 +48,7 @@ class SSADetection(BaseModel): file_path: str = None source: str = None runtime: str = None + internalVersion: int = None test: Union[UnitTest, dict] = None # @validator('name')v diff --git a/contentctl/objects/ssa_detection_tags.py b/contentctl/objects/ssa_detection_tags.py index e8e54108..55bb4720 100644 --- a/contentctl/objects/ssa_detection_tags.py +++ b/contentctl/objects/ssa_detection_tags.py @@ -12,7 +12,7 @@ class SSADetectionTags(BaseModel): asset_type: str automated_detection_testing: str = None cis20: list = None - confidence: str + confidence: int impact: int kill_chain_phases: list = None message: str @@ -40,7 +40,9 @@ class SSADetectionTags(BaseModel): risk_level: str = None observable_str: str = None evidence_str: str = None - kill_chain_phases_id: list = None + analytics_story_str: str = None + kill_chain_phases_id:dict = None + kill_chain_phases_str:str = None research_site_url: str = None event_schema: str = None mappings: list = None diff --git a/contentctl/output/ba_yml_output.py b/contentctl/output/ba_yml_output.py index 6b316f34..1298027a 100644 --- a/contentctl/output/ba_yml_output.py +++ b/contentctl/output/ba_yml_output.py @@ -61,7 +61,8 @@ def writeObjects(self, objects: list, output_path: str, type: SecurityContentTyp "nist": obj.tags.nist } - obj.runtime = "SPL-DSP" + obj.runtime = "SPL2" + obj.internalVersion = 2 # remove unncessary fields YmlWriter.writeYmlFile(file_path, obj.dict( @@ -78,6 +79,7 @@ def writeObjects(self, objects: list, output_path: str, type: SecurityContentTyp "known_false_positives": True, "references": True, "runtime": True, + "internalVersion": True, "tags": { #"analytic_story": True, diff --git a/contentctl/output/finding_report_writer.py b/contentctl/output/finding_report_writer.py index ce62f57a..4babea9d 100644 --- a/contentctl/output/finding_report_writer.py +++ b/contentctl/output/finding_report_writer.py @@ -10,6 +10,37 @@ class FindingReportObject(): @staticmethod def writeFindingReport(detection : SSADetection) -> None: + if detection.tags.confidence < 33: + detection.tags.confidence_id = 1 + elif detection.tags.confidence < 66: + detection.tags.confidence_id = 2 + else: + detection.tags.confidence_id = 3 + + if detection.tags.impact < 20: + detection.tags.impact_id = 1 + elif detection.tags.impact < 40: + detection.tags.impact_id = 2 + elif detection.tags.impact < 60: + detection.tags.impact_id = 3 + elif detection.tags.impact < 80: + detection.tags.impact_id = 4 + else: + detection.tags.impact_id = 5 + + detection.tags.kill_chain_phases_id = dict() + for kill_chain_phase in detection.tags.kill_chain_phases: + detection.tags.kill_chain_phases_id[kill_chain_phase] = SES_KILL_CHAIN_MAPPINGS[kill_chain_phase] + + kill_chain_phase_str = "[" + i = 0 + for kill_chain_phase in detection.tags.kill_chain_phases_id.keys(): + kill_chain_phase_str = kill_chain_phase_str + '{"phase": "' + kill_chain_phase + '", "phase_id": ' + str(detection.tags.kill_chain_phases_id[kill_chain_phase]) + "}" + if not i == (len(detection.tags.kill_chain_phases_id.keys()) - 1): + kill_chain_phase_str = kill_chain_phase_str + ', ' + i = i + 1 + kill_chain_phase_str = kill_chain_phase_str + ']' + detection.tags.kill_chain_phases_str = kill_chain_phase_str if detection.tags.risk_score < 20: detection.tags.risk_level_id = 0 @@ -27,15 +58,23 @@ def writeFindingReport(detection : SSADetection) -> None: detection.tags.risk_level_id = 4 detection.tags.risk_level = "Critical" - evidence_str = "create_map(" + evidence_str = "{" for i in range(len(detection.tags.observable)): - evidence_str = evidence_str + '"' + detection.tags.observable[i]["name"] + '", ' + detection.tags.observable[i]["name"].replace(".", "_") + evidence_str = evidence_str + '"' + detection.tags.observable[i]["name"] + '": ' + detection.tags.observable[i]["name"].replace(".", "_") if not i == (len(detection.tags.observable) - 1): evidence_str = evidence_str + ', ' - evidence_str = evidence_str + ')' + evidence_str = evidence_str + '}' detection.tags.evidence_str = evidence_str + analytics_story_str = "[" + for i in range(len(detection.tags.analytic_story)): + analytics_story_str = analytics_story_str + '"' + detection.tags.analytic_story[i] + '"' + if not i == (len(detection.tags.analytic_story) - 1): + analytics_story_str = analytics_story_str + ', ' + analytics_story_str = analytics_story_str + ']' + detection.tags.analytics_story_str = analytics_story_str + if "actor.user.name" in detection.tags.required_fields: actor_user_name = "actor_user_name" else: From 671140d7d182a127c4a36cfb6cd364783fe9cb84 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 19 Oct 2023 11:57:30 -0700 Subject: [PATCH 2/4] Add support for BA status validation --- contentctl/input/director.py | 3 ++- .../detection_abstract.py | 7 +++++++ contentctl/objects/enums.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 6fa10068..fb083bd0 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -25,6 +25,7 @@ from contentctl.input.story_builder import StoryBuilder from contentctl.objects.enums import SecurityContentType from contentctl.objects.enums import SecurityContentProduct +from contentctl.objects.enums import DetectionStatus from contentctl.helper.utils import Utils from contentctl.enrichments.attack_enrichment import AttackEnrichment from contentctl.objects.config import Config @@ -167,7 +168,7 @@ def createSecurityContent(self, type: SecurityContentType) -> None: elif type == SecurityContentType.ssa_detections: self.constructSSADetection(self.ssa_detection_builder, file) detection = self.ssa_detection_builder.getObject() - if detection.status == "production" or detection.status == "validated": + if detection.status in [DetectionStatus.production.value, DetectionStatus.validation.value]: self.output_dto.ssa_detections.append(detection) else: diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index a554f703..180f32a2 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -88,6 +88,13 @@ def type_valid(cls, v, values): def encode_error(cls, v, values, field): return SecurityContentObject.free_text_field_valid(cls,v,values,field) + @root_validator + def validation_for_ba_only(cls, values): + # Ensure that only a BA detection can have status: validation + if values["status"] == DetectionStatus.validation.value: + raise ValueError(f"The following is NOT an ssa_ detection, but has 'status: {values['status']} which may ONLY be used for ssa_ detections:' {values['file_path']}") + return values + # @root_validator # def search_validation(cls, values): # if 'ssa_' not in values['file_path']: diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py index a5b983af..b62ee474 100644 --- a/contentctl/objects/enums.py +++ b/contentctl/objects/enums.py @@ -64,6 +64,7 @@ class DetectionStatus(enum.Enum): production = "production" deprecated = "deprecated" experimental = "experimental" + validation = "validation" class LogLevel(enum.Enum): NONE = "NONE" From c04f3cb9a5b765ec3f83163ffdea691052a28696 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Thu, 19 Oct 2023 17:14:28 -0700 Subject: [PATCH 3/4] Rename field in config file. Reorder BA yml output to match old version. Update API outputs to match legacy version. --- contentctl/actions/generate.py | 4 ++-- contentctl/objects/config.py | 4 ++-- contentctl/objects/ssa_detection.py | 2 +- contentctl/output/api_json_output.py | 9 +++++++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/contentctl/actions/generate.py b/contentctl/actions/generate.py index 83e98631..70f0074b 100644 --- a/contentctl/actions/generate.py +++ b/contentctl/actions/generate.py @@ -40,7 +40,7 @@ def execute(self, input_dto: GenerateInputDto) -> DirectorOutputDto: return director_output_dto elif input_dto.director_input_dto.product == SecurityContentProduct.SSA: - output_path = os.path.join(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config.build_ssa.output_path) + output_path = os.path.join(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config.build_ssa.path_root) shutil.rmtree(output_path + '/srs/', ignore_errors=True) shutil.rmtree(output_path + '/complex/', ignore_errors=True) os.makedirs(output_path + '/complex/') @@ -49,7 +49,7 @@ def execute(self, input_dto: GenerateInputDto) -> DirectorOutputDto: ba_yml_output.writeObjects(director_output_dto.ssa_detections, output_path) elif input_dto.director_input_dto.product == SecurityContentProduct.API: - output_path = os.path.join(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config.build_api.output_path) + output_path = os.path.join(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config.build_api.path_root) shutil.rmtree(output_path, ignore_errors=True) os.makedirs(output_path) api_json_output = ApiJsonOutput() diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index a869de92..a654bdbb 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -148,10 +148,10 @@ class ConfigEnrichments(BaseModel): class ConfigBuildSSA(BaseModel): - output_path: str + path_root: str class ConfigBuildApi(BaseModel): - output_path: str + path_root: str class Config(BaseModel, extra=Extra.forbid): #general: ConfigGlobal = ConfigGlobal() diff --git a/contentctl/objects/ssa_detection.py b/contentctl/objects/ssa_detection.py index fd86aa9d..5ad3499c 100644 --- a/contentctl/objects/ssa_detection.py +++ b/contentctl/objects/ssa_detection.py @@ -47,9 +47,9 @@ class SSADetection(BaseModel): mappings: dict = None file_path: str = None source: str = None + test: Union[UnitTest, dict] = None runtime: str = None internalVersion: int = None - test: Union[UnitTest, dict] = None # @validator('name')v # def name_max_length(cls, v, values): diff --git a/contentctl/output/api_json_output.py b/contentctl/output/api_json_output.py index 4ae5ced8..d3ad9d13 100644 --- a/contentctl/output/api_json_output.py +++ b/contentctl/output/api_json_output.py @@ -88,7 +88,9 @@ def writeObjects(self, objects: list, output_path: str, type: SecurityContentTyp obj_array.append(baseline.dict( exclude = { - "deployment": True + "deployment": True, + "check_references":True, + "file_path":True, } )) @@ -116,6 +118,9 @@ def writeObjects(self, objects: list, output_path: str, type: SecurityContentTyp obj_array = [] for deployment in objects: deployment.id = str(deployment.id) - obj_array.append(deployment.dict(exclude_none=True)) + obj_array.append(deployment.dict(exclude_none=True, exclude = + { + "file_path":True, + })) JsonWriter.writeJsonObject(os.path.join(output_path, 'deployments.json'), {'deployments': obj_array }) From adb99f0ecda02fd7be74c0e08bb226a92dbdfb2d Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Fri, 20 Oct 2023 15:52:18 -0700 Subject: [PATCH 4/4] Copy mlfiles along with mlmodel files --- contentctl/input/director.py | 3 +-- contentctl/output/conf_output.py | 9 ++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/contentctl/input/director.py b/contentctl/input/director.py index 6fa10068..3101f9ab 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -99,9 +99,8 @@ def execute(self, input_dto: DirectorInputDto) -> None: def createSecurityContent(self, type: SecurityContentType) -> None: - objects = [] if type == SecurityContentType.ssa_detections: - files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, 'ssa_detections')) + files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, 'ssa_detections')) elif type == SecurityContentType.unit_tests: files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, 'tests')) else: diff --git a/contentctl/output/conf_output.py b/contentctl/output/conf_output.py index 4115a5fd..189f18af 100644 --- a/contentctl/output/conf_output.py +++ b/contentctl/output/conf_output.py @@ -112,12 +112,12 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> None: 'transforms.j2', self.config, objects) - #import code - #code.interact(local=locals()) + if self.input_path is None: raise(Exception(f"input_path is required for lookups, but received [{self.input_path}]")) - files = glob.iglob(os.path.join(self.input_path, 'lookups', '*.csv')) + #we want to copy all *.mlmodel files as well, not just csvs + files = list(glob.iglob(os.path.join(self.input_path, 'lookups', '*.csv'))) + list(glob.iglob(os.path.join(self.input_path, 'lookups', '*.mlmodel'))) lookup_folder = self.output_path/"lookups" if lookup_folder.exists(): # Remove it since we want to remove any previous lookups that are not @@ -125,6 +125,7 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> None: if lookup_folder.is_dir(): shutil.rmtree(lookup_folder) else: + #it's a file, but there should not be a file called lookups lookup_folder.unlink() # Make the new folder for the lookups @@ -136,6 +137,8 @@ def writeObjects(self, objects: list, type: SecurityContentType = None) -> None: if lookup_path.is_file(): lookup_target_path = self.output_path/"lookups"/lookup_path.name shutil.copy(lookup_path, lookup_target_path) + else: + raise(Exception(f"Error copying lookup/mlmodel file. Path {lookup_path} does not exist or is not a file.")) elif type == SecurityContentType.macros: ConfWriter.writeConfFile(self.output_path/'default/macros.conf',