diff --git a/analyzer/windows/bin/loader.exe b/analyzer/windows/bin/loader.exe index 76cce431e..4cb7b8fb1 100644 Binary files a/analyzer/windows/bin/loader.exe and b/analyzer/windows/bin/loader.exe differ diff --git a/analyzer/windows/bin/loader_x64.exe b/analyzer/windows/bin/loader_x64.exe index 12863df84..2727d4ff5 100644 Binary files a/analyzer/windows/bin/loader_x64.exe and b/analyzer/windows/bin/loader_x64.exe differ diff --git a/analyzer/windows/dll/CAPE_Extraction.dll b/analyzer/windows/dll/CAPE_Extraction.dll index a2819c536..1f04f6fc7 100644 Binary files a/analyzer/windows/dll/CAPE_Extraction.dll and b/analyzer/windows/dll/CAPE_Extraction.dll differ diff --git a/analyzer/windows/dll/CAPE_UPX.dll b/analyzer/windows/dll/CAPE_UPX.dll index 442766a3c..9b1f1d393 100644 Binary files a/analyzer/windows/dll/CAPE_UPX.dll and b/analyzer/windows/dll/CAPE_UPX.dll differ diff --git a/analyzer/windows/dll/cuckoomon.dll b/analyzer/windows/dll/cuckoomon.dll index 312af95ab..a93fe04ab 100644 Binary files a/analyzer/windows/dll/cuckoomon.dll and b/analyzer/windows/dll/cuckoomon.dll differ diff --git a/analyzer/windows/dll/cuckoomon_x64.dll b/analyzer/windows/dll/cuckoomon_x64.dll index 12b1841ac..cd3be1ba6 100644 Binary files a/analyzer/windows/dll/cuckoomon_x64.dll and b/analyzer/windows/dll/cuckoomon_x64.dll differ diff --git a/analyzer/windows/lib/common/abstracts.py b/analyzer/windows/lib/common/abstracts.py index ecc65320b..8b8762d15 100644 --- a/analyzer/windows/lib/common/abstracts.py +++ b/analyzer/windows/lib/common/abstracts.py @@ -155,6 +155,7 @@ def debug(self, path, args, interest): @return: process pid """ dll = self.options.get("dll") + dll_64 = self.options.get("dll_64") gw = self.options.get("setgw", None) u = Utils() @@ -168,7 +169,12 @@ def debug(self, path, args, interest): raise CuckooPackageError("Unable to execute the initial process, " "analysis aborted.") - p.debug_inject(dll, interest, childprocess=False) + is_64bit = p.is_64bit() + + if is_64bit: + p.debug_inject(dll_64, interest, childprocess=False) + else: + p.debug_inject(dll, interest, childprocess=False) p.resume() p.close() diff --git a/analyzer/windows/modules/packages/CAPE_Extraction.py b/analyzer/windows/modules/packages/CAPE_Extraction.py index f393bb6c9..a1b345922 100644 --- a/analyzer/windows/modules/packages/CAPE_Extraction.py +++ b/analyzer/windows/modules/packages/CAPE_Extraction.py @@ -19,11 +19,9 @@ def __init__(self, options={}, config=None): self.options = options self.pids = [] self.options["dll"] = "CAPE_Extraction.dll" - self.options["procmemdump"] = '0' + self.options["dll_64"] = "CAPE_Extraction_x64.dll" def start(self, path): - self.options["dll"] = "CAPE_Extraction.dll" - self.options["procmemdump"] = '0' arguments = self.options.get("arguments") # If the file doesn't have an extension, add .exe diff --git a/modules/machinery/vmwareserver.py b/modules/machinery/vmwareserver.py index ee9c708fd..893d25567 100644 --- a/modules/machinery/vmwareserver.py +++ b/modules/machinery/vmwareserver.py @@ -26,16 +26,6 @@ def _initialize_check(self): raise CuckooMachineError("VMware vmrun path missing, " "please add it to vmwareserver.conf") -# if not os.path.exists(self.options.vmwareserver.path): -# raise CuckooMachineError("VMware vmrun not found in " -# "specified path %s" % -# self.options.vmwareserver.path) - # Consistency checks. -# for machine in self.machines(): -# vmx_path = machine.label - -# snapshot = self._snapshot_from_vmx(vmx_path) -# self._check_vmx(vmx_path) self._check_snapshot(vmx_path, snapshot) # Base checks. @@ -59,7 +49,7 @@ def _check_snapshot(self, vmx_path, snapshot): @param snapshot: snapshot name @raise CuckooMachineError: if snapshot not found """ - #check_string = "strace " + \ + check_string = self.options.vmwareserver.path + \ " -T ws-shared -h " + \ self.options.vmwareserver.vmware_url + \ @@ -88,9 +78,8 @@ def start(self, vmx_path): """ snapshot = self._snapshot_from_vmx(vmx_path) - # Preventive check + # Check if the machine is already running, stop if so. if self._is_running(vmx_path): - #raise CuckooMachineError("Machine %s is already running" % vmx_path) log.debug("Machine %s is already running, attempting to stop..." % vmx_path) self.stop(vmx_path) time.sleep(3) @@ -99,7 +88,6 @@ def start(self, vmx_path): time.sleep(3) - #start_string = "strace " + \ start_string = self.options.vmwareserver.path + \ " -T ws-shared -h " + \ self.options.vmwareserver.vmware_url + \ @@ -126,8 +114,7 @@ def stop(self, vmx_path): @param vmx_path: path to vmx file @raise CuckooMachineError: if unable to stop. """ - #stop_string = "strace " + \ - #self.options.vmwareserver.path + \ + stop_string = self.options.vmwareserver.path + \ " -T ws-shared -h " + \ self.options.vmwareserver.vmware_url + \ @@ -159,8 +146,6 @@ def _revert(self, vmx_path, snapshot): """ log.debug("Revert snapshot for vm %s: %s" % (vmx_path, snapshot)) - #revert_string = "strace " + \ - #self.options.vmwareserver.path + \ revert_string = self.options.vmwareserver.path + \ " -T ws-shared -h " + \ self.options.vmwareserver.vmware_url + \ @@ -168,6 +153,8 @@ def _revert(self, vmx_path, snapshot): " -p " + self.options.vmwareserver.password + \ " revertToSnapshot " + "\"" + vmx_path + "\" " + snapshot + #log.debug("Revert string: %s" % revert_string) + try: if subprocess.call(revert_string, shell=True): raise CuckooMachineError("Unable to revert snapshot for " @@ -183,8 +170,6 @@ def _is_running(self, vmx_path): @param vmx_path: path to vmx file @return: running status """ - #list_string = "strace " + \ - #self.options.vmwareserver.path + \ list_string = self.options.vmwareserver.path + \ " -T ws-shared -h " + \ self.options.vmwareserver.vmware_url + \ @@ -192,12 +177,10 @@ def _is_running(self, vmx_path): " -p " + self.options.vmwareserver.password + \ " list " + "\"" + vmx_path + "\"" + #log.debug("List string: %s" % list_string) + try: p = subprocess.Popen(list_string, stdout=subprocess.PIPE, shell=True) - #p = subprocess.Popen(list_string, - #p = subprocess.Popen([self.options.vmware.path, "list"], - #stdout=subprocess.PIPE, - #stderr=subprocess.PIPE) output, error = p.communicate() except OSError as e: raise CuckooMachineError("Unable to check running status for %s. " @@ -216,41 +199,3 @@ def _snapshot_from_vmx(self, vmx_path): """ vm_info = self.db.view_machine_by_label(vmx_path) return vm_info.snapshot - - def dump_memory(self, vmx_path, path): - """Take a memory dump of the machine.""" - if not os.path.exists(vmx_path): - raise CuckooMachineError("Can't find .vmx file {0}. Ensure to configure a fully qualified path in vmwareserver.conf (key = vmx_path)".format(vmx_path)) - - try: - subprocess.call([self.options.vmwareserver.path, - "-T ws-shared -h", self.options.vmwareserver.vmware_url, - "-u", self.options.vmwareserver.username, "-p", self.options.vmwareserver.password, - "snapshot", - vmx_path, "memdump"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError as e: - raise CuckooMachineError("vmrun failed to take a memory dump of the machine with label %s: %s" % (vmx_path, e)) - - vmwarepath, _ = os.path.split(vmx_path) - latestvmem = max(glob.iglob(os.path.join(vmwarepath, "*.vmem")), - key=os.path.getctime) - - # We need to move the snapshot to the current analysis directory as - # vmware doesn't support an option for the destination path :-/ - shutil.move(latestvmem, path) - - # Old snapshot can be deleted, as it isn't needed any longer. - try: - subprocess.call([self.options.vmwareserver.path, - "-T ws-shared -h", vmware_url, - "-u", self.options.vmwareserver.username, "-p", self.options.vmwareserver.password, - "deleteSnapshot", - vmx_path, "memdump"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError as e: - raise CuckooMachineError("vmrun failed to delete the temporary snapshot in %s: %s" % (vmx_path, e)) - - log.info("Successfully generated memory dump for virtual machine with label %s ", vmx_path) diff --git a/modules/processing/parsers/mwcp/malwareconfigreporter.py b/modules/processing/parsers/mwcp/malwareconfigreporter.py index fb3f8ac86..fdfbea191 100644 --- a/modules/processing/parsers/mwcp/malwareconfigreporter.py +++ b/modules/processing/parsers/mwcp/malwareconfigreporter.py @@ -68,7 +68,7 @@ class malwareconfigreporter(object): metadata: Dictionary containing the metadata extracted from the malware by the parser pe: a pefile object, assuming pefile is installed and succesful parsing of file outputfiles: dictionary of entries for each ouput file. The key is the filename specified. Each entry - is a dictionary with keys of data and description. If the path key is set, the file was written + is a dictionary with keys of data, description, and md5. If the path key is set, the file was written to that path on the filesystem. fields: dictionary containing the standardized fields with each field comprising an embedded dictionary. The 1st level keys are the field names. Under that, the keys are "description", @@ -168,7 +168,7 @@ def filename(self): else: #we were passed data buffer. Lazy initialize a temp file for this if not self.__tempfilename: - with tempfile.NamedTemporaryFile(delete=False,dir=self.tempdir,prefix="mwcp-inputfile-") as tfile: + with tempfile.NamedTemporaryFile(delete=False,dir=self.managed_tempdir(),prefix="mwcp-inputfile-") as tfile: tfile.write(self.data) self.__tempfilename = tfile.name @@ -349,12 +349,12 @@ def __add_metadata_listofstringtuples(self, keyu, value): if len(values) >= 2: if values[1] not in ["tcp", "udp", "icmp"]: self.debug("Expected port type to be tcp or udp (or icmp)") - elif keyu == "registrykeyvalue": - self.add_metadata("registrykey", values[0]) + elif keyu == "registrypathdata": + self.add_metadata("registrypath", values[0]) if len(values) >= 2: - self.add_metadata("registryvalue", values[1]) + self.add_metadata("registrydata", values[1]) if len(values) != 2: - self.debug("Expected two values in type registrykeyvalue, received %i" % len(values)) + self.debug("Expected two values in type registrypathdata, received %i" % len(values)) elif keyu == "service": if values[0]: self.add_metadata("servicename", values[0]) @@ -463,9 +463,8 @@ def list_parsers(self): if modulename[-len(self.__parsernamepostfix):] == self.__parsernamepostfix and len(modulename) > len(self.__parsernamepostfix): parsers.append(modulename[:-len(self.__parsernamepostfix)]) - return parsers - - + return sorted(parsers, key=lambda s: s.lower()) + def load_parser_instance(self, name): ''' Load parser instance by parser name @@ -502,6 +501,8 @@ def get_parser_descriptions(self): parserobj = self.load_parser_instance(parsername) if parserobj: descriptions.append((parsername, parserobj.author, parserobj.description)) + else: + descriptions.append((parsername, "Parser exists but failed to load.", "Parser exists but failed to load.")) self.__return_stdout() return descriptions except (Exception, SystemExit) as e: @@ -584,13 +585,14 @@ def output_file(self, data, filename, description=''): filename: filename (basename) of file description: description of the file ''' - self.outputfiles[filename] = {'data': data, 'description': description} basename = os.path.basename(filename) + md5 = hashlib.md5(data).hexdigest() + self.outputfiles[filename] = {'data': data, 'description': description, 'md5': md5} if self.__base64outputfiles: - self.add_metadata("outputfile", [basename, description, base64.b64encode(data)]) + self.add_metadata("outputfile", [basename, description, md5, base64.b64encode(data)]) else: - self.add_metadata("outputfile", [basename, description]) + self.add_metadata("outputfile", [basename, description, md5]) if self.__disableoutputfiles: return @@ -627,8 +629,8 @@ def format_list(self, values, key=None): if key == "credential" and len(values) == 2: return "%s:%s" % (values[0],values[1]) - elif key == "outputfile" and len(values) >= 2: - return "%s %s" % (values[0],values[1]) + elif key == "outputfile" and len(values) >= 3: + return "%s %s" % (values[0],values[1],values[2]) elif key == "port" and len(values) == 2: return "%s/%s" % (values[0],values[1]) elif key == "listenport" and len(values) == 2: @@ -645,67 +647,78 @@ def format_list(self, values, key=None): return ' '.join(values) def print_keyvalue(self, key, value): - printkey = key + print self.get_printable_key_value(key, value) + + def output_text(self): + ''' + Output in human readable report format + ''' - if sys.stdout.encoding: - encoding = sys.stdout.encoding - else: - encoding = "utf8" + print self.get_output_text().encode("utf8") + + def get_printable_key_value(self, key, value): + output = u"" + printkey = key if isinstance(value, basestring): - print((u'%-20s %s' % (printkey, value)).encode(encoding, 'backslashreplace')) + output += u"{:20} {}\n".format(printkey, value) else: for item in value: if isinstance(item, basestring): - print((u'%-20s %s' % (printkey, item)).encode(encoding, 'backslashreplace')) + output += u"{:20} {}\n".format(printkey, item) else: - print((u'%-20s %s' % (printkey, self.format_list(item, key=key))).encode(encoding, 'backslashreplace')) - printkey = "" - - def output_text(self): + output += u"{:20} {}\n".format(printkey, self.format_list(item, key=key)) + printkey = u"" + + return output + + def get_output_text(self): ''' - Output in human readable report format + Get data in human readable report format. ''' - + + output = u"" infoorderlist = INFO_FIELD_ORDER fieldorderlist = STANDARD_FIELD_ORDER if 'inputfilename' in self.metadata: - print("\n----File Information----\n") + output += u"\n----File Information----\n\n" for key in infoorderlist: if key in self.metadata: - self.print_keyvalue(key,self.metadata[key]) + output += self.get_printable_key_value(key, self.metadata[key]) - print("\n----Standard Metadata----\n") + output += u"\n----Standard Metadata----\n\n" for key in fieldorderlist: if key in self.metadata: - self.print_keyvalue(key,self.metadata[key]) + output += self.get_printable_key_value(key, self.metadata[key]) + #in case we have additional fields in fields.json but the order is not updated for key in self.metadata: if key not in fieldorderlist and key not in ["other", "debug", "outputfile"] and key in self.fields: - self.print_keyvalue(key,self.metadata[key]) + output += self.get_printable_key_value(key, self.metadata[key]) if "other" in self.metadata: - print("\n----Other Metadata----\n") + output += u"\n----Other Metadata----\n\n" for key in sorted(list(self.metadata["other"])): - self.print_keyvalue(key,self.metadata["other"][key]) + output += self.get_printable_key_value(key, self.metadata["other"][key]) if "debug" in self.metadata: - print("\n----Debug----\n") - #self.print_keyvalue("debug",self.metadata["debug"]) + output += u"\n----Debug----\n\n" for item in self.metadata["debug"]: - print(item) + output += u"{}\n".format(item) if "outputfile" in self.metadata: - print("\n----Output Files----\n") - for key, value in self.metadata["outputfile"]: - self.print_keyvalue(key,value) + output += u"\n----Output Files----\n\n" + for value in self.metadata["outputfile"]: + output += self.get_printable_key_value(value[0], (value[1], value[2])) if self.errors: - print("\n----Errors----\n") + output += u"\n----Errors----\n\n" for item in self.errors: - print(item) + output += u"{}\n".format(item) + + return output def __redirect_stdout(self): #we redirect stdout during execution of parser to trap the output diff --git a/modules/processing/parsers/mwcp/malwareconfigtester.py b/modules/processing/parsers/mwcp/malwareconfigtester.py index bde12b701..b8ec06916 100644 --- a/modules/processing/parsers/mwcp/malwareconfigtester.py +++ b/modules/processing/parsers/mwcp/malwareconfigtester.py @@ -1,18 +1,19 @@ ''' Test case support for DC3-MWCP. Parser output is stored in a json file per parser. To run test cases, parser is re-run and compared to previous results. - ''' + # Standard imports import os import json import glob - -# MWCP framework imports -#from mwcp.malwareconfigreporter import malwareconfigreporter +import traceback +import sys +import locale DEFAULT_EXCLUDE_FIELDS = ["debug"] + class malwareconfigtester(object): ''' DC3-MWCP test case class @@ -20,10 +21,6 @@ class malwareconfigtester(object): # Constants INPUT_FILE_PATH = "inputfilename" FILE_EXTENSION = ".json" - PARSER = "parser" - RESULTS = "results" - PASSED = "passed" - ERRORS = "errors" # Properties reporter = None @@ -38,13 +35,15 @@ def gen_results(self, parser_name, input_file_path): Generate JSON results for the given file using the given parser name. ''' - self.reporter.run_parser(parser_name, input_file_path) - self.reporter.metadata[self.INPUT_FILE_PATH] = input_file_path + try: + data = open(input_file_path, "rb").read() + self.reporter.run_parser(parser_name, data = data) + except: + self.reporter.error(traceback.print_exc()) - for error in self.reporter.errors: - print(error) + self.reporter.metadata[self.INPUT_FILE_PATH] = input_file_path - return self.reporter.metadata + return self.reporter.metadata def list_test_files(self, parser_name): ''' @@ -161,13 +160,13 @@ def run_tests(self, parser_names = None, field_names = None, ignore_field_names results_file_list = glob.glob(os.path.join(self.results_dir, "*{0}".format(self.FILE_EXTENSION))) all_test_results = [] - all_passed = True if not parser_names: parser_names = [] if not field_names: field_names = [] - # Determine files to test (this will be a list of JSON files) + # Determine files to test (this will be a list of JSON files). If no parser name(s) is specified, run + # all tests. test_case_file_paths = [] if len(parser_names) > 0: for parser_name in parser_names: @@ -176,8 +175,12 @@ def run_tests(self, parser_names = None, field_names = None, ignore_field_names if results_file_path in results_file_list: test_case_file_paths.append(results_file_path) else: - print "Results file not found for {0} parser".format(parser_name) - print "File not found = {0}".format(results_file_path) + print u"Results file not found for {} parser".format(parser_name).encode( + encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), + errors="replace") + print u"File not found = {}".format(results_file_path).encode( + encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), + errors="replace") else: test_case_file_paths = results_file_list @@ -188,31 +191,37 @@ def run_tests(self, parser_names = None, field_names = None, ignore_field_names for result_data in results_data: input_file_path = result_data[self.INPUT_FILE_PATH] + + # Rerun the file to get the most up to date parser results new_results = self.gen_results(parser_name, input_file_path) - passed, test_results = self.compare_results(result_data, new_results, field_names, ignore_field_names=ignore_field_names) - errors = False - if len(self.reporter.errors) > 0: + + # Compare the newly generated results to previously saved test results + comparer_results = self.compare_results(result_data, new_results, field_names, ignore_field_names=ignore_field_names) + + # Determine if any of field comparisons failed + passed = True + if any([not comparer.passed for comparer in comparer_results]): passed = False - errors = True - all_test_results.append({self.PARSER: parser_name, - self.INPUT_FILE_PATH: input_file_path, - self.PASSED: passed, - self.ERRORS: errors, - self.RESULTS: test_results}) - if not passed: - all_passed = False - - # Return tuple showing if any tests failed alongside more detailed results - return all_passed, all_test_results - - def compare_results(self, results_a, results_b, field_names = [], ignore_field_names = DEFAULT_EXCLUDE_FIELDS): + + # Track the test results + debug = self.reporter.metadata["debug"] if "debug" in self.reporter.metadata else None + test_result = TestResult(parser=parser_name, + input_file_path=input_file_path, + passed=passed, + errors=self.reporter.errors, + debug=debug, + results=comparer_results) + all_test_results.append(test_result) + + return all_test_results + + def compare_results(self, results_a, results_b, field_names=None, ignore_field_names=DEFAULT_EXCLUDE_FIELDS): ''' Compare two result sets. If the field names list is not empty, then only the fields (metadata key values) in the list will be compared. ignore_field_names fields are not compared unless included in field_names. ''' - passed = True - test_results = {} + results = [] if not field_names: field_names = [] @@ -227,89 +236,69 @@ def compare_results(self, results_a, results_b, field_names = [], ignore_field_n # Begin comparing results if len(field_names) > 0: for field_name in field_names: - # Ensure field name is in both result sets - if field_name in results_a and field_name in results_b: - compare_result = self.compare_results_field(results_a, results_b, field_name) - test_results[field_name] = compare_result - if compare_result == False: - passed = False - # Field in one result set but not the other - elif field_name in results_a or field_name in results_b: - test_results[field_name] = False - passed = False + try: + comparer = self.compare_results_field(results_a, results_b, field_name) + except: + comparer = ResultComparer(field_name) + self.reporter.error(traceback.format_exc()) + results.append(comparer) else: for ignore_field in ignore_field_names: - results_a.pop(ignore_field,None) - results_b.pop(ignore_field,None) - if set(results_a.keys()) != set(results_b.keys()): - passed = False - all_keys = set(results_a.keys()).union(results_b.keys()) - for key in all_keys: - test_results[key] = self.compare_results_field(results_a, results_b, key) - else: - for key in results_a: - compare_result = self.compare_results_field(results_a, results_b, key) - test_results[key] = compare_result - if compare_result == False: - passed = False - - return passed, test_results - + results_a.pop(ignore_field, None) + results_b.pop(ignore_field, None) + all_field_names = set(results_a.keys()).union(results_b.keys()) + for field_name in all_field_names: + try: + comparer = self.compare_results_field(results_a, results_b, field_name) + except: + comparer = ResultComparer(field_name) + self.reporter.error(traceback.format_exc()) + results.append(comparer) + + return results def compare_results_field(self, results_a, results_b, field_name): ''' Compare the values for a single results field in the two passed in results. - ''' - assert type(results_a) is dict and type(results_b) is dict + Args: + results_a (dict): MWCP generated result for a given file using a given parser. + results_b (dict): MWCP generated result for a given file using a given parser. + ''' # Check if provided field_name is a valid key (based on fields.json) try: field_name_u = self.reporter.convert_to_unicode(field_name) - except Exception as e: - print "Error comparing metadata due to failure converting key to unicode: %s" % (str(e)) - return False + except: + raise Exception("Failed to conver field name '{}' to unicode.".format(field_name)) - if field_name_u in self.reporter.fields: + try: field_type = self.reporter.fields[field_name_u]['type'] - else: - print "Error comparing metadata because %s is not an allowed key" % field_name_u - return False + except: + raise Exception("Key error. Field name '{}' was not identified as a standardized field.".format(field_name)) - # Confirm key is found in the passed in result sets - if field_name_u not in results_a or field_name_u not in results_b: - return False + # Establish value to send for comparison + value_a = None + value_b = None + if field_name_u in results_a: + value_a = results_a[field_name_u] + if field_name_u in results_b: + value_b = results_b[field_name_u] # Now compare results based on field type (see "fields.json" for more details) if field_type == "listofstrings": - return set(results_a[field_name_u]) == set(results_b[field_name_u]) + comparer = ListOfStringsComparer(field_name_u) + comparer.compare(value_a, value_b) elif field_type == "listofstringtuples": - set_list_a = [set(x) for x in results_a[field_name_u]] - set_list_b = [set(x) for x in results_b[field_name_u]] - - if len(set_list_a) != len(set_list_b): - return False - - for set_a in set_list_a: - if set_a not in set_list_b: - return False - - return True + comparer = ListOfStringTuplesComparer(field_name_u) + comparer.compare(value_a, value_b) elif field_type == "dictofstrings": - dict_a = results_a[field_name_u] - dict_b = results_b[field_name_u] - - if set(dict_a.keys()) != set(dict_b.keys()): - return False - - for key in dict_a: - if dict_a[key] != dict_b[key]: - return False - - return True + comparer = DictOfStringsComparer(field_name_u) + comparer.compare(value_a, value_b) + else: + raise Exception("Unhandled field type '{}' found for field name '{}'.".format(field_type, field_name)) - # This point shouldn't be reached unless the "fields.json" file has added field types added - return False + return comparer def print_test_results(self, test_results, failed_tests = True, passed_tests = True, verbose = False, json_format = False): ''' @@ -320,46 +309,181 @@ def print_test_results(self, test_results, failed_tests = True, passed_tests = T if json_format: filtered_output = [] for test_result in test_results: - passed = test_result[self.PASSED] + passed = test_result.passed if (passed and passed_tests) or (not passed and failed_tests): - if verbose: - filtered_output.append(test_result) - else: - filtered_result = {self.PARSER : test_result[self.PARSER], - self.INPUT_FILE_PATH: test_result[self.INPUT_FILE_PATH], - self.PASSED: test_result[self.PASSED]} - filtered_output.append(filtered_result) + filtered_output.append(test_result) - print json.dumps(filtered_output, indent = 4) + print json.dumps(filtered_output, indent=4, cls=MyEncoder) else: - separator = "" + separator = u"" for test_result in test_results: - filtered_output = "" - passed = test_result[self.PASSED] - if (passed and passed_tests) or (not passed and failed_tests): - filtered_output += "{0} = {1}\n".format(self.PARSER, test_result[self.PARSER]) - filtered_output += "{0} = {1}\n".format(self.INPUT_FILE_PATH, test_result[self.INPUT_FILE_PATH]) - filtered_output += "{0} = {1}\n".format(self.PASSED, test_result[self.PASSED]) - filtered_output += "{0} = {1}\n".format(self.ERRORS, test_result[self.ERRORS]) - - if verbose: - filtered_output += "{0}:\n".format(self.RESULTS) - for key in test_result[self.RESULTS]: - filtered_output += "\t{0} = {1}\n".format(key, test_result[self.RESULTS][key]) - if filtered_output != "": - filtered_output += "{0}\n".format(separator) - print filtered_output - - - - - - - - - + filtered_output = u"" + passed = test_result.passed + if passed and passed_tests: + filtered_output += u"Parser Name = {}\n".format(test_result.parser) + filtered_output += u"Input Filename = {}\n".format(test_result.input_file_path) + filtered_output += u"Tests Passed = {}\n".format(test_result.passed) + elif not passed and failed_tests: + filtered_output += u"Parser Name = {}\n".format(test_result.parser) + filtered_output += u"Input Filename = {}\n".format(test_result.input_file_path) + filtered_output += u"Tests Passed = {}\n".format(test_result.passed) + filtered_output += u"Errors = {}".format("\n" if test_result.errors else "None\n") + if test_result.errors: + for entry in test_result.errors: + filtered_output += u"\t{0}\n".format(entry) + filtered_output += u"Debug Logs = {}".format("\n" if test_result.debug else "None\n") + if test_result.debug: + for entry in test_result.debug: + filtered_output += u"\t{0}\n".format(entry) + filtered_output += u"Results =\n" + for result in test_result.results: + if not result.passed: + filtered_output += u"{0}\n".format(result) + + if filtered_output != u"": + filtered_output += u"{0}\n".format(separator) + print filtered_output.encode( + encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), + errors="replace") + + +#################################################### +# Result class simply to store data +#################################################### + +class TestResult(object): + + def __init__(self, parser, input_file_path, passed, errors=None, debug=None, results=None): + self.parser = parser + self.input_file_path = input_file_path + self.passed = passed + self.errors = [] if not errors else errors + self.debug = [] if not debug else debug + self.results = results + +#################################################### +# Comparer classes for various MWCP field types +#################################################### + +class ResultComparer(object): + + def __init__(self, field): + self.field = field + self.passed = False + self.missing = [] # Entries found in test case but not new results + self.unexpected = [] # Entries found in new results but not test case + + def compare(self, test_case_results=None, new_results=None): + '''Compare two result sets and document any differences.''' + + self.missing = [] + self.unexpected = [] + + self.field_compare(test_case_results, new_results) + + if self.missing or self.unexpected: + self.passed = False + else: + self.passed = True + def field_compare(self, test_case_results, new_results): + '''Field specific compare function.''' + # Override in child classes + raise NotImplementedError() + def get_report(self, json=False, tabs=1): + ''' + If json parameter is False, get report as a string. + If json parameter is True, get report as a dictionary. + ''' + if json: + return self.__dict__ + else: + tab = tabs * u"\t" + tab_1 = tab + u"\t" + tab_2 = tab_1 + u"\t" + report = tab + u"{}:\n".format(self.field) + report += tab_1 + u"Passed: {}\n".format(self.passed) + if self.missing: + report += tab_1 + u"Missing From New Results:\n" + for item in self.missing: + report += tab_2 + u"{}\n".format(item) + if self.unexpected: + report += tab_1 + u"Unexpected New Results:\n" + for item in self.unexpected: + report += tab_2 + u"{}\n".format(item) + + return report.rstrip() + + def __str__(self): + return self.get_report() + + def __repr__(self): + return self.__str__() + +class ListOfStringsComparer(ResultComparer): + def __init__(self, field): + super(ListOfStringsComparer, self).__init__(field) + + def field_compare(self, test_case_results, new_results): + """Compare each string in a list of strings.""" + list_test = [] if not test_case_results else test_case_results + list_new = [] if not new_results else new_results + + self.missing += list(set(list_test) - set(list_new)) + self.unexpected += list(set(list_new) - set(list_test)) + +class ListOfStringTuplesComparer(ResultComparer): + def __init__(self, field): + super(ListOfStringTuplesComparer, self).__init__(field) + + def field_compare(self, test_case_results, new_results): + """Compare each tuple of strings in a list of tuples.""" + set_list_test = [] + set_list_new = [] + if test_case_results: + set_list_test = [set(x) for x in test_case_results] + if new_results: + set_list_new = [set(x) for x in new_results] + + for set_test in set_list_test: + if set_test not in set_list_new: + # Append the list entry here instead of the set to preserve the entries ordering + self.missing.append(test_case_results[set_list_test.index(set_test)]) + + for set_new in set_list_new: + if set_new not in set_list_test: + # Append the list entry here instead of the set to preserve the entries ordering + self.unexpected.append(new_results[set_list_new.index(set_new)]) + +class DictOfStringsComparer(ResultComparer): + def __init__(self, field): + super(DictOfStringsComparer, self).__init__(field) + + def field_compare(self, test_case_results, new_results): + """Compare each key value pair in a dictionary of strings.""" + dict_test = {} if not test_case_results else test_case_results + dict_new = {} if not new_results else new_results + + for key in dict_test: + if key not in dict_new: + self.missing.append("{}: {}".format(key, dict_test[key])) + elif set(dict_test[key]) != set(dict_new[key]): + self.missing.append("{}: {}".format(key, dict_test[key])) + + for key in dict_new: + if key not in dict_test: + self.unexpected.append("{}: {}".format(key, dict_new[key])) + elif set(dict_new[key]) != set(dict_test[key]): + self.unexpected.append("{}: {}".format(key, dict_new[key])) + +#################################################### +# JSON encoders +#################################################### + +class MyEncoder(json.JSONEncoder): + def default(self, o): + return o.__dict__ diff --git a/modules/processing/parsers/mwcp/parsers/EvilGrab.py b/modules/processing/parsers/mwcp/parsers/EvilGrab.py index 75f11ec0b..772863b67 100644 --- a/modules/processing/parsers/mwcp/parsers/EvilGrab.py +++ b/modules/processing/parsers/mwcp/parsers/EvilGrab.py @@ -83,59 +83,75 @@ def run(self): yara_offset = int(type1['$configure1']) c2_address = string_from_va(pe, yara_offset+24) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) port = str(struct.unpack('h', filebuf[yara_offset+71:yara_offset+73])[0]) - self.reporter.add_metadata('port', [port, "tcp"]) + if port: + self.reporter.add_metadata('port', [port, "tcp"]) missionid = string_from_va(pe, yara_offset+60) - self.reporter.add_metadata('missionid', missionid) + if missionid: + self.reporter.add_metadata('missionid', missionid) version = string_from_va(pe, yara_offset+90) - self.reporter.add_metadata('version', version) + if version: + self.reporter.add_metadata('version', version) injectionprocess = string_from_va(pe, yara_offset+132) - self.reporter.add_metadata('injectionprocess', injectionprocess) + if injectionprocess: + self.reporter.add_metadata('injectionprocess', injectionprocess) mutex = string_from_va(pe, yara_offset-186) - self.reporter.add_metadata('mutex', mutex) + if mutex: + self.reporter.add_metadata('mutex', mutex) if type2: yara_offset = int(type2['$configure2']) c2_address = string_from_va(pe, yara_offset+24) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) port = str(struct.unpack('h', filebuf[yara_offset+78:yara_offset+80])[0]) - self.reporter.add_metadata('port', [port, "tcp"]) + if port: + self.reporter.add_metadata('port', [port, "tcp"]) missionid = string_from_va(pe, yara_offset+67) - self.reporter.add_metadata('missionid', missionid) + if missionid: + self.reporter.add_metadata('missionid', missionid) version = string_from_va(pe, yara_offset+91) - self.reporter.add_metadata('version', version) + if version: + self.reporter.add_metadata('version', version) injectionprocess = string_from_va(pe, yara_offset+133) - self.reporter.add_metadata('injectionprocess', injectionprocess) + if injectionprocess: + self.reporter.add_metadata('injectionprocess', injectionprocess) mutex = string_from_va(pe, yara_offset-188) - self.reporter.add_metadata('mutex', mutex) + if mutex: + self.reporter.add_metadata('mutex', mutex) if type3: yara_offset = int(type3['$configure3']) c2_address = string_from_va(pe, yara_offset+38) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) port = str(struct.unpack('h', filebuf[yara_offset+99:yara_offset+101])[0]) - self.reporter.add_metadata('port', [port, "tcp"]) + if port: + self.reporter.add_metadata('port', [port, "tcp"]) missionid = string_from_va(pe, yara_offset+132) - self.reporter.add_metadata('missionid', missionid) + if missionid: + self.reporter.add_metadata('missionid', missionid) version = string_from_va(pe, yara_offset+167) - self.reporter.add_metadata('version', version) + if version: + self.reporter.add_metadata('version', version) injectionprocess = string_from_va(pe, yara_offset+195) - self.reporter.add_metadata('injectionprocess', injectionprocess) - \ No newline at end of file + if injectionprocess: + self.reporter.add_metadata('injectionprocess', injectionprocess) diff --git a/modules/processing/parsers/mwcp/parsers/HttpBrowser.py b/modules/processing/parsers/mwcp/parsers/HttpBrowser.py index 7e926daa7..d8eb34761 100644 --- a/modules/processing/parsers/mwcp/parsers/HttpBrowser.py +++ b/modules/processing/parsers/mwcp/parsers/HttpBrowser.py @@ -91,41 +91,50 @@ def run(self): yara_offset = int(type1['$connect_1']) port = ascii_from_va(pe, yara_offset+39) - self.reporter.add_metadata('port', [port, "tcp"]) + if port: + self.reporter.add_metadata('port', [port, "tcp"]) c2_address = unicode_from_va(pe, yara_offset+49) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) if type2: yara_offset = int(type2['$connect_2']) port = ascii_from_va(pe, yara_offset+35) - self.reporter.add_metadata('port', [port, "tcp"]) + if port: + self.reporter.add_metadata('port', [port, "tcp"]) c2_address = unicode_from_va(pe, yara_offset+45) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) if type3: yara_offset = int(type3['$connect_3']) port = ascii_from_va(pe, yara_offset+18) - self.reporter.add_metadata('port', [port, "tcp"]) + if port: + self.reporter.add_metadata('port', [port, "tcp"]) c2_address = unicode_from_va(pe, yara_offset+28) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) c2_address = unicode_from_va(pe, yara_offset+66) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) if type4: yara_offset = int(type4['$connect_4']) c2_address = unicode_from_va(pe, yara_offset+35) - self.reporter.add_metadata('c2_address', c2_address) + if c2_address: + self.reporter.add_metadata('c2_address', c2_address) filepath = unicode_from_va(pe, yara_offset+90) - self.reporter.add_metadata('filepath', filepath) + if filepath: + self.reporter.add_metadata('filepath', filepath) injectionprocess = unicode_from_va(pe, yara_offset-13) - self.reporter.add_metadata('injectionprocess', injectionprocess) - \ No newline at end of file + if injectionprocess: + self.reporter.add_metadata('injectionprocess', injectionprocess) diff --git a/modules/processing/parsers/mwcp/resources/fields.json b/modules/processing/parsers/mwcp/resources/fields.json index 4f9008b84..a41492f87 100644 --- a/modules/processing/parsers/mwcp/resources/fields.json +++ b/modules/processing/parsers/mwcp/resources/fields.json @@ -16,7 +16,7 @@ "type": "listofstrings" }, "c2_socketaddress": { - "description": "special case of scoketaddress, when socketaddress is used for command and control", + "description": "special case of socketaddress, when socketaddress is used for command and control", "examples": [ [ "bad.com", @@ -32,7 +32,7 @@ "type": "listofstringtuples" }, "c2_url": { - "description": "special case of address, when the address is used for command and control", + "description": "special case of url, when the url is used for command and control", "examples": [ "http://mal.com/pub/view.asp", "https://10.11.10.13:443/images/baner.jpg" @@ -56,7 +56,7 @@ "debug": { "description": "Message used for debugging or to report errors. Not malware configuration per se", "examples": [ - "Succesfully found config block, attempting decode", + "Successfully found config block, attempting decode", "Decode failed: detected config location 3735928559 outside file size of 184320" ], "type": "listofstrings" @@ -86,7 +86,7 @@ "type": "listofstrings" }, "injectionprocess": { - "description": "process into which malware is injected. Usually this is a processname but it may take other forms such as a filename of the executable.", + "description": "process into which malware is injected. Usually this is a process name but it may take other forms such as a filename of the executable.", "examples": [ "iexplore", "svchost" @@ -152,15 +152,17 @@ "type": "dictofstrings" }, "outputfile": { - "description": "relevant or related file created during parsing of malware. Tuple of filename and description", + "description": "relevant or related file created during parsing of malware. Tuple of filename, description, and md5.", "examples": [ [ "config.xml", - "extracted backdoor Foo config file" + "extracted backdoor Foo config file", + "12345678901234567890123456789012" ], [ "client.crt", - "certificate for BarRat TLS client authentication" + "certificate for BarRat TLS client authentication", + "09876543210987654321098765432109" ] ], "type": "listofstringtuples" @@ -174,7 +176,7 @@ "type": "listofstrings" }, "port": { - "description": "TCP or UDP port. A tuple of a port number and protocol. This generally refers to outbound connections where the malware is the client. Other network layer protcols, such as ICMP can be represented here. Application layer protocols, such as HTTP, should be indicated in a URL.", + "description": "TCP or UDP port. A tuple of a port number and protocol. This generally refers to outbound connections where the malware is the client. Other network layer protocols, such as ICMP can be represented here. Application layer protocols, such as HTTP, should be indicated in a URL.", "examples": [ [ "21", diff --git a/modules/reporting/submitCAPE.py b/modules/reporting/submitCAPE.py index a5e73c631..264d32827 100644 --- a/modules/reporting/submitCAPE.py +++ b/modules/reporting/submitCAPE.py @@ -230,7 +230,7 @@ def run(self, results): self.task_options = self.task["options"] # we want to switch off automatic process dumps in CAPE submissions - if 'procmemdump=1' in self.task_options: + if self.task_options and 'procmemdump=1' in self.task_options: self.task_options = self.task_options.replace(u"procmemdump=1", u"procmemdump=0", 1) if self.task_options_stack: self.task_options=','.join(self.task_options_stack)