diff --git a/splunk_search_service/DEPENDENCIES b/splunk_search_service/DEPENDENCIES new file mode 100644 index 00000000..6d46dfae --- /dev/null +++ b/splunk_search_service/DEPENDENCIES @@ -0,0 +1,5 @@ +The Splunk Search service requires the requests library [1] in order to function. + +[1] http://docs.python-requests.org/ + +sudo pip install requests diff --git a/splunk_search_service/LICENSE b/splunk_search_service/LICENSE new file mode 100644 index 00000000..73490d9d --- /dev/null +++ b/splunk_search_service/LICENSE @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2016, The MITRE Corporation. All rights reserved. + +Approved for Public Release; Distribution Unlimited 14-1511 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/splunk_search_service/README b/splunk_search_service/README new file mode 100644 index 00000000..474da3f8 --- /dev/null +++ b/splunk_search_service/README @@ -0,0 +1,61 @@ +The Splunk Search service works against Samples, Emails, Indicators, and Raw Data. +It will grab all metadata of the top-level object (TLO) and use that data to run +predetermined Splunk searches found in searches.json. + +Additionally, it has the option to parse out the contents of the TLO, and use +any domains, IPs, email addresses, hashes, or URLs it may find in additional +Splunk searches. + +There is also an option to exclude certain filetypes matching a given regex +if the service is launched based on a triage, rather than being run manually. + +For example, if an email is uploaded that contains multiple JPEG attachments +(for which a Splunk search of their hash may not be necessary in most cases), +when the JPEGs are added to CRITs as Sample TLOs, and Splunk is set to run on +triage, the default regex will tell the service not to execute those Splunk +searches. However, if it is determined later that running those searches for a +particular JPEG may be of some use, manually launching the service will run +the searches. + +### Configuring ### + +To configure your custom Splunk searches, modify searches.json with valid JSON, +maintaining the predetermined structure. TLOs (including emails, samples, and +indicators) should be referenced with their TLO object attributes. Lists of +potential indicators (domains, IPs, urls, email address, and hashes) should be +passed with a set of brackets. + +Additionally, each search name should have a unique value, otherwise it may not +display properly. + +Lastly, ensure any double-quotes are escaped with a backslash. + +For example, if you wished to run a search for a sample's MD5 when the service +is run, you could make the list of 'samples' searches look like this +(substituting your own indicies and field names), where {md5} will be replaced +with the TLO's md5 attribute: + +"samples": [ + {"name":"MD5 Search for {md5}", + "search":"index=files md5=\"{md5}\" | stats values(dest_ip) AS dest_ip values(uid) As uid values(fuid) AS fuid count by src_ip" + } + ], + +Likewise, for a search that would run against every domain in a list of domains +extracted from the TLO's data, you could make the list of 'domains' searches +look like this, where {} will be replaced with the domain: + +"domains": [ + {"name":"HTTP search for {}", + "search":"index=http domain=\"{}\" | stats values(src_ip) AS src_ip values(url) AS url count by index" + }, + {"name":"SMTP search for {}", + "search":"index=smtp sender_domain=\"*{}\" | stats values(src_ip) AS src_ip values(src_user) AS src_user count by subject" + } + ], + +Please see searches_example.json for additional examples. + +You may use test_searches.py with your configurations manually inputted to test +your Splunk searches and their results before committing them to your CRITs +production instance. diff --git a/splunk_search_service/__init__.py b/splunk_search_service/__init__.py new file mode 100644 index 00000000..3eda404b --- /dev/null +++ b/splunk_search_service/__init__.py @@ -0,0 +1,610 @@ +import re +import logging +import requests, time, json +from lxml import objectify +from urllib import quote_plus + +from django.conf import settings +from django.template.loader import render_to_string + +from crits.services.core import Service, ServiceConfigError +from crits.services.handlers import get_service_config +from crits.emails.email import Email +from crits.events.event import Event +from crits.raw_data.raw_data import RawData +from crits.samples.sample import Sample +from crits.domains.domain import TLD +from crits.indicators.indicator import Indicator +from crits.core.data_tools import make_ascii_strings +from crits.vocabulary.indicators import IndicatorTypes, IndicatorThreatTypes + +from . import forms +from searches import SplunkSearches + +logger = logging.getLogger(__name__) + + +class SplunkSearchService(Service): + """ + Craft custom Splunk searches from metadata gathered from a given TLO. + + Currently this service only runs on RawData, Samples, and Emails. + """ + + name = "SplunkSearch" + version = '1.0.0' + template = "splunk_search_service_template.html" + supported_types = ['RawData', 'Sample', 'Email', 'Indicator'] + description = "Craft custom Splunk searches based on metadata." + + + #### Handle configs stuff #### + @staticmethod + def parse_config(config): + if not config['splunk_url']: + raise ServiceConfigError("Splunk URL required.") + if not config['splunk_user']: + raise ServiceConfigError("Splunk username required.") + if not config['password']: + raise ServiceConfigError("Splunk password required.") + + @staticmethod + def get_config(existing_config): + # Generate default config from form and initial values + config = {} + fields = forms.SplunkSearchConfigForm().fields + for name, field in fields.iteritems(): + config[name] = field.initial + + # If there is a config in the database, use values from that + if existing_config: + for key, value in existing_config.iteritems(): + config[key] = value + + return config + + @staticmethod + def get_config_details(config): + display_config = {} + + # Rename keys so they render nicely. + fields = forms.SplunkSearchConfigForm().fields + for name, field in fields.iteritems(): + display_config[field.label] = config[name] + + return display_config + + @classmethod + def generate_config_form(self, config): + html = render_to_string('services_config_form.html', + {'name': self.name, + 'form': forms.SplunkSearchConfigForm(initial=config), + 'config_error': None}) + form = forms.SplunkSearchConfigForm + return form, html + + @staticmethod + def save_runtime_config(config): + del config ['splunk_user'] + del config ['password'] + + @staticmethod + def valid_for(obj): + if isinstance(obj, Sample): + if obj.filedata.grid_id == None: + raise ServiceConfigError("Missing filedata.") + + def run(self, obj, config): + self.config = config + + # Debug + self._debug("Is triage run: %s" % str(self.is_triage_run)) + + # Check if this is a triage run + if self.is_triage_run: + if isinstance(obj,Sample): + filetype = str(obj.filetype) + pattern = str(self.config['ignore_filetypes']) + if re.match(pattern, filetype): + self._info("Search was not run on triage because %s matched on the regex %s" % (filetype, str(pattern))) + return + + #if isinstance(obj, Event): + # data = obj.description + #elif isinstance(obj, RawData): + if isinstance(obj, RawData): + data = obj.data + elif isinstance(obj, Email): + data = obj.raw_body + elif isinstance(obj, Sample): + samp_data = obj.filedata.read() + data = make_ascii_strings(data=samp_data) + if not data: + self._debug("Could not find sample data to parse.") + return + elif isinstance(obj, Indicator): + data = "" + else: + self._debug("This type is not supported by this service.") + return + + #Debug + #self._debug(str(self.config)) + ips = [] + domains = [] + urls = [] + emails = [] + hashes = [] + + all_splunk_searches = [] + + ''' + if self.config['data_miner']==True: + ips = dedup(extract_ips(data)) + domains = dedup(extract_domains(data)) + urls = dedup(extract_urls(data)) + emails = dedup(extract_emails(data)) + hashes = dedup(extract_hashes(data)) + + # Run searches based on DataMiner results + datamined = {'urls': urls, + 'domains': domains, + 'ips': ips, + 'hashes': hashes, + 'emails': emails} + splunk_obj = SplunkSearches(datamined,self.config['search_config']) + splunk_searches_datamined = splunk_obj.datamined() + all_splunk_searches.append(splunk_searches_datamined) + ''' + + if self.config['url_search']==True: + urls = dedup(extract_urls(data)) + # Run searches for URLs + splunk_obj = SplunkSearches(urls,self.config['search_config']) + splunk_searches_urls = splunk_obj.url_searches() + all_splunk_searches.append(splunk_searches_urls) + + if self.config['domain_search']==True: + domains = dedup(extract_domains(data)) + # Run searches for domains + splunk_obj = SplunkSearches(domains,self.config['search_config']) + splunk_searches_domains = splunk_obj.domain_searches() + all_splunk_searches.append(splunk_searches_domains) + + if self.config['ip_search']==True: + ips = dedup(extract_ips(data)) + # Run searches for IPs + splunk_obj = SplunkSearches(ips,self.config['search_config']) + splunk_searches_ips = splunk_obj.ip_searches() + all_splunk_searches.append(splunk_searches_ips) + + if self.config['hash_search']==True: + hashes = dedup(extract_hashes(data)) + # Run searches for hashes + splunk_obj = SplunkSearches(hashes,self.config['search_config']) + splunk_searches_hashes = splunk_obj.hash_searches() + all_splunk_searches.append(splunk_searches_hashes) + + if self.config['email_addy_search']==True: + email_addys = dedup(extract_emails(data)) + # Run searches for Email addresses + splunk_obj = SplunkSearches(email_addys,self.config['search_config']) + splunk_searches_email_addys = splunk_obj.email_addy_searches() + all_splunk_searches.append(splunk_searches_email_addys) + + # Set splunk_obj for TLO + splunk_obj = SplunkSearches(obj,self.config['search_config']) + + if isinstance(obj, Email): + splunk_searches = splunk_obj.email_searches() + all_splunk_searches.append(splunk_searches) + #self._debug(str(splunk_searches)) + + elif isinstance(obj, Sample): + splunk_searches = splunk_obj.sample_searches() + all_splunk_searches.append(splunk_searches) + + elif isinstance(obj, Indicator): + splunk_searches = splunk_obj.indicator_searches() + all_splunk_searches.append(splunk_searches) + + ''' + all_splunk_searches = [{'description': 'Searches Splunk based on email attibutes', + 'searches': [{'name': 'Searching for email subjects', + 'search': 'search index=smtp subject=blah'}] + }, + {'description': 'Searches Splunk based on data mined', + 'searches': [{'name': 'Searching for domain in http', + 'search': 'search index=http domain=badguy.com'}] + } + ] + ''' + + # Once splunk_searches is finally popluated, loop through the potential searches + full_search_dict = {} + self._debug("all_splunk_searches: %s" % str(all_splunk_searches)) + all_jobs = {} + + # Get the Session Key for talking with Splunk + sessionKey = get_splunk_session_key(self.config) + self._debug("Session key %s obtained." % sessionKey) + + splunk_results = [] + + for search_group in all_splunk_searches: + if 'searches' in search_group and search_group['searches']: + for search in search_group['searches']: + if search['search']!="": + ## Set the timeframe and search limit + search['search']="earliest="+self.config['search_earliest']+" "+search['search'] + search['search']+="|head "+self.config['search_limit'] + # Make sure it starts with 'search' or a | + if not (search['search'].startswith("search") or search['search'].startswith("|")): + search['search']="search "+search['search'] + + # Start a Splunk Search Job + job_sid = start_splunk_search_job(self.config, sessionKey, search['search']) + + # Add the job_sid to a list of jobs to poll + all_jobs[search['name']]=job_sid + + + # Build a dict of search names and their searches + search_base=self.config['splunk_browse_url']+"en-US/app/search/search?q=" + full_search=search_base+quote_plus(search['search']) + full_search_dict[search['name']] = full_search + + + # Poll the jobs now that we have a list of sids + self._debug("Checking for results from the following jobs: %s" % str(all_jobs)) + poll_results = self.poll_jobs(self.config, sessionKey, all_jobs) + self._debug("Results of the polling: %s" % str(poll_results)) + + ''' + poll_results = {"done":[{'badguy.com': ''}, + {'goodguy.com': ''}], + "timeout":[], + "failed":[]} + ''' + + + # Log the results of the polls + + # Log errors + for item in poll_results['timeout']: + for k, v in item.iteritems(): + self._info("Splunk job %s for the search %s timed out." % (v,k)) + results = {k : {'results': [{'timeout':v}]}} + splunk_results.append(results) + for item in poll_results['failed']: + for k, v in item.iteritems(): + self._info("Splunk job %s for the search %s FAILED. Check Splunk for deatils." % (v,k)) + results = {k : {'results': [{'failed':v}]}} + splunk_results.append(results) + + # Get the results + for item in poll_results['done']: + self._debug("Sending the following item to Splunk to get the results: %s" % str(item)) + splunk_results.append(get_search_results(self.config, sessionKey, item)) + self._debug("splunk_results = %s" % str(splunk_results)) + + # Parse the results and add to database + + # splunk_results = [{'search for emails': }, + # {'search for domains': }] + + ''' + # splunk_results for testing + splunk_results = [{'test search 1': {'fields': [{'name': 'domain'}, + {'groupby_rank': '0', 'name': 'src_ip'}, + {'name': 'count'}], + 'results': [{'count': '1', + 'domain': 'something.com', + 'src_ip': '192.168.5.1'}, + {'count': '1', + 'domain': 'else.com', + 'src_ip': '192.168.5.2'}] + } + }, + {'test search 2': {'fields': [{'groupby_rank': '0', 'name': 'domain'}, + {'name': 'src_ip'}, + {'name': 'count'}], + 'results': [{'count': '1', + 'domain': 'badguy.com', + 'src_ip': '192.168.1.1'}, + {'count': '1', + 'domain': 'goodguy.com', + 'src_ip': '192.168.1.2'}] + } + }] + ''' + for splunk_result in splunk_results: + for name, results in splunk_result.iteritems(): + # Build the header + tdict = {"fields": [], + "full_search_dict": full_search_dict} + field_count = 0 + self._debug("%s search results = %s" % (name, str(results))) + if 'fields' in results: + for header in results['fields']: + for k, v in header.iteritems(): + if k=='name': + if 'groupby_rank' in results['fields'][field_count]: + tdict['fields'].insert(int(results['fields'][field_count]['groupby_rank']), + results['fields'][field_count]['name']) + else: + tdict['fields'].append(v) + field_count+=1 + # Build the search results + new_results = [] + for event in results['results']: + # Set single string results to a list so all search results are in a list + event = {x : ([y] if isinstance(y,unicode) else y) for x,y in event.iteritems()} + + # Check if there's a field from this event that doesn't have any data + for field in tdict['fields']: + if field not in event or not event[field]: + event[field]=['-'] + new_results.append(event) + + results['results']=new_results + # Save it to the actual service results... Finally... + self._add_result(name, results['results'], tdict) + + # If 'fields' isn't in the results, something went wrong with the search, + # or it didn't get any hits. + else: + search_error=False + + for item in poll_results['failed']: + for k,v in item.iteritems(): + if k==name: + search_error=True + tdict['fields']=['search_failed'] + results['results']=[{'search_failed': + ["Search ID %s failed. Check Splunk for details." % v]}] + self._add_result(name, results['results'], tdict) + + for item in poll_results['timeout']: + for k,v in item.iteritems(): + if k==name: + search_error=True + tdict['fields']=['search_timeout'] + results['results']=[{'search_timeout': + ["Search ID %s timed out. Check Splunk for details." % v]}] + self._add_result(name, results['results'], tdict) + + if search_error==False: + tdict['fields']=['no_matches'] + results['results']=[{'no_matches': ["No matches were found for this search"]}] + self._add_result(name, results['results'], tdict) + + + # Poll Job IDs + def poll_jobs(self, configs, sessionKey, all_jobs): + splunk_url = configs['splunk_url'] + splunk_timeout = configs['splunk_timeout'] + headers = {"Authorization":"Splunk "+str(sessionKey)} + search_link = splunk_url+"services/search/jobs/" + job_results = {"done":[], + "timeout":[], + "failed":[]} + first_run = True + + # Loop through the full list of jobs, polling status every 2 seconds + # until runDuration > configs['splunk_timeout']. Waits 5 seconds + # before first poll b/c c'mon... Splunk's not that fast. + # + # Also, if dispatchState or runDuration are not returning, something + # probably went wrong... So it breaks the while loop and returns whatever + # results it's grabbed. + + + while all_jobs: + if first_run == True: + time.sleep(5) + first_run = False + ''' + all_jobs = {'badguy.com': '', + 'goodguy.com': ''} + ''' + for name, job_sid in all_jobs.iteritems(): + runDuration = '' + dispatchState = '' + search_check = requests.get(search_link+job_sid, headers=headers, verify=False) + scheck_root = objectify.fromstring(search_check._content) + for a in scheck_root.getchildren(): + for b in a.getchildren(): + for c in b.getchildren(): + if 'name' in c.attrib and c.attrib['name']=='dispatchState': + dispatchState = c.text + if 'name' in c.attrib and c.attrib['name']=='runDuration': + runDuration = c.text + + # Emergency STOP if runDuration or dispatchState aren't found... + self._debug("Search is %s. Run duration: %s" % (dispatchState, runDuration)) + if dispatchState=='': + self._debug("dispatchState was not returned for some reason.") + all_jobs = {x : y for x,y in all_jobs.iteritems() if x!=x} + return job_results + if dispatchState!="QUEUED" and runDuration=='': + self._debug("runDuration was not returned for some reason.") + all_jobs = {x : y for x,y in all_jobs.iteritems() if x!=x} + return job_results + + if dispatchState=="DONE": + #job_results['done'].append(job_sid) + #all_jobs.remove(job_sid) + job_results['done'].append({name:job_sid}) + all_jobs = {x : y for x,y in all_jobs.iteritems() if name!=x} + elif dispatchState=="FAILED": + #job_results['failed'].append(job_sid) + #all_jobs.remove(job_sid) + job_results['failed'].append({name:job_sid}) + all_jobs = {x : y for x,y in all_jobs.iteritems() if name!=x} + if runDuration!='' and float(runDuration) > int(splunk_timeout): + #job_results['timeout'].append(job_sid) + #all_jobs.remove(job_sid) + job_results['timeout'].append({name:job_sid}) + all_jobs = {x : y for x,y in all_jobs.iteritems() if name!=x} + else: + self._debug("Search %s is currently %s. It has been running for %s seconds." + % (job_sid, dispatchState, runDuration)) + + # Sleep 2 seconds between checking all jobs + time.sleep(2) + + ''' + job_results = {"done":[{'badguy.com': ''}, + {'goodguy.com': ''}], + "timeout":[], + "failed":[]} + ''' + return job_results + + +# Get Search Results +def get_search_results(configs, sessionKey, name_sid): + splunk_url = configs['splunk_url'] + headers = {"Authorization":"Splunk "+str(sessionKey)} + data = {'output_mode': 'json'} + json_response={} + for name, sid in name_sid.iteritems(): + search_link = splunk_url+"services/search/jobs/"+sid+"/results/" + search_rez = requests.get(search_link, headers=headers, params=data, verify=False) + json_response[name] = json.loads(search_rez._content) + + ''' + json_response = {'search for emails': } + ''' + + return json_response + +# Start Splunk Search Job +def start_splunk_search_job(configs, sessionKey, search): + splunk_url = configs['splunk_url'] + headers = {"Authorization":"Splunk "+str(sessionKey)} + + search_link = splunk_url+"services/search/jobs" + search_data = {"search": search} + search_resp = requests.post(search_link, headers=headers, data=search_data, verify=False) + search_root = objectify.fromstring(search_resp._content) + job_sid = str(search_root['sid']) + + return job_sid + +# Get a Splunk session ID +def get_splunk_session_key(configs): + splunk_url = configs['splunk_url'] + login_link = "services/auth/login" + login_data = {"username": configs['splunk_user'], + "password": configs['password']} + auth_link = splunk_url+login_link + resp = requests.get(auth_link, data=login_data, verify=False) + root = objectify.fromstring(resp._content) + sessionKey = root['sessionKey'] + + return sessionKey + +# Deduplicate results from DataMiner +def dedup(seq): + seen = set() + seen_add = seen.add + return [x for x in seq if not (x in seen or seen_add(x))] + +# hack of a parser to extract potential ip addresses from data +def extract_ips(data): + pattern = r"((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)([ (\[]?(\.|dot)[ )\]]?(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3})" + ips = [each[0] for each in re.findall(pattern, data)] + for item in ips: + location = ips.index(item) + ip = re.sub("[ ()\[\]]", "", item) + ip = re.sub("dot", ".", ip) + ips.remove(item) + ips.insert(location, ip) + return ips + +# hack of a parser to extract potential domains from data +def extract_domains(data): + pattern = r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?[\.[a-zA-Z]{2,}' + domains = [each for each in re.findall(pattern, data) if len(each) > 0] + final_domains = [] + for item in domains: + if len(item) > 1 and item.find('.') != -1: + try: + tld = item.split(".")[-1] + check = TLD.objects(tld=tld).first() + if check: + final_domains.append(item) + except: + pass + return final_domains + +# hack of a parser to extract potential URIs from data +''' +def extract_uris(data): + pattern = r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?' + results = re.findall(pattern,data) + #domains = [each[1] for each in results if len(each) > 0] + uris = [each[2] for each in results if len(each) > 0] + final_uris = [] + for item in uris: + final_uris.append(item) + return final_uris +''' + +# hack of a parser to extract potential URLs (Links) from data +def extract_urls(data): + pattern = r'(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?' + results = re.findall(pattern,data) + urls = [each for each in results if len(each) >0] + final_urls = [] + for item in urls: + url = item[0]+"://"+item[1]+item[2] + final_urls.append(url) + return final_urls + + +# hack of a parser to extract potential emails from data +def extract_emails(data): + pattern = r'[a-zA-Z0-9-\.\+]+@.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?[\.[a-zA-Z]{2,}' + emails = [each for each in re.findall(pattern, data) if len(each) > 0] + final_emails = [] + for item in emails: + if len(item) > 1 and item.find('.') != -1: + try: + tld = item.split(".")[-1] + check = TLD.objects(tld=tld).first() + if check: + final_emails.append(item) + except: + pass + return final_emails + +# hack of a parser to extract potential domains from data +def extract_hashes(data): + + re_md5 = re.compile("\\b[a-f0-9]{32}\\b", re.I | re.S | re.M) + re_sha1 = re.compile("\\b[a-f0-9]{40}\\b", re.I | re.S | re.M) + re_sha256 = re.compile("\\b[a-f0-9]{64}\\b", re.I | re.S | re.M) + re_ssdeep = re.compile("\\b\\d{2}:[A-Za-z0-9/+]{3,}:[A-Za-z0-9/+]{3,}\\b", re.I | re.S | re.M) + + final_hashes = [] + md5 = IndicatorTypes.MD5 + sha1 = IndicatorTypes.SHA1 + sha256 = IndicatorTypes.SHA256 + ssdeep = IndicatorTypes.SSDEEP + final_hashes.extend( + [(md5,each) for each in re.findall(re_md5, data) if len(each) > 0] + ) + final_hashes.extend( + [(sha1,each) for each in re.findall(re_sha1, data) if len(each) > 0] + ) + final_hashes.extend( + [(sha256,each) for each in re.findall(re_sha256, data) if len(each) > 0] + ) + final_hashes.extend( + [(ssdeep,each) for each in re.findall(re_ssdeep, data) if len(each) > 0] + ) + return final_hashes + diff --git a/splunk_search_service/bootstrap b/splunk_search_service/bootstrap new file mode 100644 index 00000000..41dfdb30 --- /dev/null +++ b/splunk_search_service/bootstrap @@ -0,0 +1,147 @@ +#!/bin/sh +# (c) 2016, The MITRE Corporation. All rights reserved. +# Source code distributed pursuant to license agreement. +# +# Usage: bootstrap +# This script is designed to install all of the necessary dependencies for the +# service. + +. ../funcs.sh + +ubuntu_install() +{ + printf "${HEAD}Installing dependencies with apt-get${END}\n" + #sudo apt-add-repository universe + #sudo apt-get update + #sudo apt-get install -y --fix-missing libchm1 clamav upx wireshark + if [ $? -eq 0 ] + then + printf "${PASS}Ubuntu Install Complete${END}\n" + else + printf "${FAIL}Ubuntu Install Failed${END}\n" + fi + sudo ldconfig +} + +debian_install() +{ + printf "${HEAD}Installing dependencies with apt-get${END}\n" + #sudo apt-add-repository universe + #sudo apt-get update + #sudo apt-get install -y --fix-missing libchm1 clamav upx wireshark + if [ $? -eq 0 ] + then + printf "${PASS}Debian Install Complete${END}\n" + else + printf "${FAIL}Debian Install Failed${END}\n" + fi + sudo ldconfig +} + +darwin_install() +{ + command -v brew >/dev/null 2>&1 || { + printf "${HEAD}Installation for OSX requires Homebrew. Please visit http://brew.sh/.${END}\n" + exit + } + #brew install chmlib clamav wireshark upx + if [ $? -eq 0 ] + then + printf "${PASS}Homebrew Install Complete${END}\n" + else + printf "${FAIL}Homebrew Install Failed${END}\n" + fi +} + +freebsd_install() +{ + #printf "${HEAD}Installing Ports${END}\n" + #sudo pkg install libchm1 clamav wireshark upx + if [ $? -eq 0 ] + then + printf "${PASS}Ports Install Complete${END}\n" + else + printf "${FAIL}Ports Install Failed${END}\n" + fi +} + +red_hat_install() +{ + #printf "${HEAD}Installing Yum Packages${END}\n" + #sudo yum install upx-3.07-1 libchm1 clamav wireshark upx + if [ $? -eq 0 ] + then + printf "${PASS}Yum Install Complete${END}\n" + else + printf "${FAIL}Yum Install Failed${END}\n" + fi +} + +centos_install() +{ + #printf "${HEAD}Installing Yum Packages${END}\n" + #sudo yum install upx-3.07-1 libchm1 clamav wireshark upx + if [ $? -eq 0 ] + then + printf "${PASS}Yum Install Complete${END}\n" + else + printf "${FAIL}Yum Install Failed${END}\n" + fi +} +#=============================================================== +# This is the Beginning of the Script +#=============================================================== +# Sees if there is an argument +if [ -z $1 ]; +then + STEP=1 +else + STEP=$1 +fi + +while [ $STEP -lt 2 ] +do + case $STEP in + 1) + verify + if [ "$OS" = 'ubuntu' ] + then + #printf "${PASS}ubuntu is Supported!${END}\n" + ubuntu_install || exit_restart $STEP + depend_crits ||exit_restart $STEP + elif [ "$OS" = 'debian' ] + then + #printf "${PASS}Debian is Supported!${END}\n" + debian_install || exit_restart $STEP + depend_crits ||exit_restart $STEP + elif [ "$OS" = 'darwin' ] + then + #printf "${PASS}OS X is Supported!${END}\n" + darwin_install || exit_restart $STEP + depend_crits ||exit_restart $STEP + elif [ "$OS" = "centos" ] + then + #printf "${PASS}CentOS is Supported!${END}\n" + centos_install || exit_restart $STEP + depend_crits ||exit_restart $STEP + elif [ "$OS" = "red hat" ] + then + #printf "${PASS}Red Hat is Supported!${END}\n" + red_hat_install || exit_restart $STEP + depend_crits ||exit_restart $STEP + elif [ "$OS" = 'freebsd' ] + then + #printf "${PASS}FreeBSD is Supported${END}\n" + freebsd_install || exit_restart $STEP + depend_crits ||exit_restart $STEP + else + printf "${FAIL}OS: $OS, need Ubuntu, Debian, Darwin (OS X), CentOS, Red Hat, or FreeBSD${END}\n" + exit + fi + ;; + *) + exit + ;; + esac + STEP=$((STEP+1)) +done diff --git a/splunk_search_service/forms.py b/splunk_search_service/forms.py new file mode 100644 index 00000000..ad0b3e07 --- /dev/null +++ b/splunk_search_service/forms.py @@ -0,0 +1,79 @@ +from django import forms + +class SplunkSearchConfigForm(forms.Form): + error_css_class = 'error' + required_css_class = 'required' + splunk_url = forms.CharField(required=True, + label="Splunk URL", + initial='', + widget=forms.TextInput(), + help_text="Full URL of your Splunk API instance (e.g. https://192.168.1.100:8089/)") + splunk_browse_url = forms.CharField(required=True, + label="Splunk Browse URL", + initial='', + widget=forms.TextInput(), + help_text="Full URL of Splunk instance for your browser (e.g. https://192.168.1.100:8000/)") + splunk_user = forms.CharField(required=True, + label="Splunk username", + initial='', + widget=forms.TextInput(), + help_text="Username for Splunk instance") + password = forms.CharField(required=True, + label="Password", + initial='', + widget=forms.PasswordInput(), + help_text="Password for Splunk instance") + search_limit = forms.CharField(required=True, + label="Search limit", + initial='10', + widget=forms.NumberInput(), + help_text="Sets a limit to your searches so the db doesn't get flooded") + splunk_timeout = forms.CharField(required=True, + label = "Splunk timeout", + initial='180', + widget=forms.NumberInput(), + help_text="Sets a timeout limit for a given Splunk search to run") + search_earliest = forms.CharField(required=True, + label="Earliest", + initial='-2d@d', + widget=forms.TextInput(), + help_text="Using Splunk time syntax, this value will be the earliest your search runs.") + search_config = forms.CharField(required=True, + label="Search Config", + initial='/opt/crits/crits_services/splunk_search_service/searches.json', + widget=forms.TextInput(), + help_text="Full path of your Splunk search config file.") + url_search = forms.BooleanField(required=False, + label="URL Search", + initial='', + widget=forms.CheckboxInput(), + help_text="Explicitly run Splunk searches based on potential URls mined from this object.") + domain_search = forms.BooleanField(required=False, + label="Domain Search", + initial='', + widget=forms.CheckboxInput(), + help_text="Explicitly run Splunk searches based on potential domains mined from this object.") + ip_search = forms.BooleanField(required=False, + label="IP Search", + initial='', + widget=forms.CheckboxInput(), + help_text="Explicitly run Splunk searches based on potential IPs mined from this object.") + email_addy_search = forms.BooleanField(required=False, + label="Email Search", + initial='', + widget=forms.CheckboxInput(), + help_text="Explicitly run Splunk searches based on potential Email Addresses mined from this object.") + hash_search = forms.BooleanField(required=False, + label="Hash Search", + initial='', + widget=forms.CheckboxInput(), + help_text="Explicitly run Splunk searches based on potential hashes mined from this object.") + ignore_filetypes = forms.CharField(required=False, + label="Ignore filetypes", + initial='^[A-Za-z0-9]*( image data|ASCII text| archive data)', + widget=forms.TextInput(), + help_text="This is a regular expression telling the service we want to ignore certain samples if they are uploaded and this service is launched on triage.") + + def __init__(self, *args, **kwargs): + kwargs.setdefault('label_suffix',':') + super(SplunkSearchConfigForm, self).__init__(*args, **kwargs) diff --git a/splunk_search_service/searches.json b/splunk_search_service/searches.json new file mode 100644 index 00000000..9188b9f2 --- /dev/null +++ b/splunk_search_service/searches.json @@ -0,0 +1,168 @@ +{"searches":{"emails": [ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "samples": [ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "urls":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "urls_no_proto":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "domains": [ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "ips": [ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "email_addy": [ + {"name": "", + "search": ""}, + {"name": "", + "search": ""} + ], + "hashes": {"md5":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "sha1":[ + {"name":"", + "search":""} + ]}, + "indicators": {"API Key":[{"name": "", + "search": ""}, + {"name": "", + "search": ""}], + "AS Name":[], + "AS Number":[], + "Adjust Token":[], + "Bank account":[], + "Bitcount account":[], + "CRX":[], + "Certificate Fingerprint":[], + "Certificate Name":[], + "Checksum CRC16":[], + "Command Line":[], + "Company name":[], + "Cookie Name":[], + "Country":[], + "Debug Path":[], + "Debug String":[], + "Destination Port":[], + "Device IO":[], + "Document from URL":[], + "Domain":[], + "Email Address":[], + "Email Address From":[], + "Email Address Sender":[], + "Email Boundary":[], + "Email HELO":[], + "Email Header Field":[], + "Email Message ID":[], + "Email Originating IP":[], + "Email Reply-To":[], + "Email Subject":[], + "Email X-Mailer":[], + "Email X-Originating IP":[], + "File Created":[], + "File Deleted":[], + "File Moved":[], + "File Name":[], + "File Opened":[], + "File Path":[], + "File Read":[], + "File Written":[], + "GET Parameter":[], + "HEX String":[], + "HTML ID":[], + "HTTP Request":[], + "HTTP Response Code":[], + "IMPHASH":[], + "IPv4 Address":[], + "IPv4 Subnet":[], + "IPv6 Address":[], + "IPv6 Subnet":[], + "Latitude":[], + "Launch Agent":[], + "Location":[], + "Longitude":[], + "MAC Address":[], + "MD5":[], + "Malware Name":[], + "Memory Alloc":[], + "Memory Protect":[], + "Memory Read":[], + "Memory Written":[], + "Mutant Created":[], + "Mutex":[], + "Name Server":[], + "Other File Operation":[], + "POST Data":[], + "Password":[], + "Password Salt":[], + "Payload Data":[], + "Payload Type":[], + "Pipe":[], + "Process Name":[], + "Protocol":[], + "Referer":[], + "Registrar":[], + "Registry Key":[], + "Registry Key Created":[], + "Registry Key Deleted":[], + "Registry Key Enumerated":[], + "Registry Key Monitored":[], + "Registry Key Opened":[], + "Registry Key Value Created":[], + "Registry Key Value Deleted":[], + "Registry Key Value Modified":[], + "Registry Key Value Queried":[], + "SHA1":[], + "SHA256":[], + "SMS Origin":[], + "SSDEEP":[], + "Service Name":[], + "Source Port":[], + "TS End":[], + "TS Start":[], + "Telephone":[], + "Time Created":[], + "Time Updated":[], + "Tracking ID":[], + "URI":[], + "User Agent":[], + "User ID":[], + "Victim IP":[], + "Volume Queried":[], + "WHOIS Address 1":[], + "WHOIS Address 2":[], + "WHOIS Name":[], + "WHOIS Registrant Email Address":[], + "WHOIS Telephone":[], + "Web Payload":[], + "Webstorage Key":[], + "XPI":[] + } + } +} diff --git a/splunk_search_service/searches.py b/splunk_search_service/searches.py new file mode 100644 index 00000000..16f5f0bc --- /dev/null +++ b/splunk_search_service/searches.py @@ -0,0 +1,231 @@ +# Contains searches for a given TLO +import re, json + + +class SplunkSearches(object): + def __init__(self, obj, search_config): + + # Read config file + with open(search_config) as data_file: + self.searches = json.load(data_file) + + # Dictionary passed from DataMine can be used as a map + if isinstance(obj, dict): + self.obj = obj + # If it's a list, set it as list + elif isinstance(obj, list): + self.list = obj + # Otherwise, convert the object to a mapping dictionary + else: + fields = obj.__dict__ + self.obj = {} + for item in fields['_fields_ordered']: + self.obj[item] = getattr(obj,item) + + # Function for making sure the search name doesn't have a . \ or $ in it + def clean_keys(self, key): + return key.replace("\\","\").replace("\$","$").replace(".",".") + + + def email_searches(self): + ''' + # Potential Email attributes that can be called via {attribute} with .format(**self.obj) + boundary + cc + date + from_address + helo + message_id + originating_ip + raw_body + raw_header + reply_to + sender + subject + to + x_originating_ip + x_mailer + ''' + + ''' + self.splunk_searches = {"description": "Searches Splunk based on email attibutes", + "searches": [{"name": self.clean_keys("Email sender"), + "search": ("index=smtp src_user=\"{sender}\"" + "| stats values(src_user) AS src_user " + "values(recipient) AS recipient count by subject").format(**self.obj)}, + {"name": self.clean_keys("Email subject"), + "search": ("index=smtp subject=\"{subject}\"" + "| stats values(src_ip) AS src_ip values(recipient) AS recipient " + "count by subject").format(**self.obj)} + ] + } + ''' + self.splunk_searches = {"description": "Searches Splunk based on email attibutes", + "searches": []} + for search in self.searches['searches']['emails']: + new_search = {'name': self.clean_keys(search['name'].format(**self.obj)), + 'search': search['search'].format(**self.obj)} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + def sample_searches(self): + ''' + # Potential Email attributes that can be called via {attribute} with .format(**self.obj) + filedata + filename + filenames + filetype + md5 + mimetype + sha1 + sha256 + size + ssdeep + impfuzzy + ''' + ''' + self.splunk_searches = {"description": "Searches Splunk based on sample attributes", + "searches": [{"name": self.clean_keys("MD5 Search"), + "search": ("index=files md5=\"{md5}\"" + "|stats values(dest_ip) AS dest_ip values(global_cuid) AS global_cuid" + " values(global_fuid) AS global_fuid count by src_ip").format(**self.obj)}, + ] + } + ''' + self.splunk_searches = {"description": "Searches Splunk based on sample attibutes", + "searches": []} + for search in self.searches['searches']['samples']: + new_search = {'name': self.clean_keys(search['name'].format(**self.obj)), + 'search': search['search'].format(**self.obj)} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + + def indicator_searches(self): + ''' + # Potential Email attributes that can be called via {attribute} with .format(**self.obj) + activity + confidence + impact + ind_type + threat_types + attack_types + value + lower + ''' + self.splunk_searches = {"description": "Searches Splunk based on indicator attibutes", + "searches": []} + + for search in self.searches['searches']['indicators'][self.obj['ind_type']]: + new_search = {'name': self.clean_keys(search['name'].format(**self.obj)), + 'search': search['search'].format(**self.obj)} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + def url_searches(self): + self.splunk_searches = {"description": "Splunk searches for URLs mined from this object", + "searches": []} + for url in self.list: + ''' + self.splunk_searches['searches'].append({"name": self.clean_keys(str("smtp_links search for "+url)), + "search": ("index=smtp_links url=\""+url+"\"" + "|stats values(url) AS url values(recipient) AS recipient " + "count by src_user subject")}) + ''' + for search in self.searches['searches']['urls']: + new_search = {'name': self.clean_keys(search['name'].format(url)), + 'search': search['search'].format(url)} + + self.splunk_searches['searches'].append(new_search) + + for search in self.searches['searches']['urls_no_proto']: + # Strip the protocol + pattern = r"^(https?|ftp)://" + url_no_proto = re.sub(pattern, r"", url) + + new_search = {'name': self.clean_keys(search['name'].format(url_no_proto)), + 'search': search['search'].format(url_no_proto)} + + self.splunk_searches['searches'].append(new_search) + + ''' + self.splunk_searches['searches'].append({"name": self.clean_keys(str("http search for "+url_no_proto)), + "search": ("index=http url=\""+url_no_proto+"\"" + "|stats values(src_ip) AS src_ip values(dest_ip) AS dest_ip " + "values(url) AS url count by domain")}) + ''' + + return self.splunk_searches + + def domain_searches(self): + self.splunk_searches = {"description": "Splunk searches for domains mined from this object", + "searches": []} + for domain in self.list: + ''' + self.splunk_searches['searches'].append({"name": self.clean_keys(str("smtp_links/http search for "+domain)), + "search": ("index=smtp_links OR index=http domain=\""+domain+"\"" + "|stats values(src_ip) AS src_ip values(url) AS url " + "values(src_user) AS src_user values(subject) AS subject " + "values(recipient) AS recipient count by index")}) + ''' + for search in self.searches['searches']['domains']: + new_search = {'name': self.clean_keys(search['name'].format(domain)), + 'search': search['search'].format(domain)} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + def ip_searches(self): + self.splunk_searches = {"description": "Splunk searches for IPs mined from this object", + "searches": []} + ''' + for ip in self.list: + self.splunk_searches['searches'].append({"name": self.clean_keys(str("smtp_links/http search for "+ip+" as domain")), + "search": ("index=smtp_links OR index=http domain=\""+ip+"\"" + "|stats values(src_ip) AS src_ip AS subject values(url) AS url " + "values(src_user) AS src_user values(subject) " + "values(recipient) AS recipient count by index")}) + ''' + for ip in self.list: + for search in self.searches['searches']['ips']: + new_search = {'name': self.clean_keys(search['name'].format(ip)), + 'search': search['search'].format(ip)} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + def email_addy_searches(self): + self.splunk_searches = {"description": "Splunk searches for email addresses mined from this object", + "searches": []} + + for email_addy in self.list: + for search in self.searches['searches']['email_addy']: + new_search = {'name': self.clean_keys(search['name'].format(email_addy)), + 'search': search['search'].format(email_addy)} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + def hash_searches(self): + self.splunk_searches = {"description": "Splunk searches for hashes mined from this object", + "searches": []} + + for hash in self.list: + for search in self.searches['searches']['hashes'][hash[0]]: + new_search = {'name': self.clean_keys(search['name'].format(item[1])), + 'search': search['search'].format(item[1])} + + self.splunk_searches['searches'].append(new_search) + + return self.splunk_searches + + \ No newline at end of file diff --git a/splunk_search_service/searches_example.json b/splunk_search_service/searches_example.json new file mode 100644 index 00000000..2a9a3a1c --- /dev/null +++ b/splunk_search_service/searches_example.json @@ -0,0 +1,713 @@ +{"searches":{"emails": [ + {"name":"Email Sender", + "search":"index=smtp src_user=\"{sender}\" | stats values(src_user) AS src_user values(recipient) AS recipient count by subject"}, + {"name":"Email Subject", + "search":"index=smtp subject=\"{subject}\" | stats values(subject) AS subject values(recipient) AS recipient count by src_user"} + ], + "samples": [ + {"name":"MD5 Search", + "search":"index=files md5=\"{md5}\" | stats values(dest_ip) AS dest_ip values(uid) As uid values(fuid) AS fuid count by src_ip"}, + {"name":"Filename Search", + "search":"index=files filename=\"{filename}\" | stats values(dest_ip) AS dest_ip values(uid) As uid values(fuid) AS fuid count by src_ip"} + ], + "urls":[ + {"name":"{}", + "search":"index=smtp_links url=\"{}\" | stats values(url) AS url values(recipient) AS recipient count by src_user subject"}, + {"name":"", + "search":""} + ], + "urls_no_proto":[ + {"name":"{}", + "search":"index=http url=\"{}\" | stats values(src_ip) AS src_ip values(dest_ip) AS dest_ip values(url) AS url count by domain"} + ], + "domains": [ + {"name":"{}", + "search":"index=smtp_links OR index=http domain=\"{}\" | stats values(src_ip) AS src_ip values(url) AS url count by index"}, + {"name":"", + "search":""} + ], + "ips": [ + {"name":"{}", + "search":"index=http dest_ip=TERM({}) | stats values(src_ip) AS src_ip count by dest_ip"}, + {"name":"", + "search":""} + ], + "email_addy": [ + {"name": "{}", + "search": "index=smtp src_user=\"{}\"|stats values(src_user) AS src_user values(subject) AS subject count by recipient"}, + {"name": "", + "search": ""} + ], + "hashes": {"md5":[ + {"name":"{}", + "search":"index=files md5=\"{}\" | stats values(src_ip) AS src_ip count by filename"}, + {"name":"", + "search":""} + ], + "sha1":[ + {"name":"", + "search":""} + ]}, + "indicators": {"API Key":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "AS Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "AS Number":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Adjust Token":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Bank account":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Bitcount account":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "CRX":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Certificate Fingerprint":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Certificate Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Checksum CRC16":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Command Line":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Company name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Cookie Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Country":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Debug Path":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Debug String":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Destination Port":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Device IO":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Document from URL":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Domain":[ + {"name":"Domain Indicator: {value}", + "search":"index=http \"{value}\" | stats values(domain) AS domain values(query) AS query count by src_ip"}, + {"name":"", + "search":""} + ], + "Email Address":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Addres From":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Address Sender":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Boundary":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email HELO":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Header Field":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Message ID":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Originating IP":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Reply-To":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email Subject":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email X-Mailer":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Email X-Originating IP":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Created":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Deleted":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Moved":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Opened":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Path":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Read":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "File Written":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "GET Parameter":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "HEX String":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "HTML ID":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "HTTP Request":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "HTTP Response Code":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "IMPHASH":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "IPv4 Address":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "IPv4 Subnet":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "IPv6 Address":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "IPv6 Subnet":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Latitude":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Launch Agent":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Location":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Longitude":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "MAC Address":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "MD5":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Malware Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Memory Alloc":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Memory Protect":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Memory Read":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Memory Written":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Mutant Created":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Mutex":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Name Server":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Other File Operation":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "POST Data":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Password":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Password Salt":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Payload Data":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Payload Type":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Pipe":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Process Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Protocol":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Referer":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registrar":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Created":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Deleted":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Enumerated":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Monitored":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Opened":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Value Created":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Value Deleted":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Value Modified":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Registry Key Value Queried":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "SHA1":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "SHA256":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "SMS Origin":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "SSDEEP":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Service Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Source Port":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "TS End":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "TS Start":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Telephone":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Time Created":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Time Updated":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Tracking ID":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "URI":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "User Agent":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "User ID":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Victim IP":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Volume Queried":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "WHOIS Address 1":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "WHOIS Address 2":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "WHOIS Name":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "WHOIS Registrant Email Address":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "WHOIS Telephone":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Web Payload":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "Webstorage Key":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ], + "XPI":[ + {"name":"", + "search":""}, + {"name":"", + "search":""} + ] + } + } +} \ No newline at end of file diff --git a/splunk_search_service/templates/splunk_search_service_template.html b/splunk_search_service/templates/splunk_search_service_template.html new file mode 100644 index 00000000..31051169 --- /dev/null +++ b/splunk_search_service/templates/splunk_search_service_template.html @@ -0,0 +1,80 @@ + +{% load service_tags %} + + + + +{% regroup analysis.results by subtype as result_list %} +{% for result_subtype in result_list %} +

{{ result_subtype.grouper }}

+ {% for search_link in result_subtype.list %} + {% if forloop.first %} + {% for search, url in search_link.full_search_dict.items %} + {% if search == result_subtype.grouper %} + Re-run search in Splunk
+ {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + + + {% for result in result_subtype.list %} + {% if forloop.first %} + {# Create the header the first time through the loop. #} + + + {% if result.fields %} + {% for field in result.fields %} + + {% endfor %} + {% else %} + + {% endif %} + + + + {% endif %} + + {% if result.result%} + {% for value in result.result %} + + {% for field in result.fields %} + {% for k,v in value.items %} + {% if k == field %} + + {% endif %} + {% endfor %} + {% endfor %} + + + {% endfor %} + {% endif %} + + + {% endfor %} + +
{{ field }}Result
+ {% for val in v %} + {{val}}
+ {% endfor %} +
+{% empty %} + {% if analysis.status == 'started' %} +

This service is still running.

+ {% else %} +

This service produced no results.

+ {% endif %} +{% endfor %} + + + diff --git a/splunk_search_service/test_searches.py b/splunk_search_service/test_searches.py new file mode 100644 index 00000000..993878e3 --- /dev/null +++ b/splunk_search_service/test_searches.py @@ -0,0 +1,316 @@ +# This is a helper script used to test your searches.json file and make sure it returns +# the Splunk results you're looking for. + +# Make sure the configs match your specific options, and set the desired search type to 'True'. +# Feel free to modify the dummy data to emulate a desired TLO, or list of domains/hashes/etc. + +import json, time, requests + +from searches import SplunkSearches + +from lxml import objectify +from urllib import quote_plus +from pprint import pprint + +# Manual configs for test run +configs = {'splunk_url': 'https://192.168.1.100:8089/', + 'splunk_browse_url': 'https://192.168.1.100:8000/', + 'search_limit': '10', + 'splunk_timeout': '180', + 'splunk_user': '', + 'password': '', + 'search_earliest':'-1d@d', + 'search_config':'/opt/crits/crits_services/splunk_search_service/searches.json', + 'url_search': False, + 'domain_search': False, + 'ip_search': False, + 'hash_search': False, + 'email_addy_search': False, + 'sample_search': False, + 'email_search': False, + 'indicator_search': False} + +#### Set dummy data for Splunk searches #### + +dummy_sample = {'filedata': 'block_of_data', + 'filename': 'test_file.bin', + 'filenames': ['test1.bin','test2.bin'], + 'filetype': 'PDF document, version 1.5', + 'md5': '098f6bcd4621d373cade4e832627b4f6', + 'mimetype': 'application/pdf', + 'sha1':'a94a8fe5ccb19ba61c4c0873d391e987982fbbd3', + 'sha256':'9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', + 'size':'111530', + 'ssdeep':'3072:LeaaIQxECw+cRCd8iMXsxWcdkQyMGDPl492RMUtQpYqgJQ:LevIQxEfLYdAX/8kmGB4IRMFYjJQ', + 'impfuzzy':'None'} + +dummy_email = {'boundary': 'None', + 'cc': 'test@test.com', + 'date': 'Sat, 8 Apr 2017 05:30:00 +0700', + 'from_address': '"Badguy, Worst" ', + 'helo': 'None', + 'message_id': '<123456789@test.com>', + 'originating_ip': '192.168.1.100', + 'raw_body': 'This is the email', + 'raw_header': 'Test header', + 'reply_to': 'spoof@test.com', + 'sender': 'test@badsender.com', + 'subject': 'Test subject', + 'to': 'recip1@test.com, recip2@test.com', + 'x_originating_ip': '192.168.1.101', + 'x_mailer': 'None'} + +dummy_indicator = {'activity': 'None', + 'confidence': 'Unknown', + 'impact': 'Unknown', + 'ind_type': 'Domain', + 'threat_types': 'Unknown', + 'attack_types': 'Unknown', + 'value': 'badguy.com', + 'lower': 'None'} + +urls = ['https://badguy.com/test', 'http://goodguy.com/test'] +domains = ['badguy.com', 'goodguy.com'] +ips = ['192.168.1.100', '192.168.1.102'] +emails = ['test@test.com', 'test2@test2.com'] +hashes = [['md5', '098f6bcd4621d373cade4e832627b4f6'],['sha1','a94a8fe5ccb19ba61c4c0873d391e987982fbbd3']] + + +#### Handle Splunk Interaction #### + +# Poll Job IDs +def poll_jobs(configs, sessionKey, all_jobs): + splunk_url = configs['splunk_url'] + splunk_timeout = configs['splunk_timeout'] + headers = {"Authorization":"Splunk "+str(sessionKey)} + search_link = splunk_url+"services/search/jobs/" + job_results = {"done":[], + "timeout":[], + "failed":[]} + first_run = True + + # Loop through the full list of jobs, polling status every 2 seconds + # until runDuration > configs['splunk_timeout']. Waits 5 seconds + # before first poll b/c c'mon... Splunk's not that fast. + # + # Also, if dispatchState or runDuration are not returning, something + # probably went wrong... So it breaks the while loop and returns whatever + # results it's grabbed. + + + while all_jobs: + if first_run == True: + time.sleep(5) + first_run = False + ''' + all_jobs = {'badguy.com': '', + 'goodguy.com': ''} + ''' + for name, job_sid in all_jobs.iteritems(): + runDuration = '' + dispatchState = '' + search_check = requests.get(search_link+job_sid, headers=headers, verify=False) + scheck_root = objectify.fromstring(search_check._content) + for a in scheck_root.getchildren(): + for b in a.getchildren(): + for c in b.getchildren(): + if 'name' in c.attrib and c.attrib['name']=='dispatchState': + dispatchState = c.text + if 'name' in c.attrib and c.attrib['name']=='runDuration': + runDuration = c.text + + # Emergency STOP if runDuration or dispatchState aren't found... + print("Search is %s. Run duration: %s" % (dispatchState, runDuration)) + if dispatchState=='': + print("dispatchState was not returned for some reason.") + all_jobs = {x : y for x,y in all_jobs.iteritems() if x!=x} + return job_results + if dispatchState!="QUEUED" and runDuration=='': + print("runDuration was not returned for some reason.") + all_jobs = {x : y for x,y in all_jobs.iteritems() if x!=x} + return job_results + + if dispatchState=="DONE": + job_results['done'].append({name:job_sid}) + all_jobs = {x : y for x,y in all_jobs.iteritems() if name!=x} + elif dispatchState=="FAILED": + job_results['failed'].append({name:job_sid}) + all_jobs = {x : y for x,y in all_jobs.iteritems() if name!=x} + if runDuration!='' and float(runDuration) > int(splunk_timeout): + job_results['timeout'].append({name:job_sid}) + all_jobs = {x : y for x,y in all_jobs.iteritems() if name!=x} + else: + print("Search %s is currently %s. It has been running for %s seconds." + % (job_sid, dispatchState, runDuration)) + + # Sleep 2 seconds between checking all jobs + time.sleep(2) + + ''' + job_results = {"done":[{'badguy.com': ''}, + {'goodguy.com': ''}], + "timeout":[], + "failed":[]} + ''' + return job_results + +# Get Search Results +def get_search_results(configs, sessionKey, name_sid): + splunk_url = configs['splunk_url'] + headers = {"Authorization":"Splunk "+str(sessionKey)} + data = {'output_mode': 'json'} + json_response={} + for name, sid in name_sid.iteritems(): + search_link = splunk_url+"services/search/jobs/"+sid+"/results/" + search_rez = requests.get(search_link, headers=headers, params=data, verify=False) + json_response[name] = json.loads(search_rez._content) + + ''' + json_response = {'search for emails': } + ''' + + return json_response + +# Start Splunk Search Job +def start_splunk_search_job(configs, sessionKey, search): + splunk_url = configs['splunk_url'] + headers = {"Authorization":"Splunk "+str(sessionKey)} + + search_link = splunk_url+"services/search/jobs" + search_data = {"search": search} + search_resp = requests.post(search_link, headers=headers, data=search_data, verify=False) + search_root = objectify.fromstring(search_resp._content) + job_sid = str(search_root['sid']) + + return job_sid + +# Get a Splunk session ID +def get_splunk_session_key(configs): + splunk_url = configs['splunk_url'] + login_link = "services/auth/login" + login_data = {"username": configs['splunk_user'], + "password": configs['password']} + auth_link = splunk_url+login_link + resp = requests.get(auth_link, data=login_data, verify=False) + root = objectify.fromstring(resp._content) + sessionKey = root['sessionKey'] + + return sessionKey + +#### Run the tests #### + +all_splunk_searches = [] +all_jobs={} + +if configs['url_search']==True: + #urls = dedup(extract_urls(data)) + # Run searches for URLs + splunk_obj = SplunkSearches(urls,configs['search_config']) + splunk_searches_urls = splunk_obj.url_searches() + all_splunk_searches.append(splunk_searches_urls) + +if configs['domain_search']==True: + #domains = dedup(extract_domains(data)) + # Run searches for domains + splunk_obj = SplunkSearches(domains,configs['search_config']) + splunk_searches_domains = splunk_obj.domain_searches() + all_splunk_searches.append(splunk_searches_domains) + +if configs['ip_search']==True: + #ips = dedup(extract_ips(data)) + # Run searches for IPs + splunk_obj = SplunkSearches(ips,configs['search_config']) + splunk_searches_ips = splunk_obj.ip_searches() + all_splunk_searches.append(splunk_searches_ips) + +if configs['hash_search']==True: + #hashes = dedup(extract_hashes(data)) + # Run searches for hashes + splunk_obj = SplunkSearches(hashes,configs['search_config']) + splunk_searches_hashes = splunk_obj.hash_searches() + all_splunk_searches.append(splunk_searches_hashes) + +if configs['email_addy_search']==True: + #email_addys = dedup(extract_emails(data)) + # Run searches for Email addresses + splunk_obj = SplunkSearches(email_addys,configs['search_config']) + splunk_searches_email_addys = splunk_obj.email_addy_searches() + all_splunk_searches.append(splunk_searches_email_addys) + + +if configs['email_search']==True: + # Set splunk_obj for TLO + splunk_obj = SplunkSearches(dummy_email,configs['search_config']) + + splunk_searches = splunk_obj.email_searches() + all_splunk_searches.append(splunk_searches) + #self._debug(str(splunk_searches)) + +if configs['sample_search']==True: + # Set splunk_obj for TLO + splunk_obj = SplunkSearches(dummy_sample,configs['search_config']) + + splunk_searches = splunk_obj.sample_searches() + all_splunk_searches.append(splunk_searches) + +if configs['indicator_search']==True: + # Set splunk_obj for TLO + splunk_obj = SplunkSearches(dummy_indicator,configs['search_config']) + + splunk_searches = splunk_obj.indicator_searches() + all_splunk_searches.append(splunk_searches) + +# Get the Splunk session ID +sessionKey = get_splunk_session_key(configs) + +# Start the search jobs +for search_group in all_splunk_searches: + if 'searches' in search_group and search_group['searches']: + + splunk_results = [] + + + + for search in search_group['searches']: + if search['search']!="": + ## Set the timeframe and search limit + search['search']="earliest="+configs['search_earliest']+" "+search['search'] + search['search']+="|head "+configs['search_limit'] + # Make sure it starts with 'search' or a | + if not (search['search'].startswith("search") or search['search'].startswith("|")): + search['search']="search "+search['search'] + + # Start a Splunk Search Job + job_sid = start_splunk_search_job(configs, sessionKey, search['search']) + + # Add the job_sid to a list of jobs to poll + all_jobs[search['name']]=job_sid + + # Output + print("Launched search!\r\n%s\r\n%s\r\nJob ID: %s" % (search['name'],search['search'], job_sid)) + + # Build a dict of search names and their searches + search_base=configs['splunk_browse_url']+"en-US/app/search/search?q=" + full_search=search_base+quote_plus(search['search']) + print("Open in browser: %s \r\n\r\n" % full_search) + #full_search_dict[search['name']] = full_search + + # Poll results + poll_results = poll_jobs(configs, sessionKey, all_jobs) + print("Polling results:") + + for item in poll_results['timeout']: + for k, v in item.iteritems(): + print("Splunk job %s for the search %s timed out." % (v,k)) + + for item in poll_results['failed']: + for k, v in item.iteritems(): + print("Splunk job %s for the search %s FAILED. Check Splunk for deatils." % (v,k)) + + # Get the results + for item in poll_results['done']: + #print("Sending the following item to Splunk to get the results: %s" % str(item)) + splunk_results.append(get_search_results(configs, sessionKey, item)) + print("Finished job results:") + pprint(splunk_results) +