From 67c2a3fce7419f0c6b418b2b91da3c45b399f2b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Vorbach?= Date: Mon, 14 May 2018 15:31:58 +0200 Subject: [PATCH] added s3 storage for attack event files (#285) Added s3 storage for attack event files --- glastopf/glastopf.cfg.dist | 9 ++ glastopf/modules/events/attack.py | 4 + glastopf/modules/handlers/emulators/rfi.py | 17 ++-- .../reporting/auxiliary/base_logger.py | 7 +- .../modules/reporting/auxiliary/log_s3.py | 85 +++++++++++++++++++ glastopf/modules/reporting/main/log_sql.py | 4 +- requirements.txt | 1 + 7 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 glastopf/modules/reporting/auxiliary/log_s3.py diff --git a/glastopf/glastopf.cfg.dist b/glastopf/glastopf.cfg.dist index 960dd7cc..b454fd7b 100644 --- a/glastopf/glastopf.cfg.dist +++ b/glastopf/glastopf.cfg.dist @@ -105,3 +105,12 @@ sensorid = None [profiler] enabled = False + +[s3storage] +enabled = False +endpoint = http://localhost:8080/ +aws_access_key_id = YOUR_aws_access_key_id +aws_secret_access_key = YOUR_aws_access_key_id +bucket = glastopf +region = eu-west-1 +signature_version = s3 diff --git a/glastopf/modules/events/attack.py b/glastopf/modules/events/attack.py index fc92d619..d4d86a16 100644 --- a/glastopf/modules/events/attack.py +++ b/glastopf/modules/events/attack.py @@ -29,8 +29,10 @@ def __init__(self): self.source_addr = None self.matched_pattern = "unknown" self.file_name = None + self.file_sha256 = None self.version = None self.sensorid = None + self.known_file = False def event_dict(self): event_dict = { @@ -40,7 +42,9 @@ def event_dict(self): "request_raw": self.http_request.request_raw, "pattern": self.matched_pattern, "filename": self.file_name, + "file_sha256": self.file_sha256, "version": self.version, "sensorid": self.sensorid, + "known_file": self.known_file } return event_dict diff --git a/glastopf/modules/handlers/emulators/rfi.py b/glastopf/modules/handlers/emulators/rfi.py index b19dfec7..b0afc540 100644 --- a/glastopf/modules/handlers/emulators/rfi.py +++ b/glastopf/modules/handlers/emulators/rfi.py @@ -31,6 +31,7 @@ class RFIEmulator(base_emulator.BaseEmulator): def __init__(self, data_dir): super(RFIEmulator, self).__init__(data_dir) + self.downloaded_file_exists = False self.files_dir = os.path.join(self.data_dir, 'files/') if not os.path.exists(self.files_dir): os.mkdir(self.files_dir) @@ -46,14 +47,17 @@ def extract_url(cls, url): @classmethod def get_filename(cls, injected_file): file_name = hashlib.md5(injected_file).hexdigest() - return file_name + file_sha256 = hashlib.sha256(injected_file).hexdigest() + return file_name, file_sha256 def store_file(self, injected_file): - file_name = self.get_filename(injected_file) + file_name, file_sha256 = self.get_filename(injected_file) if not os.path.exists(os.path.join(self.files_dir, file_name)): with open(os.path.join(self.files_dir, file_name), 'w+') as local_file: local_file.write(injected_file) - return file_name + else: + self.downloaded_file_exists = True + return file_name, file_sha256 def download_file(self, url): injectd_url = self.extract_url(urllib2.unquote(url)) @@ -80,12 +84,12 @@ def download_file(self, url): # the injected file but pretend to be vulnerable. file_name = None else: - file_name = self.store_file(injected_file) - return file_name + file_name, file_sha256 = self.store_file(injected_file) + return file_name, file_sha256 def handle(self, attack_event): if attack_event.http_request.command == 'GET': - attack_event.file_name = self.download_file( + attack_event.file_name, attack_event.file_sha256 = self.download_file( attack_event.http_request.path) elif attack_event.http_request.command == 'POST': pass @@ -94,4 +98,5 @@ def handle(self, attack_event): if attack_event.file_name: response = sandbox.run(attack_event.file_name, self.data_dir) attack_event.http_request.set_raw_response(response) + attack_event.known_file = self.downloaded_file_exists return attack_event diff --git a/glastopf/modules/reporting/auxiliary/base_logger.py b/glastopf/modules/reporting/auxiliary/base_logger.py index 491f1790..6f2544e4 100644 --- a/glastopf/modules/reporting/auxiliary/base_logger.py +++ b/glastopf/modules/reporting/auxiliary/base_logger.py @@ -15,13 +15,14 @@ # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from ConfigParser import ConfigParser +from ConfigParser import SafeConfigParser +import os class BaseLogger(object): def __init__(self, config='glastopf.cfg'): - if not isinstance(config, ConfigParser): - self.config = ConfigParser() + if not isinstance(config, SafeConfigParser): + self.config = SafeConfigParser(os.environ) self.config.read(config) else: self.config = config diff --git a/glastopf/modules/reporting/auxiliary/log_s3.py b/glastopf/modules/reporting/auxiliary/log_s3.py new file mode 100644 index 00000000..e9d96e41 --- /dev/null +++ b/glastopf/modules/reporting/auxiliary/log_s3.py @@ -0,0 +1,85 @@ +# Copyright (C) 2018 Andre Vorbach @vorband +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os +import gevent + +import botocore.session, botocore.client +from botocore.exceptions import ClientError + +from glastopf.modules.reporting.auxiliary.base_logger import BaseLogger + + +logger = logging.getLogger(__name__) + + +class S3Logger(BaseLogger): + + def __init__(self, data_dir, work_dir, config="glastopf.cfg", reconnect=True): + config = os.path.join(work_dir, config) + BaseLogger.__init__(self, config) + self.files_dir = os.path.join(data_dir, 'files/') + self.enabled = self.config.getboolean("s3storage", "enabled") + self._initial_connection_happend = False + self.options = {'enabled': self.enabled} + if self.enabled: + self.endpoint = self.config.get("s3storage", "endpoint") + self.accesskey = self.config.get("s3storage", "aws_access_key_id") + self.secretkey = self.config.get("s3storage", "aws_secret_access_key") + self.version = self.config.get("s3storage", "signature_version") + self.region = self.config.get("s3storage", "region") + self.bucket = self.config.get("s3storage", "bucket") + self.enabled = True + self.s3client = None + self.s3session = None + gevent.spawn(self._start_connection) + + def _start_connection(self): + self.s3session = botocore.session.get_session() + self.s3session.set_credentials(self.accesskey, self.secretkey) + self.s3client = self.s3session.create_client( + 's3', + endpoint_url=self.endpoint, + region_name=self.region, + config=botocore.config.Config(signature_version=self.version) + ) + try: + self.s3client.head_bucket(Bucket=self.bucket) + self._initial_connection_happend = True + except ClientError as e: + logger.error("Could not establish s3 connection to bucket '%s' on %s. Received error: %s" % (self.bucket, self.endpoint, e.response['Error']['Message'])) + + def insert(self, attack_event): + if self._initial_connection_happend: + if attack_event.file_sha256 is not None: + if attack_event.known_file: + logger.debug('sha256:{0} / md5:{1} is a known file, it will not be uploaded.'.format(attack_event.file_sha256, attack_event.file_name)) + return + with file(os.path.join(self.files_dir, attack_event.file_name), 'r') as file_handler: + try: + # check if file exists in bucket + searchFile = self.s3client.list_objects_v2(Bucket=self.bucket, Prefix=attack_event.file_sha256) + if (len(searchFile.get('Contents', []))) == 1 and str(searchFile.get('Contents', [])[0]['Key']) == attack_event.file_sha256: + logger.debug('Not storing file (sha256:{0}) to s3 bucket "{1}" on {2} as it already exists in the bucket.'.format(attack_event.file_sha256, self.bucket, self.endpoint)) + return + # upload file to s3 + self.s3client.put_object(Bucket=self.bucket, Body=file_handler, Key=attack_event.file_sha256) + logger.debug('Storing file (sha256:{0}) using s3 bucket "{1}" on {2}'.format(attack_event.file_sha256, self.bucket, self.endpoint)) + except ClientError as e: + logger.warning("Received error: %s", e.response['Error']['Message']) + else: + logger.warning('Not storing attack file because initial s3 connection test did not succeeded') + diff --git a/glastopf/modules/reporting/main/log_sql.py b/glastopf/modules/reporting/main/log_sql.py index 5f3e12f8..b663fab2 100644 --- a/glastopf/modules/reporting/main/log_sql.py +++ b/glastopf/modules/reporting/main/log_sql.py @@ -18,7 +18,7 @@ #import json import logging -from sqlalchemy import Table, Column, Integer, String, MetaData, TEXT +from sqlalchemy import Table, Column, Integer, String, MetaData, TEXT, Boolean from sqlalchemy.orm import sessionmaker from sqlalchemy import exc import glastopf.modules.processing.ip_profile as ipp @@ -78,8 +78,10 @@ def setup_mapping(self): Column('request_raw', TEXT), Column('pattern', String(20)), Column('filename', String(500)), + Column('file_sha256', String(500)), Column('version', String(10)), Column('sensorid', String(36)), + Column('known_file', Boolean()) ) #only creates if it cant find the schema meta.create_all(self.engine) diff --git a/requirements.txt b/requirements.txt index a59fa1c4..500223d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ hpfeeds pylibinjection libtaxii>=1.1 python-logstash +botocore