added s3 storage for attack event files (#285)

Added s3 storage for attack event files

glastopf/glastopf.cfg.dist

@@ -105,3 +105,12 @@ sensorid = None
 
 [profiler]
 enabled = False
+
+[s3storage]
+enabled = False
+endpoint = http://localhost:8080/
+aws_access_key_id = YOUR_aws_access_key_id
+aws_secret_access_key = YOUR_aws_access_key_id
+bucket = glastopf
+region = eu-west-1
+signature_version = s3	

glastopf/modules/events/attack.py

@@ -29,8 +29,10 @@ def __init__(self):
         self.source_addr = None
         self.matched_pattern = "unknown"
         self.file_name = None
+        self.file_sha256 = None
         self.version = None
         self.sensorid = None
+        self.known_file = False
 
     def event_dict(self):
         event_dict = {
@@ -40,7 +42,9 @@ def event_dict(self):
             "request_raw": self.http_request.request_raw,
             "pattern": self.matched_pattern,
             "filename": self.file_name,
+            "file_sha256": self.file_sha256,
             "version": self.version,
             "sensorid": self.sensorid,
+            "known_file": self.known_file
         }
         return event_dict

glastopf/modules/handlers/emulators/rfi.py

@@ -31,6 +31,7 @@
 class RFIEmulator(base_emulator.BaseEmulator):
     def __init__(self, data_dir):
         super(RFIEmulator, self).__init__(data_dir)
+        self.downloaded_file_exists = False
         self.files_dir = os.path.join(self.data_dir, 'files/')
         if not os.path.exists(self.files_dir):
             os.mkdir(self.files_dir)
@@ -46,14 +47,17 @@ def extract_url(cls, url):
     @classmethod
     def get_filename(cls, injected_file):
         file_name = hashlib.md5(injected_file).hexdigest()
-        return file_name
+        file_sha256 = hashlib.sha256(injected_file).hexdigest()
+        return file_name, file_sha256
 
     def store_file(self, injected_file):
-        file_name = self.get_filename(injected_file)
+        file_name, file_sha256 = self.get_filename(injected_file)
         if not os.path.exists(os.path.join(self.files_dir, file_name)):
             with open(os.path.join(self.files_dir, file_name), 'w+') as local_file:
                 local_file.write(injected_file)
-        return file_name
+        else:
+            self.downloaded_file_exists = True
+        return file_name, file_sha256
 
     def download_file(self, url):
         injectd_url = self.extract_url(urllib2.unquote(url))
@@ -80,12 +84,12 @@ def download_file(self, url):
             # the injected file but pretend to be vulnerable.
             file_name = None
         else:
-            file_name = self.store_file(injected_file)
-        return file_name
+            file_name, file_sha256 = self.store_file(injected_file)
+        return file_name, file_sha256
 
     def handle(self, attack_event):
         if attack_event.http_request.command == 'GET':
-            attack_event.file_name = self.download_file(
+            attack_event.file_name, attack_event.file_sha256  = self.download_file(
                 attack_event.http_request.path)
         elif attack_event.http_request.command == 'POST':
             pass
@@ -94,4 +98,5 @@ def handle(self, attack_event):
         if attack_event.file_name:
             response = sandbox.run(attack_event.file_name, self.data_dir)
             attack_event.http_request.set_raw_response(response)
+        attack_event.known_file = self.downloaded_file_exists
         return attack_event

glastopf/modules/reporting/auxiliary/base_logger.py

@@ -15,13 +15,14 @@
 # Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-from ConfigParser import ConfigParser
+from ConfigParser import SafeConfigParser
+import os
 
 
 class BaseLogger(object):
     def __init__(self, config='glastopf.cfg'):
-        if not isinstance(config, ConfigParser):
-            self.config = ConfigParser()
+        if not isinstance(config, SafeConfigParser):
+            self.config = SafeConfigParser(os.environ)
             self.config.read(config)
         else:
             self.config = config

glastopf/modules/reporting/auxiliary/log_s3.py

@@ -0,0 +1,85 @@
+# Copyright (C) 2018 Andre Vorbach @vorband
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import gevent
+
+import botocore.session, botocore.client
+from botocore.exceptions import ClientError
+
+from glastopf.modules.reporting.auxiliary.base_logger import BaseLogger
+
+
+logger = logging.getLogger(__name__)
+
+
+class S3Logger(BaseLogger):
+
+    def __init__(self, data_dir, work_dir, config="glastopf.cfg", reconnect=True):
+        config = os.path.join(work_dir, config)
+        BaseLogger.__init__(self, config)
+        self.files_dir = os.path.join(data_dir, 'files/')
+        self.enabled = self.config.getboolean("s3storage", "enabled")
+        self._initial_connection_happend = False
+        self.options = {'enabled': self.enabled}
+        if self.enabled:
+            self.endpoint = self.config.get("s3storage", "endpoint")
+            self.accesskey = self.config.get("s3storage", "aws_access_key_id")
+            self.secretkey = self.config.get("s3storage", "aws_secret_access_key")
+            self.version = self.config.get("s3storage", "signature_version")
+            self.region = self.config.get("s3storage", "region")
+            self.bucket = self.config.get("s3storage", "bucket")
+            self.enabled = True
+            self.s3client = None
+            self.s3session = None
+            gevent.spawn(self._start_connection)
+
+    def _start_connection(self):
+        self.s3session = botocore.session.get_session()
+        self.s3session.set_credentials(self.accesskey, self.secretkey)
+        self.s3client = self.s3session.create_client(
+            's3',
+            endpoint_url=self.endpoint,
+            region_name=self.region,
+            config=botocore.config.Config(signature_version=self.version)
+        )
+        try:
+            self.s3client.head_bucket(Bucket=self.bucket)
+            self._initial_connection_happend = True
+        except ClientError as e:
+            logger.error("Could not establish s3 connection to bucket '%s' on %s. Received error: %s" % (self.bucket, self.endpoint, e.response['Error']['Message']))
+
+    def insert(self, attack_event):
+        if self._initial_connection_happend:
+            if attack_event.file_sha256 is not None:
+                if attack_event.known_file:
+                    logger.debug('sha256:{0} / md5:{1} is a known file, it will not be uploaded.'.format(attack_event.file_sha256, attack_event.file_name))
+                    return
+                with file(os.path.join(self.files_dir, attack_event.file_name), 'r') as file_handler:
+                    try:
+                        # check if file exists in bucket
+                        searchFile = self.s3client.list_objects_v2(Bucket=self.bucket, Prefix=attack_event.file_sha256)
+                        if (len(searchFile.get('Contents', []))) == 1 and str(searchFile.get('Contents', [])[0]['Key']) == attack_event.file_sha256:
+                            logger.debug('Not storing file (sha256:{0}) to s3 bucket "{1}" on {2} as it already exists in the bucket.'.format(attack_event.file_sha256, self.bucket, self.endpoint))
+                            return
+                        # upload file to s3
+                        self.s3client.put_object(Bucket=self.bucket, Body=file_handler, Key=attack_event.file_sha256)
+                        logger.debug('Storing file (sha256:{0}) using s3 bucket "{1}" on {2}'.format(attack_event.file_sha256, self.bucket, self.endpoint))
+                    except ClientError as e:
+                        logger.warning("Received error: %s", e.response['Error']['Message'])
+        else:
+            logger.warning('Not storing attack file because initial s3 connection test did not succeeded')
+

glastopf/modules/reporting/main/log_sql.py

@@ -18,7 +18,7 @@
 #import json
 import logging
 
-from sqlalchemy import Table, Column, Integer, String, MetaData, TEXT
+from sqlalchemy import Table, Column, Integer, String, MetaData, TEXT, Boolean
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import exc
 import glastopf.modules.processing.ip_profile as ipp
@@ -78,8 +78,10 @@ def setup_mapping(self):
             Column('request_raw', TEXT),
             Column('pattern', String(20)),
             Column('filename', String(500)),
+            Column('file_sha256', String(500)),
             Column('version', String(10)),
             Column('sensorid', String(36)),
+            Column('known_file', Boolean())
         )
         #only creates if it cant find the schema
         meta.create_all(self.engine)

requirements.txt

@@ -14,3 +14,4 @@ hpfeeds
 pylibinjection
 libtaxii>=1.1
 python-logstash
+botocore