Added more precise timestamps

* usage of system timezone
* try to use response Date header from items added from context menu

ElasticBurp.py

@@ -20,9 +20,12 @@
 from elasticsearch_dsl import Index
 from doc_HttpRequestResponse import DocHTTPRequestResponse
 from datetime import datetime
+from email.utils import parsedate_tz, mktime_tz
+from tzlocal import get_localzone
 import re
 
-reHeader = re.compile("^(.*?):\s*(.*)$")
+tz = get_localzone()
+reDateHeader = re.compile("^Date:\s*(.*)$", flags=re.IGNORECASE)
 
 ### Config (TODO: move to config tab) ###
 ES_host = "localhost"
@@ -40,14 +43,16 @@ def registerExtenderCallbacks(self, callbacks):
         self.callbacks.registerContextMenuFactory(self)
         self.out = callbacks.getStdout()
 
+        self.lastTimestamp = None
+
         res = connections.create_connection(hosts=[ES_host])
         idx = Index(ES_index)
         idx.doc_type(DocHTTPRequestResponse)
         DocHTTPRequestResponse.init()
         try:
             idx.create()
         except:
-            print("Index already exists")
+            pass
 
     ### IHttpListener ###
     def processHttpMessage(self, tool, isRequest, msg):
@@ -70,14 +75,14 @@ def menuAddToES(e):
             i = 0
             for msg in msgs:
                 if not Burp_onlyResponses or msg.getResponse():
-                    self.saveToES(msg)
+                    self.saveToES(msg, timeStampFromResponse=True)
                 i += 1
                 progress.setProgress(i)
             progress.close()
         return menuAddToES
 
     ### Interface to ElasticSearch ###
-    def saveToES(self, msg):
+    def saveToES(self, msg, timeStampFromResponse=False):
         httpService = msg.getHttpService()
         doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(), host=httpService.getHost(), port=httpService.getPort())
 
@@ -147,9 +152,13 @@ def saveToES(self, msg):
             doc.response.inferred_content_type = iResponse.getInferredMimeType()
 
             headers = iResponse.getHeaders()
+            dateHeader = None
             for header in headers:
                 try:
                     doc.add_response_header(header)
+                    match = reDateHeader.match(header)
+                    if match:
+                        dateHeader = match.group(1)
                 except:
                     doc.response.responseline = header
 
@@ -167,4 +176,12 @@ def saveToES(self, msg):
             bodyOffset = iResponse.getBodyOffset()
             doc.response.body = response[bodyOffset:].tostring().decode("ascii", "replace")
 
+            if timeStampFromResponse:
+                if dateHeader:
+                    try:
+                        doc.timestamp = datetime.fromtimestamp(mktime_tz(parsedate_tz(dateHeader)), tz) # try to use date from response header "Date"
+                        self.lastTimestamp = doc.timestamp
+                    except:
+                        doc.timestamp = self.lastTimestamp      # fallback: last stored timestamp. Else: now
+
         doc.save()

README.md

@@ -23,3 +23,4 @@ combines Burp Suite with the search power of ElasticSearch.
   * Set `host: "127.0.0.1"` in `/opt/kibana/config/kibana.yml`.
 3. Load ElasticBurp.py as Python extension in Burp Extender.
   * Jython module folder must be set to WASE source root (for elasticsearch and elasticsearch_dsl)
+  * tzlocal must be installed in the used Jython environment.

doc_HttpRequestResponse.py

@@ -20,11 +20,12 @@
 from datetime import datetime
 import re
 from WASEHTMLParser import WASEHTMLParser
+from tzlocal import get_localzone
 
 reHeader = re.compile("^(.*?):\s*(.*)$")
+tz = get_localzone()
 
 def parse_header(header):
-    # BUG: erste Zeile auch enthalten
     # TODO: support for multiline headers
     match = reHeader.search(header)
     if match:
@@ -117,7 +118,8 @@ def add_response_cookie(self, name, value, domain=None, path=None, expiration=No
         self.response.cookienames.append(cookie['name'])
 
     def save(self, **kwargs):
-        self.timestamp = datetime.now()                 # TODO: adjust timestamp to current timezone / TODO: timestamp options: now (as is), request and response
+        if not self.timestamp:
+            self.timestamp = datetime.now(tz)                 # TODO: timestamp options: now (as is), request and response
         if (self.response.inferred_content_type and self.response.inferred_content_type == "HTML") or (not self.response.inferred_content_type and "HTML" in self.response.content_type or "html" in self.response.content_type):
             parser = WASEHTMLParser()
             parser.feed(self.response.body)