Initial Proof of Concept with test code

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+*.class

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "elasticsearch-py"]
+	path = elasticsearch-py
+	url = https://github.com/elastic/elasticsearch-py.git
+[submodule "elasticsearch-dsl-py"]
+	path = elasticsearch-dsl-py
+	url = https://github.com/elastic/elasticsearch-dsl-py.git

README.md

@@ -0,0 +1,24 @@
+# WASE
+
+WASE is a shortcut for Web Audit Search Engine. It's a framework for indexing HTTP requests/responses while web
+application audits in an ElasticSearch instance and enriching it with useful data. The indexed data can then be searched
+and aggregated with ElasticSearch queries or with Kibana.
+
+Currently WASE contains the following parts:
+
+* doc\_HttpRequestResponse.py: a library that implements the DocHTTPRequestResponse class. This class is an
+  elasticsearch\_dsl-based storage class of HTTP requests/responses (derived from Burps data structures and API).
+* ElasticBurp: a Burp plugin that feeds requests/responses into ElasticSearch.
+
+## ElasticBurp
+
+Scared about the weak searching performance of Burp Suite? Are you missing possibilities to search in Burp? ElasticBurp
+combines Burp Suite with the search power of ElasticSearch.
+
+### Installation
+
+1. Install ElasticSearch and Kibana.
+2. Configure both - For security reasons it is recommend to let them listen on localhost:
+  * Set `network.host: 127.0.0.1` in `/etc/elasticsearch/elasticsearch.yml`.
+  * Set `host: "127.0.0.1"` in `/opt/kibana/config/kibana.yml`.
+3. Load ElasticBurp.py as Python extension in Burp Extender.

doc_HttpRequestResponse.py

@@ -0,0 +1,104 @@
+# WASE - Web Audit Search Engine
+# doc_HttpRequestResponse.py: Implementation of the core data structure
+# 
+# Copyright 2016 Thomas Patzke <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from elasticsearch_dsl import DocType, String, Integer, Short, Date, Object, Nested
+from datetime import datetime
+import re
+
+reHeader = re.compile("^(.*?):\s*(.*)$")
+
+def parse_header(header):
+    match = reHeader.search(header)
+    if match:
+        return { 'name': match.group(1), 'value': match.group(2) }
+    else:
+        raise ValueError("No header matched")
+
+class DocHTTPRequestResponse(DocType):
+    class Meta:
+        doc_type = 'HTTPRequestResponse'
+
+    timestamp = Date()
+    protocol = String()
+    host = String()
+    port = Integer()
+    request = Object(
+            properties = {
+                'method': String(),
+                'url': String(),
+                'content_type': String(),
+                'headers': Nested(
+                    properties = {
+                        'name': String(),
+                        'value': String()
+                        }
+                    ),
+                'parameters': Nested(
+                    properties = {
+                        'type': String(),
+                        'name': String(),
+                        'value': String()
+                        }
+                    ),
+                'body': String()
+                }
+            )
+    response = Object(
+            properties = {
+                'status': Short(),
+                'stated_content_type': String(),
+                'inferred_content_type': String(),
+                'headers': Nested(
+                    properties = {
+                        'name': String(),
+                        'value': String()
+                        }
+                    ),
+                'cookies': Nested(
+                    properties = {
+                        'domain': String(),
+                        'expiration': Date(),
+                        'name': String(),
+                        'path': String(),
+                        'value': String()
+                        }
+                    ),
+                'body': String()
+                }
+            )
+
+    def add_request_header(self, header):
+        parsed = parse_header(header)
+        self.request.headers.append(parsed)
+
+    def add_response_header(self, header):
+        parsed = parse_header(header)
+        self.response.headers.append(parsed)
+
+    def add_request_parameter(self, typename, name, value):
+        param = { 'type': typename, 'name': name, 'value': value }
+        self.request.parameters.append(param)
+
+    def add_response_cookie(self, name, value, domain=None, path=None, expiration=None):
+        cookie = { 'name': name, 'value': value, 'domain': domain, 'path': path, 'expiration': expiration }
+        self.response.cookies.append(cookie)
+
+    def save(self, **kwargs):
+        self.timestamp = datetime.now()
+        return super(DocHTTPRequestResponse, self).save(**kwargs)
+

queries.txt

@@ -0,0 +1,63 @@
+All requests with HEADERNAME header:
+{
+    "query": {
+        "nested": {
+            "path": "response.headers",
+            "query": {
+                "match_phrase": {
+                    "response.headers.name": "HEADERNAME"
+                }
+            }
+        }
+    }
+}
+
+All requests without HEADERNAME header:
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "nested": {
+                    "path": "response.headers",
+                    "query": {
+                        "match_phrase": {
+                            "response.headers.name": "HEADERNAME"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+All POST requests:
+{
+    "query": {
+        "match_phrase": {
+            "request.method": "POST"
+        }
+    }
+}
+
+All POST requests without parameter PARAMNAME:
+{
+    "query": {
+        "bool": {
+            "must": {
+                "match_phrase": {
+                    "request.method": "POST"
+                }
+            },
+            "must_not": {
+                "nested": {
+                    "path": "request.parameters",
+                    "query": {
+                        "match_phrase": {
+                            "request.parameters.name": "PARAMNAME"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

test.py

@@ -0,0 +1,76 @@
+from doc_HttpRequestResponse import DocHTTPRequestResponse
+from elasticsearch_dsl.connections import connections
+from elasticsearch_dsl import Index
+from datetime import datetime
+
+connections.create_connection(hosts=["localhost"])
+
+idx = Index("test")
+idx.doc_type(DocHTTPRequestResponse)
+#idx.create()
+
+DocHTTPRequestResponse.init()
+
+d = DocHTTPRequestResponse(
+        protocol="http",
+        host="foobar.com",
+        port=80
+        )
+d.add_request_header("User-Agent: foobar")
+d.add_request_parameter("url", "id", "123")
+d.add_request_parameter("url", "doc", "234")
+d.add_response_header("X-Content-Type-Options: nosniff")
+d.add_response_header("X-Frame-Options: DENY")
+d.add_response_header("X-XSS-Protection: 1; mode=block")
+d.add_response_cookie("SESSIONID", "foobar1234")
+d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
+d.response.body = "This is a test!"
+d.request.method = "GET"
+d.save()
+
+d = DocHTTPRequestResponse(
+        protocol="http",
+        host="foobar.com",
+        port=80
+        )
+d.add_request_header("User-Agent: foobar")
+d.add_request_parameter("url", "id", "123")
+d.add_request_parameter("url", "doc", "456")
+d.add_response_header("X-Frame-Options: SAMEORIGIN")
+d.add_response_cookie("SESSIONID", "foobar1234")
+d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
+d.request.method = "GET"
+d.response.body = "This is a test!"
+d.save()
+
+d = DocHTTPRequestResponse(
+        protocol="http",
+        host="foobar.com",
+        port=80
+        )
+d.add_request_header("User-Agent: foobar")
+d.add_request_parameter("body", "action", "add")
+d.add_request_parameter("body", "doc", "456")
+d.add_request_parameter("body", "content", "Test")
+d.add_request_parameter("body", "csrftoken", "trulyrandom")
+d.add_response_header("X-Frame-Options: SAMEORIGIN")
+d.add_response_cookie("SESSIONID", "foobar1234")
+d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
+d.request.method = "POST"
+d.response.body = "Added!"
+d.save()
+
+d = DocHTTPRequestResponse(
+        protocol="http",
+        host="foobar.com",
+        port=80
+        )
+d.add_request_header("User-Agent: foobar")
+d.add_request_parameter("body", "action", "delete")
+d.add_request_parameter("body", "doc", "456")
+d.add_response_header("X-Frame-Options: SAMEORIGIN")
+d.add_response_cookie("SESSIONID", "foobar1234")
+d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
+d.request.method = "POST"
+d.response.body = "Deleted!"
+d.save()