diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fad32f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.swp +*.class diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..34d6d77 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "elasticsearch-py"] + path = elasticsearch-py + url = https://github.com/elastic/elasticsearch-py.git +[submodule "elasticsearch-dsl-py"] + path = elasticsearch-dsl-py + url = https://github.com/elastic/elasticsearch-dsl-py.git diff --git a/README.md b/README.md new file mode 100644 index 0000000..4b1fcd8 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# WASE + +WASE is a shortcut for Web Audit Search Engine. It's a framework for indexing HTTP requests/responses while web +application audits in an ElasticSearch instance and enriching it with useful data. The indexed data can then be searched +and aggregated with ElasticSearch queries or with Kibana. + +Currently WASE contains the following parts: + +* doc\_HttpRequestResponse.py: a library that implements the DocHTTPRequestResponse class. This class is an + elasticsearch\_dsl-based storage class of HTTP requests/responses (derived from Burps data structures and API). +* ElasticBurp: a Burp plugin that feeds requests/responses into ElasticSearch. + +## ElasticBurp + +Scared about the weak searching performance of Burp Suite? Are you missing possibilities to search in Burp? ElasticBurp +combines Burp Suite with the search power of ElasticSearch. + +### Installation + +1. Install ElasticSearch and Kibana. +2. Configure both - For security reasons it is recommend to let them listen on localhost: + * Set `network.host: 127.0.0.1` in `/etc/elasticsearch/elasticsearch.yml`. + * Set `host: "127.0.0.1"` in `/opt/kibana/config/kibana.yml`. +3. Load ElasticBurp.py as Python extension in Burp Extender. diff --git a/doc_HttpRequestResponse.py b/doc_HttpRequestResponse.py new file mode 100644 index 0000000..339f7c5 --- /dev/null +++ b/doc_HttpRequestResponse.py @@ -0,0 +1,104 @@ +# WASE - Web Audit Search Engine +# doc_HttpRequestResponse.py: Implementation of the core data structure +# +# Copyright 2016 Thomas Patzke +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from elasticsearch_dsl import DocType, String, Integer, Short, Date, Object, Nested +from datetime import datetime +import re + +reHeader = re.compile("^(.*?):\s*(.*)$") + +def parse_header(header): + match = reHeader.search(header) + if match: + return { 'name': match.group(1), 'value': match.group(2) } + else: + raise ValueError("No header matched") + +class DocHTTPRequestResponse(DocType): + class Meta: + doc_type = 'HTTPRequestResponse' + + timestamp = Date() + protocol = String() + host = String() + port = Integer() + request = Object( + properties = { + 'method': String(), + 'url': String(), + 'content_type': String(), + 'headers': Nested( + properties = { + 'name': String(), + 'value': String() + } + ), + 'parameters': Nested( + properties = { + 'type': String(), + 'name': String(), + 'value': String() + } + ), + 'body': String() + } + ) + response = Object( + properties = { + 'status': Short(), + 'stated_content_type': String(), + 'inferred_content_type': String(), + 'headers': Nested( + properties = { + 'name': String(), + 'value': String() + } + ), + 'cookies': Nested( + properties = { + 'domain': String(), + 'expiration': Date(), + 'name': String(), + 'path': String(), + 'value': String() + } + ), + 'body': String() + } + ) + + def add_request_header(self, header): + parsed = parse_header(header) + self.request.headers.append(parsed) + + def add_response_header(self, header): + parsed = parse_header(header) + self.response.headers.append(parsed) + + def add_request_parameter(self, typename, name, value): + param = { 'type': typename, 'name': name, 'value': value } + self.request.parameters.append(param) + + def add_response_cookie(self, name, value, domain=None, path=None, expiration=None): + cookie = { 'name': name, 'value': value, 'domain': domain, 'path': path, 'expiration': expiration } + self.response.cookies.append(cookie) + + def save(self, **kwargs): + self.timestamp = datetime.now() + return super(DocHTTPRequestResponse, self).save(**kwargs) + diff --git a/elasticsearch-dsl-py b/elasticsearch-dsl-py new file mode 160000 index 0000000..c7e6dcc --- /dev/null +++ b/elasticsearch-dsl-py @@ -0,0 +1 @@ +Subproject commit c7e6dcc6c5b1829eb11eaa050d70ec8d54b79559 diff --git a/elasticsearch-py b/elasticsearch-py new file mode 160000 index 0000000..271376f --- /dev/null +++ b/elasticsearch-py @@ -0,0 +1 @@ +Subproject commit 271376f85054c7cdf207690db0e9de1f1d716fc1 diff --git a/queries.txt b/queries.txt new file mode 100644 index 0000000..ecaa4dd --- /dev/null +++ b/queries.txt @@ -0,0 +1,63 @@ +All requests with HEADERNAME header: +{ + "query": { + "nested": { + "path": "response.headers", + "query": { + "match_phrase": { + "response.headers.name": "HEADERNAME" + } + } + } + } +} + +All requests without HEADERNAME header: +{ + "query": { + "bool": { + "must_not": { + "nested": { + "path": "response.headers", + "query": { + "match_phrase": { + "response.headers.name": "HEADERNAME" + } + } + } + } + } + } +} + +All POST requests: +{ + "query": { + "match_phrase": { + "request.method": "POST" + } + } +} + +All POST requests without parameter PARAMNAME: +{ + "query": { + "bool": { + "must": { + "match_phrase": { + "request.method": "POST" + } + }, + "must_not": { + "nested": { + "path": "request.parameters", + "query": { + "match_phrase": { + "request.parameters.name": "PARAMNAME" + } + } + } + } + } + } +} diff --git a/test.py b/test.py new file mode 100644 index 0000000..9139095 --- /dev/null +++ b/test.py @@ -0,0 +1,76 @@ +from doc_HttpRequestResponse import DocHTTPRequestResponse +from elasticsearch_dsl.connections import connections +from elasticsearch_dsl import Index +from datetime import datetime + +connections.create_connection(hosts=["localhost"]) + +idx = Index("test") +idx.doc_type(DocHTTPRequestResponse) +#idx.create() + +DocHTTPRequestResponse.init() + +d = DocHTTPRequestResponse( + protocol="http", + host="foobar.com", + port=80 + ) +d.add_request_header("User-Agent: foobar") +d.add_request_parameter("url", "id", "123") +d.add_request_parameter("url", "doc", "234") +d.add_response_header("X-Content-Type-Options: nosniff") +d.add_response_header("X-Frame-Options: DENY") +d.add_response_header("X-XSS-Protection: 1; mode=block") +d.add_response_cookie("SESSIONID", "foobar1234") +d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now()) +d.response.body = "This is a test!" +d.request.method = "GET" +d.save() + +d = DocHTTPRequestResponse( + protocol="http", + host="foobar.com", + port=80 + ) +d.add_request_header("User-Agent: foobar") +d.add_request_parameter("url", "id", "123") +d.add_request_parameter("url", "doc", "456") +d.add_response_header("X-Frame-Options: SAMEORIGIN") +d.add_response_cookie("SESSIONID", "foobar1234") +d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now()) +d.request.method = "GET" +d.response.body = "This is a test!" +d.save() + +d = DocHTTPRequestResponse( + protocol="http", + host="foobar.com", + port=80 + ) +d.add_request_header("User-Agent: foobar") +d.add_request_parameter("body", "action", "add") +d.add_request_parameter("body", "doc", "456") +d.add_request_parameter("body", "content", "Test") +d.add_request_parameter("body", "csrftoken", "trulyrandom") +d.add_response_header("X-Frame-Options: SAMEORIGIN") +d.add_response_cookie("SESSIONID", "foobar1234") +d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now()) +d.request.method = "POST" +d.response.body = "Added!" +d.save() + +d = DocHTTPRequestResponse( + protocol="http", + host="foobar.com", + port=80 + ) +d.add_request_header("User-Agent: foobar") +d.add_request_parameter("body", "action", "delete") +d.add_request_parameter("body", "doc", "456") +d.add_response_header("X-Frame-Options: SAMEORIGIN") +d.add_response_cookie("SESSIONID", "foobar1234") +d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now()) +d.request.method = "POST" +d.response.body = "Deleted!" +d.save()