implement getting the source IP address of a request

liberapay/main.py

@@ -1,5 +1,6 @@
 from __future__ import division
 
+from ipaddress import ip_address
 import os
 import signal
 import string
@@ -24,9 +25,9 @@
 from liberapay.security import authentication, csrf, set_default_security_headers
 from liberapay.utils import b64decode_s, b64encode_s, erase_cookie, http_caching, i18n, set_cookie
 from liberapay.utils.state_chain import (
-    create_response_object, canonize, insert_constants, _dispatch_path_to_filesystem,
-    merge_exception_into_response, return_500_for_exception, turn_socket_error_into_50X,
-    overwrite_status_code_of_gateway_errors,
+    attach_environ_to_request, create_response_object, canonize, insert_constants,
+    _dispatch_path_to_filesystem, merge_exception_into_response, return_500_for_exception,
+    turn_socket_error_into_50X, overwrite_status_code_of_gateway_errors,
 )
 from liberapay.renderers import csv_dump, jinja2, jinja2_jswrapped, jinja2_xml_min, scss
 from liberapay.website import website
@@ -105,6 +106,7 @@ def _assert(x):
 algorithm = website.algorithm
 algorithm.functions = [
     algorithm['parse_environ_into_request'],
+    attach_environ_to_request,
     algorithm['insert_variables_for_aspen'],
     algorithm['parse_body_into_request'],
     algorithm['raise_200_for_OPTIONS'],
@@ -152,6 +154,37 @@ def _assert(x):
 # Monkey patch aspen and pando
 # ============================
 
+if hasattr(pando.http.request.Request, 'source'):
+    raise Warning('pando.http.request.Request.source already exists')
+def _source(self):
+    def f():
+        addr = ip_address(self.environ[b'REMOTE_ADDR'].decode('ascii'))
+        trusted_proxies = getattr(self.website, 'trusted_proxies', None)
+        forwarded_for = self.headers.get(b'X-Forwarded-For')
+        if not trusted_proxies or not forwarded_for:
+            return addr
+        for networks in trusted_proxies:
+            is_trusted = False
+            for network in networks:
+                is_trusted = addr.is_private if network == 'private' else addr in network
+                if is_trusted:
+                    break
+            if not is_trusted:
+                return addr
+            i = forwarded_for.rfind(b',')
+            try:
+                addr = ip_address(forwarded_for[i+1:].decode('ascii').strip())
+            except (UnicodeDecodeError, ValueError):
+                return addr
+            if i == -1:
+                return addr
+            forwarded_for = forwarded_for[:i]
+        return addr
+    r = f()
+    self.__dict__['source'] = r
+    return r
+pando.http.request.Request.source = property(_source)
+
 if hasattr(pando.Response, 'encode_url'):
     raise Warning('pando.Response.encode_url() already exists')
 def _encode_url(url):

liberapay/utils/__init__.py

@@ -402,3 +402,12 @@ def check_address(addr):
     if addr['Country'] == 'US' and not addr.get('Region'):
         return False
     return True
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(path):
+            return
+        raise

liberapay/utils/state_chain.py

@@ -16,6 +16,11 @@
 from ..exceptions import LazyResponse
 
 
+def attach_environ_to_request(environ, request, website):
+    request.environ = environ
+    request.website = website
+
+
 def create_response_object(request, website):
     response = Response()
     response.request = request

liberapay/wireup.py

@@ -1,16 +1,20 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from collections import OrderedDict
+from ipaddress import ip_network
 import json
 import logging
 import os
 import re
 import socket
 import signal
 from subprocess import call
+from time import time
 import traceback
 
 from six import text_type as str
+from six.moves.urllib.parse import quote as urlquote
+from six.moves.urllib.request import urlretrieve
 
 from algorithm import Algorithm
 import pando
@@ -35,7 +39,7 @@
 from liberapay.models.repository import Repository
 from liberapay.models import DB
 from liberapay.security.authentication import ANON
-from liberapay.utils import find_files, markdown
+from liberapay.utils import find_files, markdown, mkdir_p
 from liberapay.utils.emails import compile_email_spt
 from liberapay.utils.http_caching import asset_etag
 from liberapay.utils.i18n import (
@@ -159,6 +163,7 @@ class AppConf(object):
         smtp_username=str,
         smtp_password=str,
         smtp_use_tls=bool,
+        trusted_proxies=list,
         twitch_id=str,
         twitch_secret=str,
         twitter_callback=str,
@@ -204,6 +209,33 @@ def app_conf(db):
     return {'app_conf': app_conf}
 
 
+def trusted_proxies(app_conf, env):
+    if not app_conf:
+        return
+    def parse_network(net):
+        if net == 'private':
+            return [net]
+        elif net.startswith('https://'):
+            d = env.log_dir + '/trusted_proxies/'
+            mkdir_p(d)
+            filename = d + urlquote(net, '')
+            skip_download = (
+                os.path.exists(filename) and
+                os.stat(filename).st_size > 0 and
+                os.stat(filename).st_mtime > time() - 60*60*24*7
+            )
+            if not skip_download:
+                urlretrieve(net, filename)
+            with open(filename, 'rb') as f:
+                return [ip_network(x) for x in f.read().decode('ascii').strip().split()]
+        else:
+            return [ip_network(net)]
+    return {'trusted_proxies': [
+        sum((parse_network(net) for net in networks), [])
+        for networks in (app_conf.trusted_proxies or ())
+    ]}
+
+
 def mail(app_conf, project_root='.'):
     if not app_conf:
         return
@@ -601,6 +633,7 @@ def s3(env):
     accounts_elsewhere,
     load_scss_variables,
     s3,
+    trusted_proxies,
 )
 
 

requirements_base.txt

@@ -165,3 +165,7 @@ botocore==1.5.31 \
 boto3==1.4.4 \
     --hash=sha256:5050c29353fec97301116386f469fa5858ccf47201623b53cf9f74e603bda52f \
     --hash=sha256:518f724c4758e5a5bed114fbcbd1cf470a15306d416ff421a025b76f1d390939
+
+ipaddress==1.0.18 \
+    --hash=sha256:d34cf15d95ce9a734560f7400a8bd2ac2606f378e2a1d0eadbf1c98707e7c74a \
+    --hash=sha256:5d8534c8e185f2d8a1fda1ef73f2c8f4b23264e8e30063feeb9511d492a413e1

sql/branch.sql

@@ -43,4 +43,5 @@ SELECT clean_up_counters('test', 0.01) = 1;
 
 
 INSERT INTO app_conf (key, value) VALUES
-    ('clean_up_counters_every', '3600'::jsonb);
+    ('clean_up_counters_every', '3600'::jsonb),
+    ('trusted_proxies', '[]'::jsonb);

tests/py/test_request_source.py

@@ -0,0 +1,59 @@
+# coding: utf8
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from ipaddress import IPv4Network
+
+from liberapay.testing import Harness
+
+
+class Tests(Harness):
+
+    @classmethod
+    def setUpClass(cls):
+        super(Tests, cls).setUpClass()
+        cls.website._trusted_proxies = getattr(cls.website, 'trusted_proxies', None)
+        cls.website.trusted_proxies = [
+            [IPv4Network('10.0.0.0/8')],
+            [IPv4Network('141.101.64.0/18')],
+        ]
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.website.trusted_proxies = cls.website._trusted_proxies
+        super(Tests, cls).tearDownClass()
+
+    def request(self, forwarded_for, source, **kw):
+        kw['HTTP_X_FORWARDED_FOR'] = forwarded_for
+        kw['REMOTE_ADDR'] = source
+        kw.setdefault('return_after', 'attach_environ_to_request')
+        kw.setdefault('want', 'request')
+        return self.client.GET('/', **kw).source
+
+    def test_request_source_with_invalid_header_from_trusted_proxy(self):
+        source = str(self.request(b'f\xc3\xa9e, \t bar', b'10.0.0.1'))
+        assert source == '10.0.0.1'
+
+    def test_request_source_with_invalid_header_from_untrusted_proxy(self):
+        source = str(self.request(b'f\xc3\xa9e, \tbar', b'8.8.8.8'))
+        assert source == '8.8.8.8'
+
+    def test_request_source_with_valid_headers_from_trusted_proxies(self):
+        source = str(self.request(b'8.8.8.8,141.101.69.139', b'10.0.0.1'))
+        assert source == '8.8.8.8'
+        source = str(self.request(b'8.8.8.8', b'10.0.0.2'))
+        assert source == '8.8.8.8'
+
+    def test_request_source_with_valid_headers_from_untrusted_proxies(self):
+        # 8.8.8.8 claims that the request came from 0.0.0.0, but we don't trust 8.8.8.8
+        source = str(self.request(b'0.0.0.0, 8.8.8.8,141.101.69.140', b'10.0.0.1'))
+        assert source == '8.8.8.8'
+        source = str(self.request(b'0.0.0.0, 8.8.8.8', b'10.0.0.1'))
+        assert source == '8.8.8.8'
+
+    def test_request_source_with_forged_headers_from_untrusted_client(self):
+        # 8.8.8.8 claims that the request came from a trusted proxy, but we don't trust 8.8.8.8
+        source = str(self.request(b'0.0.0.0,141.101.69.141, 8.8.8.8,141.101.69.142', b'10.0.0.1'))
+        assert source == '8.8.8.8'
+        source = str(self.request(b'0.0.0.0, 141.101.69.143, 8.8.8.8', b'10.0.0.1'))
+        assert source == '8.8.8.8'