add asn support, zoho provider

blacklanternsecurity · Jan 28, 2024 · 7dadc77 · 7dadc77
1 parent a8639c8
commit 7dadc77
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 18 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -54,6 +54,11 @@ jobs:
         run: |
           pip install poetry
           poetry install
+      - name: Install ASN database
+        run: |
+          pip install pyasn
+          pyasn_util_download.py --latestv46
+          pyasn_util_convert.py --single rib.*.bz2 asn.db
       - name: Update cloud_providers.json
         run: poetry run cloudcheck forceupdate
       - name: git add cloud_providers.json
@@ -71,7 +76,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Dynamic version
-        run: "sed -i s/2.0.0.0/2.1.0.$(git rev-list HEAD --count)/g pyproject.toml"
+        run: "sed -i s/3.0.0.0/3.0.0.$(git rev-list HEAD --count)/g pyproject.toml"
       - uses: actions/setup-python@v4
         with:
           python-version: "3.x"

diff --git a/cloud_providers.json b/cloud_providers.json
@@ -1,5 +1,6 @@
 {
     "Akamai": {
+        "asns": [],
         "bucket_name_regex": "",
         "cidrs": [
             "104.64.0.0/10",
@@ -103,12 +104,13 @@
             "srtcdn.net"
         ],
         "ips_url": "https://techdocs.akamai.com/property-manager/pdfs/akamai_ipv4_ipv6_CIDRs-txt.zip",
-        "last_updated": "2024-01-28T03:55:07.228000",
+        "last_updated": "2024-01-27T23:48:59.259783",
         "name": "Akamai",
         "provider_type": "cdn",
         "regexes": {}
     },
     "Amazon": {
+        "asns": [],
         "bucket_name_regex": "[a-z0-9_][a-z0-9-\\.]{1,61}[a-z0-9]",
         "cidrs": [
             "100.20.0.0/14",
@@ -6527,7 +6529,7 @@
             "thinkboxsoftware.com"
         ],
         "ips_url": "https://ip-ranges.amazonaws.com/ip-ranges.json",
-        "last_updated": "2024-01-28T03:55:07.204434",
+        "last_updated": "2024-01-27T23:48:59.258779",
         "name": "Amazon",
         "provider_type": "cloud",
         "regexes": {
@@ -6537,6 +6539,7 @@
         }
     },
     "Azure": {
+        "asns": [],
         "bucket_name_regex": "[a-z0-9][a-z0-9-_\\.]{1,61}[a-z0-9]",
         "cidrs": [
             "102.133.0.0/18",
@@ -10223,7 +10226,7 @@
             "windowsazurestatus.cn"
         ],
         "ips_url": "https://download.microsoft.com/download/0/1/8/018E208D-54F8-44CD-AA26-CD7BC9524A8C/PublicIPs_20200824.xml",
-        "last_updated": "2024-01-28T03:55:07.040782",
+        "last_updated": "2024-01-27T23:48:59.259920",
         "name": "Azure",
         "provider_type": "cloud",
         "regexes": {
@@ -10233,6 +10236,7 @@
         }
     },
     "Cloudflare": {
+        "asns": [],
         "bucket_name_regex": "",
         "cidrs": [
             "103.21.244.0/22",
@@ -10292,12 +10296,13 @@
             "workers.dev"
         ],
         "ips_url": "https://api.cloudflare.com/client/v4/ips",
-        "last_updated": "2024-01-28T03:55:07.046258",
+        "last_updated": "2024-01-27T23:48:58.068943",
         "name": "Cloudflare",
         "provider_type": "cdn",
         "regexes": {}
     },
     "DigitalOcean": {
+        "asns": [],
         "bucket_name_regex": "[a-z0-9][a-z0-9-]{2,62}",
         "cidrs": [
             "103.253.145.0/24",
@@ -11906,7 +11911,7 @@
             "nginxconfig.io"
         ],
         "ips_url": "https://digitalocean.com/geo/google.csv",
-        "last_updated": "2024-01-28T03:55:07.324668",
+        "last_updated": "2024-01-27T23:48:58.066018",
         "name": "DigitalOcean",
         "provider_type": "cloud",
         "regexes": {
@@ -11916,6 +11921,7 @@
         }
     },
     "GitHub": {
+        "asns": [],
         "bucket_name_regex": "",
         "cidrs": [
             "104.208.0.0/19",
@@ -16183,12 +16189,13 @@
             "github.com"
         ],
         "ips_url": "https://api.github.com/meta",
-        "last_updated": "2024-01-28T03:55:07.119871",
+        "last_updated": "2024-01-27T23:48:58.068373",
         "name": "GitHub",
         "provider_type": "cdn",
         "regexes": {}
     },
     "Google": {
+        "asns": [],
         "bucket_name_regex": "[a-z0-9][a-z0-9-_\\.]{1,61}[a-z0-9]",
         "cidrs": [
             "104.154.113.0/24",
@@ -16857,7 +16864,7 @@
             "googleapis.com"
         ],
         "ips_url": "https://www.gstatic.com/ipranges/cloud.json",
-        "last_updated": "2024-01-28T03:55:07.127058",
+        "last_updated": "2024-01-27T23:48:59.258491",
         "name": "Google",
         "provider_type": "cloud",
         "regexes": {
@@ -16868,6 +16875,7 @@
         }
     },
     "Oracle": {
+        "asns": [],
         "bucket_name_regex": "",
         "cidrs": [
             "129.146.0.0/21",
@@ -17555,9 +17563,81 @@
             "sun.com"
         ],
         "ips_url": "https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json",
-        "last_updated": "2024-01-28T03:55:07.400588",
+        "last_updated": "2024-01-27T23:48:59.259601",
         "name": "Oracle",
         "provider_type": "cloud",
         "regexes": {}
+    },
+    "Zoho": {
+        "asns": [],
+        "bucket_name_regex": "",
+        "cidrs": [
+            "135.84.80.0/23",
+            "135.84.82.0/23",
+            "136.143.160.0/23",
+            "136.143.162.0/23",
+            "136.143.176.0/23",
+            "136.143.178.0/23",
+            "136.143.180.0/23",
+            "136.143.182.0/23",
+            "136.143.184.0/23",
+            "136.143.184.0/24",
+            "136.143.186.0/23",
+            "136.143.188.0/23",
+            "136.143.190.0/23",
+            "165.173.128.0/23",
+            "165.173.164.0/24",
+            "165.173.165.0/24",
+            "165.173.174.0/23",
+            "165.173.178.0/24",
+            "165.173.180.0/23",
+            "165.173.182.0/23",
+            "165.173.186.0/23",
+            "165.173.186.0/24",
+            "165.173.188.0/24",
+            "165.173.189.0/24",
+            "199.67.64.0/23",
+            "199.67.69.0/24",
+            "199.67.84.0/23",
+            "199.67.84.0/24",
+            "199.67.85.0/24",
+            "199.67.86.0/24",
+            "199.67.87.0/24",
+            "199.67.92.0/24",
+            "204.141.160.0/24",
+            "204.141.32.0/23",
+            "204.141.42.0/23",
+            "2602:801:c000::/47",
+            "2602:801:c004::/47",
+            "2602:801:c006::/47",
+            "2602:801:c006::/48",
+            "2602:801:c008::/47",
+            "2602:801:c00e::/48",
+            "2607:13c0::/47",
+            "31.186.226.0/24",
+            "31.186.243.0/24",
+            "65.154.166.0/24",
+            "8.39.54.0/23"
+        ],
+        "domains": [
+            "zoho.com",
+            "zoho.com.au",
+            "zoho.eu",
+            "zoho.in",
+            "zohocdn.com",
+            "zohomeetups.com",
+            "zohomerchandise.com",
+            "zohopublic.com",
+            "zohoschools.com",
+            "zohostatic.com",
+            "zohostatic.in",
+            "zohouniversity.com",
+            "zohowebstatic.com"
+        ],
+        "ips_url": "",
+        "last_updated": "2024-01-27T23:48:58.069073",
+        "name": "Zoho",
+        "provider_type": "cloud",
+        "regexes": {}
     }
 }
diff --git a/cloudcheck/providers/base.py b/cloudcheck/providers/base.py
@@ -1,5 +1,6 @@
 import re
 import httpx
+import pyasn
 import logging
 import ipaddress
 import traceback
@@ -13,6 +14,12 @@
 log = logging.getLogger("cloudcheck.providers")
 
 
+try:
+    asndb = pyasn.pyasn("asn.db")
+except Exception:
+    asndb = None
+
+
 class CloudProviderJSON(BaseModel):
     name: str = ""
     domains: List[str] = []
@@ -22,6 +29,7 @@ class CloudProviderJSON(BaseModel):
     regexes: Dict[str, List[str]] = {}
     provider_type: str = "cloud"
     ips_url: str = ""
+    asns: List[str] = []
 
     @field_validator("cidrs")
     @classmethod
@@ -38,6 +46,7 @@ class BaseCloudProvider:
     headers = {
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36"
     }
+    asns = []
 
     def __init__(self, j, httpx_client=None):
         self._httpx_client = httpx_client
@@ -63,17 +72,27 @@ def __init__(self, j, httpx_client=None):
 
     async def update(self):
         try:
-            response = await self.httpx_client.get(
-                self.ips_url, follow_redirects=True, headers=self.headers
-            )
-            ranges = self.parse_response(response)
-            if ranges:
-                self.ranges = CidrRanges(ranges)
-                self.last_updated = datetime.now()
+            self.last_updated = datetime.now()
+            self.ranges = CidrRanges(self.get_subnets())
+            if self.ips_url:
+                response = await self.httpx_client.get(
+                    self.ips_url, follow_redirects=True, headers=self.headers
+                )
+                ranges = self.parse_response(response)
+                if ranges:
+                    self.ranges.cidrs.update(ranges)
         except Exception as e:
             log.warning(f"Error retrieving {self.ips_url}: {e}")
             log.warning(traceback.format_exc())
 
+    def get_subnets(self):
+        subnets = set()
+        if asndb is not None:
+            for asn in self.asns:
+                for subnet in asndb.get_as_prefixes(asn):
+                    subnets.add(ipaddress.ip_network(subnet, strict=False))
+        return subnets
+
     def to_json(self):
         """
         domains: List[str] = []

diff --git a/cloudcheck/providers/zoho.py b/cloudcheck/providers/zoho.py
@@ -0,0 +1,23 @@
+from .base import BaseCloudProvider
+
+
+class Zoho(BaseCloudProvider):
+    domains = [
+        "zoho.com",
+        "zoho.com.au",
+        "zoho.eu",
+        "zoho.in",
+        "zohocdn.com",
+        "zohomeetups.com",
+        "zohomerchandise.com",
+        "zohopublic.com",
+        "zohoschools.com",
+        "zohostatic.com",
+        "zohostatic.in",
+        "zohouniversity.com",
+        "zohowebstatic.com",
+    ]
+    asns = [
+        2639,
+    ]
+    provider_type = "cloud"
diff --git a/cloudcheck/test/test_cloudcheck.py b/cloudcheck/test/test_cloudcheck.py
@@ -58,6 +58,9 @@ async def test_cloudcheck():
 
     assert cloud_providers.last_updated
 
+    zoho = cloud_providers.providers["zoho"]
+    assert zoho.asns
+
 
 if __name__ == "__main__":
     asyncio.run(test_cloudcheck())
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cloudcheck"
-version = "2.0.0.0"
+version = "3.0.0.0"
 description = "Check whether an IP address belongs to a cloud provider"
 authors = ["TheTechromancer"]
 license = "GPL-3.0"
@@ -20,6 +20,7 @@ cloudcheck = 'cloudcheck.cloudcheck:main'
 python = "^3.9"
 pydantic = "^2.4.2"
 httpx = "^0.26.0"
+pyasn = "^1.6.2"
 
 [tool.poetry.group.dev.dependencies]
 black = ">=22.12,<25.0"