Add blocklist verification

src/tests/Blocklist.js

@@ -0,0 +1,61 @@
+/**
+ * List of URLs for different blocklist categories.
+ */
+const lists = {
+	malware: 'https://blocklistproject.github.io/Lists/alt-version/malware-nl.txt',
+	piracy: 'https://blocklistproject.github.io/Lists/alt-version/piracy-nl.txt',
+	porn: 'https://blocklistproject.github.io/Lists/alt-version/porn-nl.txt'
+};
+const cache = {}; // To cache the fetched lists
+
+/**
+ * Checks if a given domain is present in any of the blocklists.
+ *
+ * @param {string} domain - The domain to check against the blocklists.
+ * @returns {Promise<void>} Resolves if the domain is not found in any list, otherwise throws an error.
+ * @throws Will throw an error if the domain is found in any blocklist, specifying the category.
+ */
+export default async function(domain) {
+	const listPromises = Object.entries(lists).map(async ([list, url]) => {
+		const domainSet = await fetchAndCacheList(url);
+		if (domainSet.has(domain)) {
+			throw new Error(`${domain} is categorized as a ${list} website.`);
+		}
+	});
+
+	await Promise.all(listPromises);
+	console.debug(`Blocklist ${domain} is not found in any list`);
+}
+
+/**
+ * Fetches a blocklist from a given URL, parses it, and caches the result.
+ *
+ * @param {string} url - The URL of the blocklist to fetch.
+ * @returns {Promise<Set<string>>} A promise that resolves to a set of domains from the blocklist.
+ */
+async function fetchAndCacheList(url) {
+	if (!cache[url]) {
+		const res = await fetch(url, {
+			headers: {
+				'user-agent': '2factorauth/twofactorauth (+https://2fa.directory/bots)'
+			},
+			cf: {
+				cacheEverything: true,
+				cacheTtl: 24 * 60 // Cache 1 day
+			}
+		});
+		const text = await res.text();
+		const lines = text.split('\n');
+		const domainSet = new Set();
+
+		lines.forEach(line => {
+			const trimmedLine = line.trim();
+			if (trimmedLine && !trimmedLine.startsWith('#')) {
+				domainSet.add(trimmedLine);
+			}
+		});
+
+		cache[url] = domainSet;
+	}
+	return cache[url];
+}

src/tests/SimilarWeb.js

@@ -15,7 +15,7 @@ export default async function similarWeb(domain, env) {
 	const json = await res.json();
 
 	// Soft fail on failure
-	console.log(json.meta.status)
+	// console.log(json.meta.status)
 	if(json.meta.status !== 'Success') return 1;
 
 	if (!Object.keys(json).includes('similar_rank'))

src/twofactorauth.js

@@ -1,5 +1,6 @@
 import similarWeb from './tests/SimilarWeb.js';
 import Facebook from './tests/Facebook.js';
+import Blocklist from './tests/Blocklist.js';
 
 export default async function(req, env) {
 	const { pr, repo } = req.params;
@@ -10,12 +11,15 @@ export default async function(req, env) {
 
 	for (const entry of entries) {
 		try {
+
 			// Validate primary domain
 			await similarWeb(entry.domain, env);
+			await Blocklist(entry.domain)
 
 			// Validate any additional domains
 			for (const domain of entry['additional-domains'] || []) {
 				await similarWeb(domain, env);
+				await Blocklist(entry.domain);
 			}
 
 			// Validate Facebook contact if present