diff --git a/README.txt b/README.txt index fe37f6a66beed..d68c1bed99818 100644 --- a/README.txt +++ b/README.txt @@ -1,3 +1,28 @@ +# Argo? + +## Setup + +**this is unverified, but seems right** + +- You need an `origin` git remote of the Mozilla Mercurial repo, along with whatever remote you're using for this fork repo: + ``` + origin hg::https://hg.mozilla.org/mozilla-unified (fetch) + origin hg::https://hg.mozilla.org/mozilla-unified (push) + opfn git@github.com:operating-function/gecko-dev.git (fetch) + opfn git@github.com:operating-function/gecko-dev.git (push) + mozilla git@github.com:mozilla/gecko-dev.git (fetch) + mozilla git@github.com:mozilla/gecko-dev.git (push) + ``` + - You probably need to follow the directions Mozilla provides in general first, including having a `git cinnabar` installation working. (https://firefox-source-docs.mozilla.org/contributing/contribution_quickref.html) + - We have a `mozilla-central` branch which is based off of the _gecko-dev_ readonly git mirror's `master`. This seems to track the `mozilla-unified` repo's `bookmarks/central` branch. + - For the sake of our purposes here, consider our `mozilla-central` branch to be "master", I suppose. + - We should attempt to keep our `mozilla-central` branch reasonably up-to-date with the upstream; or else lock to a version for now and push off the daunting task of keeping our fork updated... +- Once you have an `origin` remote and a local `gecko-dev` mirror, you should be able to `./mach build` from the root of this repo. The build process seems to require having the Mercurial `origin` present. + +--- + +# `README.txt` From original fork: + An explanation of the Firefox Source Code Directory Structure and links to project pages with documentation can be found at: diff --git a/browser/app/profile/extensions.json b/browser/app/profile/extensions.json new file mode 100644 index 0000000000000..bccaabd562568 --- /dev/null +++ b/browser/app/profile/extensions.json @@ -0,0 +1,40 @@ +{ + "addons": [ + { + "id": "warc-indicator@mozilla.org", + "syncGUID": "{c1619560-4d77-4744-997b-b0eb33c49444}", + "version": "1.0", + "type": "extension", + "loader": null, + "updateDate": 1705507200000, + "path": "features/warc-indicator@mozilla.org", + "defaultLocale": { + "name": "WARC Availability Indicator", + "description": "Shows when a WARC archive is available" + }, + "visible": true, + "active": true, + "userDisabled": false, + "appDisabled": false, + "installDate": 1705507200000 + }, + { + "id": "warc-recorder@mozilla.org", + "syncGUID": "{d1619560-4d77-4744-997b-b0eb33c49444}", + "version": "1.0", + "type": "extension", + "loader": null, + "updateDate": 1705507200000, + "path": "features/warc-recorder@mozilla.org", + "defaultLocale": { + "name": "WARC Recorder", + "description": "Records web traffic in WARC format", + }, + "visible": true, + "active": true, + "userDisabled": false, + "appDisabled": false, + "installDate": 1705507200000 + } + ] +} diff --git a/browser/app/profile/firefox.js b/browser/app/profile/firefox.js index a2c4d67dd0a30..3d4ae8d1e75b6 100644 --- a/browser/app/profile/firefox.js +++ b/browser/app/profile/firefox.js @@ -21,11 +21,21 @@ pref("browser.hiddenWindowChromeURL", "chrome://browser/content/hiddenWindowMac.xhtml"); +pref("extensions.installedDistroAddon.warc-indicator@mozilla.org", true); +pref("extensions.systemAddonSet", '{"schema":1,"addons":{"warc-indicator@mozilla.org":{"version":"1.0"}}}'); +pref("extensions.installDistroAddons", true); + +pref("extensions.installedDistroAddon.warc-recorder@mozilla.org", true); +pref("extensions.systemAddonSet", '{"schema":1,"addons":{"warc-recorder@mozilla.org":{"version":"1.0"}}}'); + +pref("extensions.activeThemeID", "default-theme@mozilla.org"); + + // Set add-ons abuse report related prefs specific to Firefox Desktop. pref("extensions.abuseReport.enabled", true); // Enables some extra Extension System Logging (can reduce performance) -pref("extensions.logging.enabled", false); +pref("extensions.logging.enabled", true); // Disables strict compatibility, making addons compatible-by-default. pref("extensions.strictCompatibility", false); @@ -56,9 +66,10 @@ pref("extensions.systemAddon.update.enabled", true); // Disable add-ons that are not installed by the user in all scopes by default. // See the SCOPE constants in AddonManager.sys.mjs for values to use here. -pref("extensions.autoDisableScopes", 15); +pref("extensions.autoDisableScopes", 0); // Scopes to scan for changes at startup. -pref("extensions.startupScanScopes", 0); +// OPFN: changed from 0 to 31 to include ALL scopes +pref("extensions.startupScanScopes", 31); pref("extensions.geckoProfiler.acceptedExtensionIds", "geckoprofiler@mozilla.com,quantum-foxfooding@mozilla.com,raptor@mozilla.org"); @@ -358,6 +369,11 @@ pref("browser.startup.windowsLaunchOnLogin.disableLaunchOnLoginPrompt", false); // Show an upgrade dialog on major upgrades. pref("browser.startup.upgradeDialog.enabled", false); +// Enable verbose extension logging +pref("extensions.logging.enabled", true); +pref("extensions.webextensions.log.level", "debug"); +pref("extensions.webextensions.log.stdout", true); + pref("browser.chrome.site_icons", true); // browser.warnOnQuit == false will override all other possible prompts when quitting or restarting pref("browser.warnOnQuit", true); @@ -635,7 +651,11 @@ pref("browser.urlbar.sponsoredTopSites", false); // Global toggle for whether the show search terms feature // can be used at all, and enabled/disabled by the user. +#if defined(EARLY_BETA_OR_EARLIER) +pref("browser.urlbar.showSearchTerms.featureGate", true); +#else pref("browser.urlbar.showSearchTerms.featureGate", false); +#endif // If true, show the search term in the Urlbar while on // a default search engine results page. @@ -2019,6 +2039,8 @@ pref("browser.newtabpage.activity-stream.hideTopSitesWithSearchParam", "mfadid=a pref("browser.aboutwelcome.enabled", true); // Used to set multistage welcome UX pref("browser.aboutwelcome.screens", ""); +// Used to enable window modal onboarding +pref("browser.aboutwelcome.showModal", false); // Experiment Manager // See Console.sys.mjs LOG_LEVELS for all possible values @@ -2263,9 +2285,6 @@ pref("privacy.trackingprotection.fingerprinting.enabled", true); // Enable cryptomining blocking by default for all channels, only on desktop. pref("privacy.trackingprotection.cryptomining.enabled", true); -// Skip earlyBlankFirstPaint by default if resistFingerprinting is enabled. -pref("privacy.resistFingerprinting.skipEarlyBlankFirstPaint", true); - pref("browser.contentblocking.database.enabled", true); // Enable Strip on Share by default on desktop @@ -3314,12 +3333,8 @@ pref("browser.backup.template.fallback-download.aurora", "https://www.mozilla.or pref("browser.backup.template.fallback-download.nightly", "https://www.mozilla.org/firefox/channel/desktop/?utm_medium=firefox-desktop&utm_source=backup&utm_campaign=firefox-backup-2024&utm_content=control#nightly"); pref("browser.backup.template.fallback-download.esr", "https://www.mozilla.org/firefox/enterprise/?utm_medium=firefox-desktop&utm_source=backup&utm_campaign=firefox-backup-2024&utm_content=control#download"); -#ifdef NIGHTLY_BUILD - // Pref to enable the new profiles - pref("browser.profiles.enabled", true); -#else - pref("browser.profiles.enabled", false); -#endif +// Pref to enable the new profiles +pref("browser.profiles.enabled", false); pref("browser.profiles.profile-name.updated", false); // Whether to allow the user to merge profile data pref("browser.profiles.sync.allow-danger-merge", false); diff --git a/browser/extensions/moz.build b/browser/extensions/moz.build index 3c6e7eb886892..59545177123b9 100644 --- a/browser/extensions/moz.build +++ b/browser/extensions/moz.build @@ -11,4 +11,7 @@ DIRS += [ "report-site-issue", "pictureinpicture", "search-detection", + "warc-indicator", + "warc-recorder", + "peer-sidebar", ] diff --git a/browser/extensions/moz.configure b/browser/extensions/moz.configure new file mode 100644 index 0000000000000..9301e3ffb5571 --- /dev/null +++ b/browser/extensions/moz.configure @@ -0,0 +1,3 @@ +imply_option('warc-indicator', True) +imply_option('warc-recorder', True) +imply_option('peer-sidebar', True) diff --git a/browser/extensions/peer-sidebar/background/background.js b/browser/extensions/peer-sidebar/background/background.js new file mode 100644 index 0000000000000..fb468e2f2893e --- /dev/null +++ b/browser/extensions/peer-sidebar/background/background.js @@ -0,0 +1,54 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +let gPeers = []; +// const REFRESH_INTERVAL = 30000; // 30 seconds +const REFRESH_INTERVAL = 3000; // 3 seconds + +// Handle messages from the sidebar panel +browser.runtime.onMessage.addListener((message, sender) => { + switch (message.type) { + case "get-peers": + return Promise.resolve(gPeers); + default: + console.warn("Unknown message type:", message.type); + return Promise.reject(new Error("Unknown message type")); + } +}); + +async function fetchPeers() { + try { + const response = await fetch("http://localhost:8000/users", {}); + + gPeers = await response.json(); + console.log("gPeers ", { gPeers }); + + // Notify all sidebar instances of the update + const tabs = await browser.tabs.query({}); + for (const tab of tabs) { + try { + console.log('sending "peers-updated"'); + browser.runtime.sendMessage({ + type: "peers-updated", + peers: gPeers, + }); + } catch (e) { + // Ignore errors from tabs that can't receive the message + console.debug("Could not send peers-updated to tab", tab.id, e); + } + } + } catch (error) { + // Suppress CORS errors in console + if (!error.message.includes("CORS")) { + console.error("Error fetching peers:", error); + } + // Keep the old peer list on error + } +} + +// Initialize polling when the background script starts +fetchPeers(); // Initial fetch +setInterval(fetchPeers, REFRESH_INTERVAL); diff --git a/browser/extensions/peer-sidebar/icon.png b/browser/extensions/peer-sidebar/icon.png new file mode 100644 index 0000000000000..2eff084301724 --- /dev/null +++ b/browser/extensions/peer-sidebar/icon.png @@ -0,0 +1 @@ +[Binary file - a simple icon would need to be provided] diff --git a/browser/extensions/peer-sidebar/knowledge.md b/browser/extensions/peer-sidebar/knowledge.md new file mode 100644 index 0000000000000..927bd7c76d488 --- /dev/null +++ b/browser/extensions/peer-sidebar/knowledge.md @@ -0,0 +1,18 @@ +# Peer Sidebar Extension + +## Known Issues + +### CORS Configuration +The backend at localhost:8000 needs CORS headers to allow requests from the extension. +Backend should add: +``` +Access-Control-Allow-Origin: moz-extension://* +Access-Control-Allow-Methods: GET +``` + +Currently using `mode: 'no-cors'` as temporary workaround. + +## Development Notes +- Polling interval: 3s (dev), 30s (prod) +- Background script manages state and polling +- Panel script handles UI only diff --git a/browser/extensions/peer-sidebar/manifest.json b/browser/extensions/peer-sidebar/manifest.json new file mode 100644 index 0000000000000..4ca8e486857de --- /dev/null +++ b/browser/extensions/peer-sidebar/manifest.json @@ -0,0 +1,27 @@ +{ + "manifest_version": 2, + "name": "Peer Sidebar", + "version": "1.0", + "description": "Shows peer information in a sidebar", + "browser_specific_settings": { + "gecko": { + "id": "peer-sidebar@mozilla.org" + } + }, + "icons": { + "48": "icon.png", + "96": "icon.png" + }, + "permissions": [ + "http://localhost:8000/*", + "tabs" + ], + "background": { + "scripts": ["background/background.js"] + }, + "sidebar_action": { + "default_title": "Peer List", + "default_panel": "sidebar/panel.html", + "default_icon": "icon.png" + } +} diff --git a/browser/extensions/peer-sidebar/moz.build b/browser/extensions/peer-sidebar/moz.build new file mode 100644 index 0000000000000..e139ffa228c4a --- /dev/null +++ b/browser/extensions/peer-sidebar/moz.build @@ -0,0 +1,26 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DEFINES['MOZ_APP_VERSION'] = CONFIG['MOZ_APP_VERSION'] +DEFINES['MOZ_APP_MAXVERSION'] = CONFIG['MOZ_APP_MAXVERSION'] + +FINAL_TARGET_FILES.features['peer-sidebar@mozilla.org'] += [ + 'manifest.json', +] + +FINAL_TARGET_FILES.features['peer-sidebar@mozilla.org'].background += [ + 'background/background.js', +] + +FINAL_TARGET_FILES.features['peer-sidebar@mozilla.org'].sidebar += [ + 'sidebar/panel.css', + 'sidebar/panel.html', + 'sidebar/panel.js', +] + +FINAL_TARGET_FILES.features['peer-sidebar@mozilla.org'].icons += [ + 'icon.png', +] diff --git a/browser/extensions/peer-sidebar/sidebar/panel.css b/browser/extensions/peer-sidebar/sidebar/panel.css new file mode 100644 index 0000000000000..ed6d2f068c963 --- /dev/null +++ b/browser/extensions/peer-sidebar/sidebar/panel.css @@ -0,0 +1,30 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + padding: 16px; + background-color: #ffffff; +} + +.peer-item { + border: 1px solid #ddd; + margin-bottom: 8px; + padding: 12px; + border-radius: 4px; +} + +.peer-name { + font-weight: bold; + margin-bottom: 4px; +} + +.peer-ip { + color: #666; + font-family: monospace; + margin-bottom: 4px; +} + +.peer-key { + color: #444; + font-family: monospace; + font-size: 0.9em; + word-break: break-all; +} diff --git a/browser/extensions/peer-sidebar/sidebar/panel.html b/browser/extensions/peer-sidebar/sidebar/panel.html new file mode 100644 index 0000000000000..6d7fb7dc5564b --- /dev/null +++ b/browser/extensions/peer-sidebar/sidebar/panel.html @@ -0,0 +1,14 @@ + + + + + + + +
+

Peer List

+
+
+ + + diff --git a/browser/extensions/peer-sidebar/sidebar/panel.js b/browser/extensions/peer-sidebar/sidebar/panel.js new file mode 100644 index 0000000000000..1a0482f4140a1 --- /dev/null +++ b/browser/extensions/peer-sidebar/sidebar/panel.js @@ -0,0 +1,67 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +async function init() { + // Get initial state + try { + const peers = await browser.runtime.sendMessage({ type: "get-peers" }); + displayPeers(peers); + } catch (error) { + console.error("Error fetching initial peers:", error); + showError(error); + } + + // Listen for updates + browser.runtime.onMessage.addListener(message => { + if (message.type === "peers-updated") { + console.log('got "peers-updated"'); + displayPeers(message.peers); + } + }); +} + +function displayPeers(peers) { + const peerList = document.getElementById("peer-list"); + peerList.innerHTML = ""; + + if (!peers || peers.length === 0) { + peerList.innerHTML = '
No peers found
'; + return; + } + + console.log("here them peers ", { peers }); + + peers.forEach(peer => { + const peerElement = document.createElement("div"); + peerElement.className = "peer-item"; + + peerElement.innerHTML = ` +
${escapeHtml(peer.name)}
+
${escapeHtml(peer.ip_address)}
+
${escapeHtml(peer.public_key)}
+ `; + + console.log("peerEl ", { peerElement }); + peerList.appendChild(peerElement); + }); +} + +function showError(error) { + document.getElementById("peer-list").innerHTML = + `
Error loading peer data: ${error.message}
`; +} + +function escapeHtml(unsafe) { + return unsafe + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +// Initialize when the panel loads +init(); diff --git a/browser/extensions/warc-indicator/browser.ini b/browser/extensions/warc-indicator/browser.ini new file mode 100644 index 0000000000000..029b983986915 --- /dev/null +++ b/browser/extensions/warc-indicator/browser.ini @@ -0,0 +1,5 @@ +[DEFAULT] +support-files = + webextension/* + +[browser_warc_indicator.js] diff --git a/browser/extensions/warc-indicator/browser_warc_indicator.js b/browser/extensions/warc-indicator/browser_warc_indicator.js new file mode 100644 index 0000000000000..e51d5ecfb6834 --- /dev/null +++ b/browser/extensions/warc-indicator/browser_warc_indicator.js @@ -0,0 +1,5 @@ +"use strict"; + +add_task(async function test_warc_indicator_loads() { + ok(true, "WARC indicator loaded"); +}); diff --git a/browser/extensions/warc-indicator/build.mk b/browser/extensions/warc-indicator/build.mk new file mode 100644 index 0000000000000..e67ba727acd6e --- /dev/null +++ b/browser/extensions/warc-indicator/build.mk @@ -0,0 +1 @@ +XPI_NAME = warc-indicator@mozilla.org diff --git a/browser/extensions/warc-indicator/moz.build b/browser/extensions/warc-indicator/moz.build new file mode 100644 index 0000000000000..dbae450ab3dad --- /dev/null +++ b/browser/extensions/warc-indicator/moz.build @@ -0,0 +1,29 @@ +DEFINES['MOZ_APP_VERSION'] = CONFIG['MOZ_APP_VERSION'] +export('DEFINES') + +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'] += [ + 'webextension/manifest.json' +] + +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'].background += [ + 'webextension/background/background.js', + 'webextension/background/warcChecker.js' +] + +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'].icons += [ + 'webextension/icons/available-19.png', + 'webextension/icons/available-38.png', + 'webextension/icons/icon-48.png', + 'webextension/icons/icon-96.png', + 'webextension/icons/inactive-19.png', + 'webextension/icons/inactive-38.png', +] + +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'].sidebar += [ + 'webextension/sidebar/panel.css', + 'webextension/sidebar/panel.html', + 'webextension/sidebar/panel.js' +] + +with Files('**'): + BUG_COMPONENT = ('Firefox', 'General') diff --git a/browser/extensions/warc-indicator/moz.configure b/browser/extensions/warc-indicator/moz.configure new file mode 100644 index 0000000000000..79a051af0ed7c --- /dev/null +++ b/browser/extensions/warc-indicator/moz.configure @@ -0,0 +1,9 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: + +@depends('warc-indicator') +def warc_indicator(warc_indicator): + return bool(warc_indicator) + +set_config('MOZ_WARC_INDICATOR', warc_indicator) +set_define('MOZ_WARC_INDICATOR', warc_indicator) diff --git a/browser/extensions/warc-indicator/webextension/background.js b/browser/extensions/warc-indicator/webextension/background.js new file mode 100644 index 0000000000000..1b242de3b6a66 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/background.js @@ -0,0 +1,80 @@ +console.log("background.js loaded"); +import { WarcChecker } from "./warcChecker.js"; + +const warcChecker = new WarcChecker(); + +// Track which tabs are in WARC mode +const warcModeTabs = new Set(); + +async function updateStatus(tabId, url) { + console.log("background: updateStatus called for", url); + + if (!url || url.startsWith(warcChecker.pywbEndpoint)) { + return; + } + + const hasWarc = await warcChecker.checkAvailability(url); + console.log("background: WARC availability:", hasWarc); + + // Send status to sidebar if it's open + browser.runtime.sendMessage({ + type: "warc-status-update", + hasWarc, + url, + isWarcMode: warcModeTabs.has(tabId) + }).catch(err => { + // This error is expected if sidebar isn't open + console.log("background: couldn't send message (sidebar probably not open)"); + }); +} + +// Listen for tab updates +browser.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.status === "complete") { + console.log("background: tab updated, checking", tab.url); + updateStatus(tabId, tab.url); + } +}); + +// Handle toggle between live and WARC mode +async function toggleWarcMode(tab) { + if (tab.url.startsWith(warcChecker.pywbEndpoint)) { + // We're in WARC mode, get original URL + const originalUrl = tab.url.split("/mp_/")[1]; + await browser.tabs.update(tab.id, { url: originalUrl }); + warcModeTabs.delete(tab.id); + } else { + // We're in live mode, check WARC availability + const hasWarc = await warcChecker.checkAvailability(tab.url); + if (hasWarc) { + console.log("background: WARC available, switching to proxy URL"); + const proxyUrl = warcChecker.getProxyUrl(tab.url); + console.log("background: Proxy URL:", proxyUrl); + await browser.tabs.update(tab.id, { url: proxyUrl }); + warcModeTabs.add(tab.id); + } else { + console.log("background: No WARC available"); + } + } +} + +// Handle keyboard shortcut +browser.commands.onCommand.addListener(async command => { + if (command === "toggle-warc") { + const tabs = await browser.tabs.query({ + active: true, + currentWindow: true, + }); + if (tabs[0]) { + await toggleWarcMode(tabs[0]); + } + } +}); + +// Clear cache periodically +setInterval( + () => { + warcChecker.clearCache(); + }, + 30 * 60 * 1000 +); // Every 30 minutes diff --git a/browser/extensions/warc-indicator/webextension/background/background.js b/browser/extensions/warc-indicator/webextension/background/background.js new file mode 100644 index 0000000000000..33c1735847c30 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/background/background.js @@ -0,0 +1,87 @@ +console.log("background.js loaded"); +import { WarcChecker } from "./warcChecker.js"; + +const warcChecker = new WarcChecker(); + +// Track which tabs are in WARC mode +const warcModeTabs = new Set(); + +async function updateStatus(tabId, url) { + console.log("background: updateStatus called for", url); + + if (!url || url.startsWith(warcChecker.pywbEndpoint)) { + return; + } + + const hasWarc = await warcChecker.checkAvailability(url); + console.log("background: WARC availability:", hasWarc); + + // Send status to sidebar if it's open + browser.runtime + .sendMessage({ + type: "warc-status-update", + hasWarc, + url, + isWarcMode: warcModeTabs.has(tabId), + }) + .catch(err => { + // This error is expected if sidebar isn't open + console.log( + "background: couldn't send message (sidebar probably not open)" + ); + }); +} + +// Listen for tab updates +browser.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.status === "complete") { + console.log("background: tab updated, checking", tab.url); + updateStatus(tabId, tab.url); + } +}); + +// Handle toggle between live and WARC mode +async function toggleWarcMode(tab) { + if (tab.url.startsWith(warcChecker.pywbEndpoint)) { + // We're in WARC mode, get original URL + // Extract original URL from pywb replay URL + const match = tab.url.match(/^http:\/\/localhost:8080\/local\/(?:\d+|mp_)\/(.+)$/); + const originalUrl = match ? match[1] : null; + console.log("background: extracted original URL:", originalUrl); + await browser.tabs.update(tab.id, { url: originalUrl }); + warcModeTabs.delete(tab.id); + } else { + // We're in live mode, check WARC availability + const hasWarc = await warcChecker.checkAvailability(tab.url); + if (hasWarc) { + console.log("background: WARC available, switching to proxy URL"); + const proxyUrl = warcChecker.getProxyUrl(tab.url); + console.log("background: Proxy URL:", proxyUrl); + await browser.tabs.update(tab.id, { url: proxyUrl }); + warcModeTabs.add(tab.id); + } else { + console.log("background: No WARC available"); + } + } +} + +// Handle keyboard shortcut +browser.commands.onCommand.addListener(async command => { + if (command === "toggle-warc") { + const tabs = await browser.tabs.query({ + active: true, + currentWindow: true, + }); + if (tabs[0]) { + await toggleWarcMode(tabs[0]); + } + } +}); + +// Clear cache periodically +setInterval( + () => { + warcChecker.clearCache(); + }, + 30 * 60 * 1000 +); // Every 30 minutes diff --git a/browser/extensions/warc-indicator/webextension/background/warcChecker.js b/browser/extensions/warc-indicator/webextension/background/warcChecker.js new file mode 100644 index 0000000000000..87907486dc097 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/background/warcChecker.js @@ -0,0 +1,64 @@ +console.log("warcChecker.js loaded"); +// TODO: collection name ("local") should be a var. +export class WarcChecker { + constructor() { + this.cache = new Map(); + this.CACHE_DURATION = 5 * 60 * 1000; // 5 minutes + this.pywbEndpoint = "http://localhost:8080"; + } + + async checkAvailability(url) { + console.log("checking ", url); + const cached = this.getCached(url); + if (cached !== null) { + return cached; + } + + try { + const cdxEndpoint = `${this.pywbEndpoint}/local/cdx?url=${encodeURIComponent(url)}`; + const response = await fetch(cdxEndpoint); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const cdxData = await response.text(); + const available = cdxData.trim().length > 0; + + this.setCached(url, available); + return available; + } catch (error) { + console.error("Error checking WARC availability:", error); + return false; + } + } + + getCached(url) { + console.log("getCached ", url); + const now = Date.now(); + const cached = this.cache.get(url); + + if (cached && now - cached.timestamp < this.CACHE_DURATION) { + return cached.available; + } + + return null; + } + + setCached(url, available) { + console.log("setCached ", url); + this.cache.set(url, { + available, + timestamp: Date.now(), + }); + } + + clearCache() { + console.log("clearCache "); + this.cache.clear(); + } + + getProxyUrl(originalUrl) { + console.log("getProxyUrl ", originalUrl); + return `${this.pywbEndpoint}/local/mp_/${originalUrl}`; + } +} diff --git a/browser/extensions/warc-indicator/webextension/icons/available-19.png b/browser/extensions/warc-indicator/webextension/icons/available-19.png new file mode 100644 index 0000000000000..2a30b28829da5 Binary files /dev/null and b/browser/extensions/warc-indicator/webextension/icons/available-19.png differ diff --git a/browser/extensions/warc-indicator/webextension/icons/available-38.png b/browser/extensions/warc-indicator/webextension/icons/available-38.png new file mode 100644 index 0000000000000..8af9b2ece4b3a Binary files /dev/null and b/browser/extensions/warc-indicator/webextension/icons/available-38.png differ diff --git a/browser/extensions/warc-indicator/webextension/icons/icon-48.png b/browser/extensions/warc-indicator/webextension/icons/icon-48.png new file mode 100644 index 0000000000000..f51e55bc21fdf Binary files /dev/null and b/browser/extensions/warc-indicator/webextension/icons/icon-48.png differ diff --git a/browser/extensions/warc-indicator/webextension/icons/icon-96.png b/browser/extensions/warc-indicator/webextension/icons/icon-96.png new file mode 100644 index 0000000000000..acf54c152206a Binary files /dev/null and b/browser/extensions/warc-indicator/webextension/icons/icon-96.png differ diff --git a/browser/extensions/warc-indicator/webextension/icons/inactive-19.png b/browser/extensions/warc-indicator/webextension/icons/inactive-19.png new file mode 100644 index 0000000000000..b0ff16e36cad0 Binary files /dev/null and b/browser/extensions/warc-indicator/webextension/icons/inactive-19.png differ diff --git a/browser/extensions/warc-indicator/webextension/icons/inactive-38.png b/browser/extensions/warc-indicator/webextension/icons/inactive-38.png new file mode 100644 index 0000000000000..67d84264c5866 Binary files /dev/null and b/browser/extensions/warc-indicator/webextension/icons/inactive-38.png differ diff --git a/browser/extensions/warc-indicator/webextension/manifest.json b/browser/extensions/warc-indicator/webextension/manifest.json new file mode 100644 index 0000000000000..4e8c5f341b684 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/manifest.json @@ -0,0 +1,57 @@ +{ + "manifest_version": 2, + "name": "WARC Availability Indicator", + "version": "1.0", + "description": "Shows when a WARC archive is available for the current page", + "applications": { + "gecko": { + "id": "warc-indicator@mozilla.org" + } + }, + "browser_specific_settings": { + "gecko": { + "id": "warc-indicator@mozilla.org" + } + }, + "permissions": [ + "activeTab", + "tabs", + "webRequest", + "storage", + "http://localhost:8080/*", + "" + ], + "icons": { + "48": "icons/icon-48.png", + "96": "icons/icon-96.png" + }, + "sidebar_action": { + "default_title": "WARC Archive", + "default_panel": "sidebar/panel.html", + "default_icon": { + "19": "icons/inactive-19.png", + "38": "icons/inactive-38.png" + } + }, + "background": { + "type": "module", + "scripts": ["background/background.js", "background/warcChecker.js"] + }, + "permissions": [ + "activeTab", + "tabs", + "webRequest", + "storage", + "http://localhost:8080/*", + "" + ], + + "commands": { + "toggle-warc": { + "suggested_key": { + "default": "Ctrl+Shift+W" + }, + "description": "Toggle between live and WARC version" + } + } +} diff --git a/browser/extensions/warc-indicator/webextension/moz.build b/browser/extensions/warc-indicator/webextension/moz.build new file mode 100644 index 0000000000000..5f3098cf4ed77 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/moz.build @@ -0,0 +1,19 @@ +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'].background += [ + 'background/background.js', + 'background/warcChecker.js', +] + +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'].sidebar += [ + 'sidebar/panel.css', + 'sidebar/panel.html', + 'sidebar/panel.js', +] + +FINAL_TARGET_FILES.features['warc-indicator@mozilla.org'].icons += [ + 'icons/available-19.png', + 'icons/available-38.png', + 'icons/icon-48.png', + 'icons/icon-96.png', + 'icons/inactive-19.png', + 'icons/inactive-38.png', +] diff --git a/browser/extensions/warc-indicator/webextension/popup/info.css b/browser/extensions/warc-indicator/webextension/popup/info.css new file mode 100644 index 0000000000000..e85b30f2c8ef3 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/popup/info.css @@ -0,0 +1,24 @@ +body { + width: 300px; + padding: 10px; + font-family: + system-ui, + -apple-system, + sans-serif; +} + +#container { + display: flex; + flex-direction: column; + gap: 10px; +} + +#status { + font-weight: bold; + margin-bottom: 5px; +} + +#captures { + font-size: 0.9em; + color: #666; +} diff --git a/browser/extensions/warc-indicator/webextension/popup/info.html b/browser/extensions/warc-indicator/webextension/popup/info.html new file mode 100644 index 0000000000000..71fac9f5fac9c --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/popup/info.html @@ -0,0 +1,14 @@ + + + + + + + +
+
+
+
+ + + diff --git a/browser/extensions/warc-indicator/webextension/popup/info.js b/browser/extensions/warc-indicator/webextension/popup/info.js new file mode 100644 index 0000000000000..dc988a55b386a --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/popup/info.js @@ -0,0 +1,31 @@ +document.addEventListener("DOMContentLoaded", async () => { + const tabs = await browser.tabs.query({ active: true, currentWindow: true }); + const currentTab = tabs[0]; + + const statusDiv = document.getElementById("status"); + const capturesDiv = document.getElementById("captures"); + + if (currentTab.url.startsWith("http://localhost:8080")) { + statusDiv.textContent = "Currently viewing archived version"; + const originalUrl = currentTab.url.split("/mp_/")[1]; + capturesDiv.textContent = `Original URL: ${originalUrl}`; + } else { + try { + const cdxEndpoint = `http://localhost:8080/local/cdx?url=${encodeURIComponent(currentTab.url)}`; + const response = await fetch(cdxEndpoint); + const cdxData = await response.text(); + + if (cdxData.trim().length > 0) { + statusDiv.textContent = "WARC archive available"; + capturesDiv.textContent = + "Click the toolbar icon to view archived version"; + } else { + statusDiv.textContent = "No WARC archive available"; + capturesDiv.textContent = "This page has not been archived"; + } + } catch (error) { + statusDiv.textContent = "Error checking WARC availability"; + capturesDiv.textContent = "Unable to connect to archive server"; + } + } +}); diff --git a/browser/extensions/warc-indicator/webextension/sidebar/panel.css b/browser/extensions/warc-indicator/webextension/sidebar/panel.css new file mode 100644 index 0000000000000..048676770d497 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/sidebar/panel.css @@ -0,0 +1,41 @@ +body { + padding: 16px; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; +} + +#warc-status { + display: flex; + align-items: center; + margin-bottom: 16px; +} + +#status-icon { + width: 16px; + height: 16px; + border-radius: 50%; + margin-right: 8px; + background-color: #ccc; +} + +#status-icon.available { + background-color: #2ecc71; +} + +#status-icon.unavailable { + background-color: #95a5a6; +} + +#view-warc { + width: 100%; + padding: 8px; + border: none; + border-radius: 4px; + background-color: #2ecc71; + color: white; + cursor: pointer; +} + +#view-warc:disabled { + background-color: #95a5a6; + cursor: not-allowed; +} diff --git a/browser/extensions/warc-indicator/webextension/sidebar/panel.html b/browser/extensions/warc-indicator/webextension/sidebar/panel.html new file mode 100644 index 0000000000000..b2472e4e64169 --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/sidebar/panel.html @@ -0,0 +1,17 @@ + + + + + + + +
+
+
Checking current page...
+
+
+ +
+ + + diff --git a/browser/extensions/warc-indicator/webextension/sidebar/panel.js b/browser/extensions/warc-indicator/webextension/sidebar/panel.js new file mode 100644 index 0000000000000..5fa365273bf9c --- /dev/null +++ b/browser/extensions/warc-indicator/webextension/sidebar/panel.js @@ -0,0 +1,105 @@ +import { WarcChecker } from "/background/warcChecker.js"; + +console.log("warc playback sidebar: panel.js loaded"); + +const warcChecker = new WarcChecker(); +const statusIcon = document.getElementById("status-icon"); +const statusText = document.getElementById("status-text"); +const viewWarcButton = document.getElementById("view-warc"); + +let currentUrl = null; +let currentTabId = null; + +async function updateStatus(url) { + console.log("sidebar: updateStatus called for", url); + + if (!url || url === "about:blank") { + statusIcon.className = ""; + statusText.textContent = "No page loaded"; + viewWarcButton.disabled = true; + return; + } + + currentUrl = url; + statusText.textContent = "Checking WARC availability..."; + viewWarcButton.disabled = true; + + try { + const hasWarc = await warcChecker.checkAvailability(url); + console.log("sidebar: WARC availability:", hasWarc); + + statusIcon.className = hasWarc ? "available" : "unavailable"; + + if (url.startsWith(warcChecker.pywbEndpoint)) { + statusText.textContent = "Viewing archived version"; + viewWarcButton.textContent = "View Live Version"; + viewWarcButton.disabled = false; + } else { + statusText.textContent = hasWarc + ? "WARC archive available!" + : "No WARC archive available"; + viewWarcButton.textContent = "View Archived Version"; + viewWarcButton.disabled = !hasWarc; + } + } catch (err) { + console.error("sidebar: Error checking WARC status:", err); + statusIcon.className = "unavailable"; + statusText.textContent = "Error checking WARC status"; + viewWarcButton.disabled = true; + } +} + +// Listen for tab changes +browser.tabs.onActivated.addListener(async activeInfo => { + console.log("sidebar: tab activated"); + currentTabId = activeInfo.tabId; + const tab = await browser.tabs.get(activeInfo.tabId); + updateStatus(tab.url); +}); + +browser.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.url && tabId === currentTabId) { + console.log("sidebar: tab updated"); + updateStatus(changeInfo.url); + } +}); + +// Handle button click to toggle WARC mode +viewWarcButton.addEventListener("click", async () => { + if (!currentUrl || !currentTabId) return; + + const tab = await browser.tabs.get(currentTabId); + if (tab.url.startsWith(warcChecker.pywbEndpoint)) { + // Switch to live version + console.log("sidebar: switching to live version from", tab.url); + // Extract original URL from pywb replay URL + const match = tab.url.match( + /^http:\/\/localhost:8080\/local\/(?:\d+|mp_)\/(.+)$/ + ); + const originalUrl = match ? match[1] : null; + console.log("sidebar: extracted original URL:", originalUrl); + console.log("sidebar: original URL is", originalUrl); + await browser.tabs.update(currentTabId, { url: originalUrl }); + } else { + // Switch to archived version + const proxyUrl = warcChecker.getProxyUrl(currentUrl); + await browser.tabs.update(currentTabId, { url: proxyUrl }); + } +}); + +// Initial status check +browser.tabs.query({ active: true, currentWindow: true }).then(tabs => { + console.log("sidebar: initial status check"); + if (tabs[0]) { + currentTabId = tabs[0].id; + updateStatus(tabs[0].url); + } +}); + +// Listen for status updates from background script +browser.runtime.onMessage.addListener(message => { + console.log("sidebar: received message", message); + if (message.type === "warc-status-update") { + updateStatus(message.url); + } +}); diff --git a/browser/extensions/warc-recorder/browser.ini b/browser/extensions/warc-recorder/browser.ini new file mode 100644 index 0000000000000..1ee2a341ceb1f --- /dev/null +++ b/browser/extensions/warc-recorder/browser.ini @@ -0,0 +1,5 @@ +[DEFAULT] +support-files = + webextension/* + +[browser_warc_recorder.js] diff --git a/browser/extensions/warc-recorder/browser_warc_recorder.js b/browser/extensions/warc-recorder/browser_warc_recorder.js new file mode 100644 index 0000000000000..92070a729f9e1 --- /dev/null +++ b/browser/extensions/warc-recorder/browser_warc_recorder.js @@ -0,0 +1,5 @@ +"use strict"; + +add_task(async function test_warc_recorder_loads() { + ok(true, "WARC recorder loaded"); +}); diff --git a/browser/extensions/warc-recorder/build.mk b/browser/extensions/warc-recorder/build.mk new file mode 100644 index 0000000000000..6598ca0962b35 --- /dev/null +++ b/browser/extensions/warc-recorder/build.mk @@ -0,0 +1 @@ +XPI_NAME = warc-recorder@mozilla.org diff --git a/browser/extensions/warc-recorder/moz.build b/browser/extensions/warc-recorder/moz.build new file mode 100644 index 0000000000000..af5d02d526fb4 --- /dev/null +++ b/browser/extensions/warc-recorder/moz.build @@ -0,0 +1,23 @@ +DEFINES['MOZ_APP_VERSION'] = CONFIG['MOZ_APP_VERSION'] +export('DEFINES') + +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'] += [ + 'webextension/manifest.json' +] + +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'].background += [ + 'webextension/background/background.js', +] + +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'].icons += [ + 'webextension/icons/pen.svg', +] + +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'].sidebar += [ + 'webextension/sidebar/panel.css', + 'webextension/sidebar/panel.html', + 'webextension/sidebar/panel.js' +] + +with Files('**'): + BUG_COMPONENT = ('Firefox', 'General') diff --git a/browser/extensions/warc-recorder/moz.configure b/browser/extensions/warc-recorder/moz.configure new file mode 100644 index 0000000000000..5c16d14cb3469 --- /dev/null +++ b/browser/extensions/warc-recorder/moz.configure @@ -0,0 +1,9 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: + +@depends('warc-recorder') +def warc_recorder(warc_recorder): + return bool(warc_recorder) + +set_config('MOZ_WARC_RECORDER', warc_recorder) +set_define('MOZ_WARC_RECORDER', warc_recorder) diff --git a/browser/extensions/warc-recorder/webextension/README.md b/browser/extensions/warc-recorder/webextension/README.md new file mode 100755 index 0000000000000..5d121f1696ed7 --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/README.md @@ -0,0 +1,3 @@ +# WARC Recorder + +Firefox webextension that records all your browsing data and sends WARC records to http://localhost:12345 diff --git a/browser/extensions/warc-recorder/webextension/background/background.js b/browser/extensions/warc-recorder/webextension/background/background.js new file mode 100755 index 0000000000000..37bebf7c6f8bf --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/background/background.js @@ -0,0 +1,317 @@ +const requestStore = new Map(); +const redirectChains = new Map(); + +function generateWarcRecord(request, response) { + const warcVersion = "WARC/1.1"; + const recordId = crypto.randomUUID(); + const date = new Date().toISOString(); + + // Generate request record + let requestRecord = `${warcVersion}\r\n`; + requestRecord += `WARC-Type: request\r\n`; + requestRecord += `WARC-Date: ${date}\r\n`; + requestRecord += `WARC-Record-ID: \r\n`; + requestRecord += `Content-Type: application/http;msgtype=request\r\n`; + requestRecord += `WARC-Target-URI: ${request.url}\r\n`; + + // Add WARC-Concurrent-To headers if this was part of a redirect chain + if (request.previousRequestIds) { + for (const prevId of request.previousRequestIds) { + requestRecord += `WARC-Concurrent-To: \r\n`; + } + } + + // Create request headers block + let requestHeaders = `${request.method} ${request.url} HTTP/1.1\r\n`; + for (const [name, value] of Object.entries(request.headers)) { + requestHeaders += `${name}: ${value}\r\n`; + } + requestHeaders += "\r\n"; + + // Convert headers to Uint8Array + const requestHeadersBytes = new TextEncoder().encode(requestHeaders); + + // Combine headers and body + const requestContent = new Uint8Array([ + ...requestHeadersBytes, + ...(request.body || new Uint8Array(0)), + ]); + + requestRecord += `Content-Length: ${requestContent.byteLength}\r\n\r\n`; + + // Generate response record + let responseRecord = `${warcVersion}\r\n`; + responseRecord += `WARC-Type: response\r\n`; + responseRecord += `WARC-Date: ${date}\r\n`; + responseRecord += `WARC-Record-ID: \r\n`; + responseRecord += `WARC-Concurrent-To: \r\n`; + responseRecord += `Content-Type: application/http;msgtype=response\r\n`; + responseRecord += `WARC-Target-URI: ${request.url}\r\n`; + if (response.ip) { + responseRecord += `WARC-IP-Address: ${response.ip}\r\n`; + } + + // Parse HTTP version from status line + const httpVersion = response.statusLine + ? response.statusLine.split(" ")[0].split("/")[1] + : "1.1"; + + // Create response headers block + let responseHeaders = `HTTP/${httpVersion} ${response.statusCode} ${response.statusLine ? response.statusLine.split(" ").slice(2).join(" ") : ""}\r\n`; + for (const [name, value] of Object.entries(response.headers)) { + responseHeaders += `${name}: ${value}\r\n`; + } + responseHeaders += "\r\n"; + + // Convert headers to Uint8Array + const responseHeadersBytes = new TextEncoder().encode(responseHeaders); + + // Combine headers and body + const responseContent = new Uint8Array([ + ...responseHeadersBytes, + ...(response.body || new Uint8Array(0)), + ]); + + responseRecord += `Content-Length: ${responseContent.byteLength}\r\n\r\n`; + + // Combine everything into final WARC record + const recordParts = [ + new TextEncoder().encode(requestRecord), + requestContent, + new TextEncoder().encode("\r\n\r\n"), + new TextEncoder().encode(responseRecord), + responseContent, + new TextEncoder().encode("\r\n\r\n"), + ]; + + return new Blob(recordParts); +} + +async function uploadWarcRecord(warcRecord) { + try { + const response = await fetch("http://localhost:12345", { + method: "POST", + headers: { + "Content-Type": "application/warc", + }, + body: warcRecord, + }); + + if (!response.ok) { + console.error("Failed to upload WARC record:", response.statusText); + } + } catch (error) { + console.error("Error uploading WARC record:", error); + } +} + +browser.webRequest.onBeforeSendHeaders.addListener( + (details) => { + const request = { + id: details.requestId, + method: details.method, + url: details.url, + headers: {}, + body: null, + }; + + details.requestHeaders.forEach((header) => { + request.headers[header.name] = header.value; + }); + + requestStore.set(details.requestId, { request }); + }, + { urls: [""] }, + ["requestHeaders"], +); + +browser.webRequest.onBeforeRequest.addListener( + (details) => { + // Create new request entry + const request = { + id: details.requestId, + method: details.method, + url: details.url, + headers: {}, + body: null, + }; + + // Handle request body if present + if (details.requestBody) { + if (details.requestBody.raw) { + const totalLength = details.requestBody.raw.reduce( + (sum, chunk) => sum + chunk.bytes.byteLength, + 0, + ); + const combined = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of details.requestBody.raw) { + combined.set(new Uint8Array(chunk.bytes), offset); + offset += chunk.bytes.byteLength; + } + request.body = combined; + } else if (details.requestBody.formData) { + // Convert form data to URLSearchParams format + const formData = new URLSearchParams(); + for (const [name, values] of Object.entries( + details.requestBody.formData, + )) { + for (const value of values) { + formData.append(name, value); + } + } + request.body = new TextEncoder().encode(formData.toString()); + } + } + + // Check if this request was the target of a redirect + if (redirectChains.has(details.url)) { + request.previousRequestIds = redirectChains.get(details.url); + redirectChains.delete(details.url); + } + + requestStore.set(details.requestId, { request }); + }, + { urls: [""] }, + ["requestBody"], +); + +browser.webRequest.onResponseStarted.addListener( + (details) => { + if (requestStore.has(details.requestId)) { + const stored = requestStore.get(details.requestId); + stored.ip = details.ip; // Firefox provides the IP address here + requestStore.set(details.requestId, stored); + } + }, + { urls: [""] }, +); + +browser.webRequest.onHeadersReceived.addListener( + (details) => { + if (requestStore.has(details.requestId)) { + const filter = browser.webRequest.filterResponseData(details.requestId); + const chunks = []; + + filter.ondata = (event) => { + chunks.push(new Uint8Array(event.data)); + filter.write(event.data); + }; + + filter.onstop = () => { + const stored = requestStore.get(details.requestId); + if (stored) { + // Combine all chunks into one Uint8Array + const totalLength = chunks.reduce( + (acc, chunk) => acc + chunk.length, + 0, + ); + stored.response.body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + stored.response.body.set(chunk, offset); + offset += chunk.length; + } + + const warcRecord = generateWarcRecord( + stored.request, + stored.response, + ); + uploadWarcRecord(warcRecord); + + requestStore.delete(details.requestId); + } + filter.disconnect(); + }; + + const stored = requestStore.get(details.requestId); + const response = { + statusCode: details.statusCode, + statusLine: details.statusLine, + headers: {}, + body: null, + }; + + details.responseHeaders.forEach((header) => { + response.headers[header.name] = header.value; + }); + + stored.response = response; + requestStore.set(details.requestId, stored); + } + }, + { urls: [""] }, + ["responseHeaders", "blocking"], +); + +browser.webRequest.onBeforeRedirect.addListener( + (details) => { + // Store the relationship between the old request and the new one + if (!redirectChains.has(details.redirectUrl)) { + redirectChains.set(details.redirectUrl, []); + } + redirectChains.get(details.redirectUrl).push(details.requestId); + + // Mark the current request as part of a redirect chain + if (requestStore.has(details.requestId)) { + const stored = requestStore.get(details.requestId); + stored.isRedirect = true; + stored.redirectUrl = details.redirectUrl; + requestStore.set(details.requestId, stored); + } + }, + { urls: [""] }, +); + +/* +browser.webRequest.filterResponseData.addListener( + (details) => { + const chunks = []; + const filter = browser.webRequest.filterResponseData(details.requestId); + + filter.ondata = (event) => { + chunks.push(new Uint8Array(event.data)); + filter.write(event.data); + }; + + filter.onstop = () => { + if (requestStore.has(details.requestId)) { + const stored = requestStore.get(details.requestId); + + // Combine all chunks into one Uint8Array + const totalLength = chunks.reduce( + (acc, chunk) => acc + chunk.length, + 0, + ); + stored.response.body = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + stored.response.body.set(chunk, offset); + offset += chunk.length; + } + + const warcRecord = generateWarcRecord(stored.request, stored.response); + uploadWarcRecord(warcRecord); + + requestStore.delete(details.requestId); + } + filter.disconnect(); + }; + }, + { urls: [""] }, +); +*/ + +browser.webRequest.onCompleted.addListener( + (details) => { + requestStore.delete(details.requestId); + }, + { urls: [""] }, +); + +browser.webRequest.onErrorOccurred.addListener( + (details) => { + requestStore.delete(details.requestId); + }, + { urls: [""] }, +); diff --git a/browser/extensions/warc-recorder/webextension/icons/pen.svg b/browser/extensions/warc-recorder/webextension/icons/pen.svg new file mode 100755 index 0000000000000..65a3f42f7f251 --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/icons/pen.svg @@ -0,0 +1 @@ + diff --git a/browser/extensions/warc-recorder/webextension/manifest.json b/browser/extensions/warc-recorder/webextension/manifest.json new file mode 100755 index 0000000000000..fa3e0f1412ea2 --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/manifest.json @@ -0,0 +1,46 @@ +{ + "manifest_version": 2, + "name": "WARC Recorder", + "version": "1.0", + "description": "Records web traffic in WARC format", + "applications": { + "gecko": { + "id": "warc-recorder@mozilla.org" + } + }, + + "permissions": [ + "webRequest", + "webRequestBlocking", + "http://localhost:12345/", + "activeTab", + "tabs", + "storage", + "http://localhost:8080/*", + "" + ], + + "background": { + "type": "module", + "scripts": ["background/background.js"] + }, + + "icons": { + "48": "icons/pen.svg" + }, + + "browser_specific_settings": { + "gecko": { + "id": "warc-recorder@mozilla.org" + } + }, + + "sidebar_action": { + "default_title": "WARC Recorder", + "default_panel": "sidebar/panel.html", + "default_icon": { + "19": "icons/pen.svg", + "38": "icons/pen.svg" + } + } +} diff --git a/browser/extensions/warc-recorder/webextension/moz.build b/browser/extensions/warc-recorder/webextension/moz.build new file mode 100644 index 0000000000000..bda4f75465ea7 --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/moz.build @@ -0,0 +1,13 @@ +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'].background += [ + 'background/background.js', +] + +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'].sidebar += [ + 'sidebar/panel.css', + 'sidebar/panel.html', + 'sidebar/panel.js', +] + +FINAL_TARGET_FILES.features['warc-recorder@mozilla.org'].icons += [ + 'icons/pen.svg', +] diff --git a/browser/extensions/warc-recorder/webextension/sidebar/panel.css b/browser/extensions/warc-recorder/webextension/sidebar/panel.css new file mode 100644 index 0000000000000..f9a7dd73c00ec --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/sidebar/panel.css @@ -0,0 +1,4 @@ +body { + padding: 16px; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; +} diff --git a/browser/extensions/warc-recorder/webextension/sidebar/panel.html b/browser/extensions/warc-recorder/webextension/sidebar/panel.html new file mode 100644 index 0000000000000..c88e980d58669 --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/sidebar/panel.html @@ -0,0 +1,11 @@ + + + + + + + + This is the warc recorder panel, for testing. + + + diff --git a/browser/extensions/warc-recorder/webextension/sidebar/panel.js b/browser/extensions/warc-recorder/webextension/sidebar/panel.js new file mode 100644 index 0000000000000..8bed10461a3be --- /dev/null +++ b/browser/extensions/warc-recorder/webextension/sidebar/panel.js @@ -0,0 +1 @@ +console.log("warc record sidebar: panel.js loaded");