From f6bf59a4ed44ec7323a76b11317f715c929a69b4 Mon Sep 17 00:00:00 2001 From: hackademix Date: Sep 01 2018 06:31:51 +0000 Subject: [PATCH 1/2] Moved external licenses check into response pre-processing, in order to avoid filterResponse() as much as possible (fixing issue #11) --- diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js index 7c264a4..e3d6c45 100644 --- a/bg/ResponseProcessor.js +++ b/bg/ResponseProcessor.js @@ -33,8 +33,8 @@ class ResponseProcessor { static install(handler, types = ["main_frame", "sub_frame", "script"]) { if (listeners.has(handler)) return false; - let listener = - request => new ResponseTextFilter(request).process(handler); + let listener = + async request => await new ResponseTextFilter(request).process(handler); listeners.set(handler, listener); webRequestEvent.addListener( listener, @@ -70,12 +70,12 @@ class ResponseTextFilter { (type === "script" || /\bhtml\b/i.test(md.contentType)); } - process(handler) { + async process(handler) { if (!this.canProcess) return ResponseProcessor.ACCEPT; let {metaData, request} = this; let response = {request, metaData}; // we keep it around allowing callbacks to store state if (typeof handler.pre === "function") { - let res = handler.pre(response); + let res = await handler.pre(response); if (res) return res; if (handler.post) handler = handler.post; if (typeof handler !== "function") ResponseProcessor.ACCEPT; @@ -101,7 +101,7 @@ class ResponseTextFilter { } catch(e) { console.error(e); } - if (metaData.forcedUTF8 || + if (metaData.forcedUTF8 && request.type !== "script" || editedText !== null && response.text !== editedText) { // if we changed the charset, the text or both, let's re-encode filter.write(new TextEncoder().encode(editedText)); diff --git a/content/externalLicenseChecker.js b/content/externalLicenseChecker.js index 89f871f..f150cea 100644 --- a/content/externalLicenseChecker.js +++ b/content/externalLicenseChecker.js @@ -79,9 +79,13 @@ } browser.runtime.onMessage.addListener(async m => { - if (m.action in handlers) { + if (m.action in handlers) try { debug("Received message", m); - return await handlers[m.action](m); + let result = await handlers[m.action](m); + console.debug("Returning", result); + return result; + } catch (e) { + console.error(e); } }); } diff --git a/main_background.js b/main_background.js index 0737926..a521d22 100644 --- a/main_background.js +++ b/main_background.js @@ -853,9 +853,9 @@ var ResponseHandler = { /** * Enforce white/black lists for url/site early (hashes will be handled later) */ - pre(response) { + async pre(response) { let {request} = response; - let {url, type, tabId} = request; + let {url, type, tabId, frameId} = request; url = ListStore.urlItem(url); let site = ListStore.siteItem(url); @@ -878,15 +878,31 @@ var ResponseHandler = { }); } else { let whitelistedSite = whitelist.contains(site); - if ((response.whitelisted = (whitelistedSite || whitelist.contains(url))) - && type === "script") { - // accept the script and stop processing - addReportEntry(tabId, url, {url: topUrl, - "whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]}); - return ResponseProcessor.ACCEPT; + let whitelisted = response.whitelisted = whitelistedSite || whitelist.contains(url); + if (type === "script") { + if (whitelisted) { + // accept the script and stop processing + addReportEntry(tabId, url, {url: topUrl, + "whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]}); + return ResponseProcessor.ACCEPT; + } else { + let scriptInfo = await ExternalLicenses.check({url, tabId, frameId}); + if (scriptInfo) { + let verdict, ret; + let msg = scriptInfo.toString(); + if (scriptInfo.allFree) { + verdict = "accepted"; + ret = ResponseProcessor.ACCEPT; + } else { + verdict = "blocked"; + ret = ResponseProcessor.REJECT; + } + addReportEntry(tabId, url, {url, [verdict]: [url, msg]}); + return ret; + } + } } } - // it's a page (it's too early to report) or an unknown script: // let's keep processing return ResponseProcessor.CONTINUE; @@ -909,21 +925,6 @@ async function handle_script(response, whitelisted){ let {text, request} = response; let {url, tabId, frameId} = request; url = ListStore.urlItem(url); - if (!whitelisted) { - let scriptInfo = await ExternalLicenses.check({url, tabId, frameId}); - if (scriptInfo) { - let verdict; - let msg = scriptInfo.toString(); - if (scriptInfo.allFree) { - verdict = "accepted"; - } else { - verdict = "blocked"; - text = `/* ${msg} */`; - } - addReportEntry(tabId, url, {url, [verdict]: [url, msg]}); - return text; - } - } let edited = await get_script(text, url, tabId, whitelisted, -2); return Array.isArray(edited) ? edited[0] : edited; } From 35daafa5e5f7d094698933816e812d666c739982 Mon Sep 17 00:00:00 2001 From: hackademix Date: Sep 01 2018 22:15:48 +0000 Subject: [PATCH 2/2] Fixed weblabels not being checked in whitelisted documents unless they precede scripts. --- diff --git a/bg/ExternalLicenses.js b/bg/ExternalLicenses.js index 74692d7..0b2c911 100644 --- a/bg/ExternalLicenses.js +++ b/bg/ExternalLicenses.js @@ -35,13 +35,24 @@ let licensesByURL = new Map(); } } +let cachedHrefs = new Map(); + var ExternalLicenses = { + purgeCache(tabId) { + cachedHrefs.delete(tabId); + }, + async check(script) { - let {url, tabId, frameId} = script; + let {url, tabId, frameId, documentUrl} = script; + let tabCache = cachedHrefs.get(tabId); + let frameCache = tabCache && tabCache.get(frameId); + let cache = frameCache && frameCache.get(documentUrl); let scriptInfo = await browser.tabs.sendMessage(tabId, { action: "checkLicensedScript", - url + url, + cache, }, {frameId}); + if (!(scriptInfo && scriptInfo.licenseURLs.length)) { return null; } @@ -65,12 +76,24 @@ var ExternalLicenses = { /** * moves / creates external license references before any script in the page - * if needed, to have them ready when the first script load is triggered + * if needed, to have them ready when the first script load is triggered. + * It also caches the external licens href by page URL, to help not actually + * modify the rendered HTML but rather feed the content script on demand. * Returns true if the document has been actually modified, false otherwise. */ - optimizeDocument(document) { + optimizeDocument(document, cachePointer) { + let cache = {}; + let {tabId, frameId, documentUrl} = cachePointer; + let frameCache = cachedHrefs.get(tabId); + if (!frameCache) { + cachedHrefs.set(tabId, frameCache = new Map()); + } + frameCache.set(frameId, new Map([[documentUrl, cache]])); + let link = document.querySelector(`link[rel="jslicense"], link[data-jslicense="1"], a[rel="jslicense"], a[data-jslicense="1"]`); if (link) { + let href = link.getAttribute("href"); + cache.webLabels = {href}; let move = () => !!document.head.insertBefore(link, document.head.firstChild); if (link.parentNode === document.head) { for (let node; node = link.previousElementSibling;) { @@ -82,12 +105,13 @@ var ExternalLicenses = { if (link.tagName.toUpperCase() === "A") { let newLink = document.createElement("link"); newLink.rel = "jslicense"; - newLink.setAttribute("href", link.getAttribute("href")); + newLink.setAttribute("href", href); link = newLink; } return move(); } } + return false; } }; diff --git a/content/externalLicenseChecker.js b/content/externalLicenseChecker.js index f150cea..be09ef1 100644 --- a/content/externalLicenseChecker.js +++ b/content/externalLicenseChecker.js @@ -22,12 +22,12 @@ { let licensedScripts = null; - let fetchWebLabels = async (map = new Map()) => { + let fetchWebLabels = async args => { // see https://www.gnu.org/software/librejs/free-your-javascript.html#step3 - + let {map, cache} = args; let link = document.querySelector(`link[rel="jslicense"], link[data-jslicense="1"], a[rel="jslicense"], a[data-jslicense="1"]`); - if (link) try { - let baseURL = link.href; + let baseURL = link ? link.href : cache.webLabels && new URL(cache.webLabels.href, document.baseURI); + if (baseURL) try { let response = await fetch(baseURL); if (!response.ok) throw `${response.status} ${response.statusText}`; let doc = new DOMParser().parseFromString( @@ -55,16 +55,16 @@ return map; } - let fetchLicenseInfo = async () => { + let fetchLicenseInfo = async cache => { let map = new Map(); - + let args = {map, cache}; // in the fetchXxx methods we add to a map whatever license(s) // URLs and source code references we can find in various formats // (WebLabels is currently the only implementation), keyed by script URLs. await Promise.all([ - fetchWebLabels(map), - // fetchXmlSpdx(), - // fetchTxtSpdx(), + fetchWebLabels(args), + // fetchXmlSpdx(args), + // fetchTxtSpdx(args), // ... ]); return map; @@ -72,8 +72,8 @@ let handlers = { async checkLicensedScript(m) { - let {url} = m; - if (!licensedScripts) licensedScripts = await fetchLicenseInfo(); + let {url, cache} = m; + if (!licensedScripts) licensedScripts = await fetchLicenseInfo(cache); return licensedScripts.get(url); } } @@ -82,7 +82,6 @@ if (m.action in handlers) try { debug("Received message", m); let result = await handlers[m.action](m); - console.debug("Returning", result); return result; } catch (e) { console.error(e); diff --git a/main_background.js b/main_background.js index a521d22..e2c3635 100644 --- a/main_background.js +++ b/main_background.js @@ -424,6 +424,7 @@ function delete_removed_tab_info(tab_id, remove_info){ if(activeMessagePorts[tab_id] !== undefined){ delete activeMessagePorts[tab_id]; } + ExternalLicenses.purgeCache(tab_id); } /** @@ -855,14 +856,14 @@ var ResponseHandler = { */ async pre(response) { let {request} = response; - let {url, type, tabId, frameId} = request; + let {url, type, tabId, frameId, documentUrl} = request; url = ListStore.urlItem(url); let site = ListStore.siteItem(url); let blacklistedSite = blacklist.contains(site); let blacklisted = blacklistedSite || blacklist.contains(url); - let topUrl = request.frameAncestors && request.frameAncestors.pop() || request.documentUrl; + let topUrl = request.frameAncestors && request.frameAncestors.pop() || documentUrl; if (blacklisted) { if (type === "script") { @@ -886,7 +887,7 @@ var ResponseHandler = { "whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]}); return ResponseProcessor.ACCEPT; } else { - let scriptInfo = await ExternalLicenses.check({url, tabId, frameId}); + let scriptInfo = await ExternalLicenses.check({url, tabId, frameId, documentUrl}); if (scriptInfo) { let verdict, ret; let msg = scriptInfo.toString(); @@ -991,23 +992,26 @@ function read_metadata(meta_element){ return false; } } - /** + * Reads/changes the HTML of a page and the scripts within it. */ -function edit_html(html,url,tabid,wl){ +function edit_html(html, documentUrl, tabId, frameId, whitelisted){ + return new Promise((resolve, reject) => { - if(wl == true){ - // Don't bother, page is whitelisted - resolve(html); - } - + var parser = new DOMParser(); var html_doc = parser.parseFromString(html, "text/html"); // moves external licenses reference, if any, before any