#10 WebLabels-based license checking implementation.
Merged 5 years ago by quidam. Opened 5 years ago by gioma1.
gioma1/librejs feature/weblabels  into  master

@@ -0,0 +1,96 @@ 

+ /**

+ * GNU LibreJS - A browser add-on to block nonfree nontrivial JavaScript.

+ *

+ * Copyright (C) 2018 Giorgio Maone <giorgio@maone.net>

+ *

+ * This file is part of GNU LibreJS.

+ *

+ * GNU LibreJS is free software: you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as published by

+ * the Free Software Foundation, either version 3 of the License, or

+ * (at your option) any later version.

+ *

+ * GNU LibreJS is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public License

+ * along with GNU LibreJS.  If not, see <http://www.gnu.org/licenses/>.

+ */

+ 

+ /**

+   Singleton to handle external licenses, e.g. WebLabels

+ */

+ 

+ "use strict";

+ 

+ let licensesByURL = new Map();

+ {

+   let {licenses} = require("../license_definitions");

+   for (let l of Object.values(licenses).filter(l => l.canonicalUrl)) {

+     for (let url of l.canonicalUrl) {

+       licensesByURL.set(url, l);

+     }

+   }

+ }

+ 

+ var ExternalLicenses = {

+   async check(script) {

+     let {url, tabId, frameId} = script;

+     let scriptInfo = await browser.tabs.sendMessage(tabId, {

+       action: "checkLicensedScript",

+       url

+     }, {frameId});

+     if (!(scriptInfo && scriptInfo.licenseURLs.length)) {

+       return null;

+     }

+     scriptInfo.licenses = new Set();

+     scriptInfo.allFree = true;

+     scriptInfo.toString = function() {

+       let licenseIds = [...this.licenses].map(l => l.identifier).sort().join(", ");

+       return this.allFree ? `Free license${licenseIds.length > 1 ? "s" : ""} (${licenseIds})` : `Mixed free (${licenseIds}) and unknown licenses`;

+     }

+     

+     for (let u of scriptInfo.licenseURLs) {

+       if (licensesByURL.has(u)) {

+         scriptInfo.licenses.add(licensesByURL.get(u));

+       } else {

+         scriptInfo.allFree = false;

+         break;

+       }

+     }

+     return scriptInfo;

+   },

+   

+   /**

+   * moves / creates external license references before any script in the page

+   * if needed, to have them ready when the first script load is triggered

+   * Returns true if the document has been actually modified, false otherwise.

+   */

+   optimizeDocument(document) {

+     let link = document.querySelector(`link[rel="jslicense"], link[data-jslicense="1"], a[rel="jslicense"], a[data-jslicense="1"]`);

+     if (link) {

+       let move = () => !!document.head.insertBefore(link, document.head.firstChild);

+       if (link.parentNode === document.head) {

+         for (let node; node = link.previousElementSibling;) {

+           if (node.tagName.toUpperCase() === "SCRIPT") {

+             return move();

+           }

+         }

+       } else { // the reference is only in the body

+         if (link.tagName.toUpperCase() === "A") {

+           let newLink = document.createElement("link");

+           newLink.rel = "jslicense";

+           newLink.setAttribute("href", link.getAttribute("href"));

+           link = newLink;

+         }

+         return move();

+       }

+     }

+     return false;

+   }

+ };

+ 

+ 

+ module.exports = { ExternalLicenses };

file modified
+1
@@ -11,6 +11,7 @@ 

  # Move source files to temp directory

  cp -r icons ./build_temp

  cp -r ./html ./build_temp

+ cp -r ./content ./build_temp

  cp manifest.json ./build_temp

  cp contact_finder.js ./build_temp

  cp bundle.js ./build_temp

@@ -0,0 +1,87 @@ 

+ /**

+ * GNU LibreJS - A browser add-on to block nonfree nontrivial JavaScript.

+ *

+ * Copyright (C) 2018 Giorgio Maone <giorgio@maone.net>

+ *

+ * This file is part of GNU LibreJS.

+ *

+ * GNU LibreJS is free software: you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as published by

+ * the Free Software Foundation, either version 3 of the License, or

+ * (at your option) any later version.

+ *

+ * GNU LibreJS is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public License

+ * along with GNU LibreJS.  If not, see <http://www.gnu.org/licenses/>.

+ */

+ "use strict";

+ {

+   let licensedScripts = null;

+     

+   let fetchWebLabels = async (map = new Map()) => {

+     // see https://www.gnu.org/software/librejs/free-your-javascript.html#step3

+     

+     let link = document.querySelector(`link[rel="jslicense"], link[data-jslicense="1"], a[rel="jslicense"], a[data-jslicense="1"]`);

+     if (link) try {

+       let baseURL = link.href;

+       let response = await fetch(baseURL);

+       if (!response.ok) throw `${response.status} ${response.statusText}`;

+       let doc = new DOMParser().parseFromString(

+           await response.text(),

+           "text/html"

+       );

+       let base = doc.querySelector("base");

+       if (base) {

+         base.href = base.href;

+       } else {

+         doc.head.appendChild(doc.createElement("base")).href = baseURL;

+       }

+       let firstURL = parent => parent.querySelector("a").href;

+       let allURLs = parent => Array.map(parent.querySelectorAll("a"), a => a.href);

+       for (let row of doc.querySelectorAll("table#jslicense-labels1 tr")) {

+         let cols = row.querySelectorAll("td");

+         let scriptURL = firstURL(cols[0]);

+         let licenseURLs = allURLs(cols[1]);

+         let sourceURLs = cols[2] ? allURLs(cols[2]) : [];

+         map.set(scriptURL, {scriptURL, licenseURLs, sourceURLs});

+       }

+     } catch (e) {

+       console.error("Error fetching Web Labels at %o", link, e);

+     }

+     return map;

+   }

+   

+   let fetchLicenseInfo = async () => {

+     let map = new Map();

+     

+     // in the fetchXxx methods we add to a map whatever license(s)

+     // URLs and source code references we can find in various formats 

+     // (WebLabels is currently the only implementation), keyed by script URLs.

+     await Promise.all([

+     fetchWebLabels(map),

+     // fetchXmlSpdx(),

+     // fetchTxtSpdx(),

+     // ...

+     ]);

+     return map;

+   }

+   

+   let handlers = {

+     async checkLicensedScript(m) {

+       let {url} = m;

+       if (!licensedScripts) licensedScripts = await fetchLicenseInfo();

+       return licensedScripts.get(url);

+     }

+   }

+ 

+   browser.runtime.onMessage.addListener(async m => {

+     if (m.action in handlers) {

+       debug("Received message", m);

+       return await handlers[m.action](m);

+     }

+   });

+ }

file modified
+36 -15
@@ -28,6 +28,7 @@ 

  var {ResponseProcessor} = require("./bg/ResponseProcessor");

  var {Storage, ListStore} = require("./bg/Storage");

  var {ListManager} = require("./bg/ListManager");

+ var {ExternalLicenses} = require("./bg/ExternalLicenses");

  

  console.log("main_background.js");

  /**
@@ -152,18 +153,21 @@ 

  

  var activeMessagePorts = {};

  var activityReports = {};

- function createReport(initializer = null) {

+ async function createReport(initializer = null) {

  	let template =  {

  		"accepted": [],

  		"blocked": [],

  		"blacklisted": [],

  		"whitelisted": [],

  		"unknown": [],

- 		url: "",

  	};

  	if (initializer) {

  		template = Object.assign(template, initializer);

+ 		if (!template.url && initializer.tabId) {

+ 			template.url = (await browser.tabs.get(initializer.tabId)).url;

+ 		}

  	}

+ 	

  	template.site = ListStore.siteItem(template.url);

  	template.siteStatus = listManager.getStatus(template.site);

  	return template;
@@ -177,7 +181,7 @@ 

  async function openReportInTab(data) {

  	let popupURL = await browser.browserAction.getPopup({});

  	let tab = await browser.tabs.create({url: `${popupURL}#fromTab=${data.tabId}`});

- 	activityReports[tab.id] = createReport(data);

+ 	activityReports[tab.id] = await createReport(data);

  }

  

  /**
@@ -222,9 +226,9 @@ 

  *	Make sure it will use the right URL when refering to a certain script.

  * 

  */

- function updateReport(tabId, oldReport, updateUI = false){

+ async function updateReport(tabId, oldReport, updateUI = false){

  	let {url} = oldReport;

- 	let newReport = createReport({url, tabId});

+ 	let newReport = await createReport({url, tabId});

  	for (let property of Object.keys(oldReport)) {

  		let entries = oldReport[property];

  		if (!Array.isArray(entries)) continue;
@@ -263,10 +267,10 @@ 

  *	Make sure it will use the right URL when refering to a certain script.

  *

  */

- async function addReportEntry(tabId, scriptHashOrUrl, action, update = false) {

+ async function addReportEntry(tabId, scriptHashOrUrl, action) {

  	let report = activityReports[tabId];

  	if (!report) report = activityReports[tabId] = 

- 			createReport({url: (await browser.tabs.get(tabId)).url});

+ 			await createReport({tabId});

  			

  	let type, actionValue;

  	for (type of ["accepted", "blocked", "whitelisted", "blacklisted"]) {
@@ -312,7 +316,7 @@ 

  	}

  	

  	browser.sessions.setTabValue(tabId, report.url, report);

- 	

+ 	updateBadge(tabId, report);

  	return entryType;

  }

  
@@ -397,7 +401,7 @@ 

  					p.postMessage({"show_info": activityReports[tab.id]});

  				} else{

  					// create a new entry

- 					let report = activityReports[tab.id] = createReport({"url": tab.url, tabId: tab.id});

+ 					let report = activityReports[tab.id] = await createReport({"url": tab.url, tabId: tab.id});

  					p.postMessage({show_info: report});							

  					dbg_print(`[TABID: ${tab.id}] No data found, creating a new entry for this window.`);	

  				}
@@ -781,7 +785,7 @@ 

  	

  	let sourceHash = hash(response);

   	let domain = get_domain(url);

- 	let report = activityReports[tabId] || (activityReports[tabId] = createReport({url, tabId}));

+ 	let report = activityReports[tabId] || (activityReports[tabId] = await createReport({tabId}));

  	updateBadge(tabId, report, !verdict);

  	let category = await addReportEntry(tabId, sourceHash, {"url": domain, [verdict ? "accepted" : "blocked"]: [url, reason]});

  	let scriptSource = verdict ? response : editedSource;
@@ -903,8 +907,23 @@ 

  */

  async function handle_script(response, whitelisted){

  	let {text, request} = response;

- 	let {url, tabId} = request;

+ 	let {url, tabId, frameId} = request;

  	url = ListStore.urlItem(url);

+ 	if (!whitelisted) {

+ 		let scriptInfo = await ExternalLicenses.check({url, tabId, frameId});

+ 		if (scriptInfo) {

+ 			let verdict;

+ 			let msg = scriptInfo.toString();

+ 			if (scriptInfo.allFree) {

+ 				verdict = "accepted";

+ 			} else {

+ 				verdict = "blocked";

+ 				text = `/* ${msg} */`;

+ 			}

+ 			addReportEntry(tabId, url, {url, [verdict]: [url, msg]});

+ 			return text;

+ 		}

+ 	}

    let edited = await get_script(text, url, tabId, whitelisted, -2);

  	return Array.isArray(edited) ? edited[0] : edited;

  }
@@ -985,7 +1004,10 @@ 

  		

  		var parser = new DOMParser();

  		var html_doc = parser.parseFromString(html, "text/html");

- 

+ 		

+ 		// moves external licenses reference, if any, before any <SCRIPT> element

+ 		ExternalLicenses.optimizeDocument(html_doc); 

+ 		

  		var amt_scripts = 0;

  		var total_scripts = 0;

  		var scripts = html_doc.scripts;
@@ -1077,12 +1099,11 @@ 

  async function handle_html(response, whitelisted) {

  	let {text, request} = response;

  	let {url, tabId, type} = request;

- 	url = ListStore.urlItem(url);

  	if (type === "main_frame") { 

- 		activityReports[tabId] = createReport({url, tabId});

+ 		activityReports[tabId] = await createReport({url, tabId});

  		updateBadge(tabId);

  	}

- 	return await edit_html(text, url, tabId, whitelisted);

+ 	return await edit_html(text, ListStore.urlItem(url), tabId, whitelisted);

  }

  

  var whitelist = new ListStore("pref_whitelist", Storage.CSV);

file modified
+12 -2
@@ -41,6 +41,16 @@ 

    ],

    "background": {

      "scripts": ["bundle.js"]

-   }

- 

+   },

+   "content_scripts": [

+     {

+       "run_at": "document_start",

+       "matches": ["<all_urls>"],

+       "match_about_blank": true,

+       "all_frames": true,

+       "js": [

+         "content/externalLicenseChecker.js"

+       ]

+     }

+   ]

  }

Crude example page with two scripts, one declaring all free licenses (accepted) and one declaring mixed licenses (blocked) here: https://noscript.net/test/librejs/weblabels/

2 new commits added

  • Fixed regression: activity reports being accidentally reset when top document URL contains a query string.
  • Added the new ./content directory to the xpi.
5 years ago

Pull-Request has been merged by quidam

5 years ago