Skip to content

Commit

Permalink
LinkedIn support (beta)
Browse files Browse the repository at this point in the history
  • Loading branch information
stijn-uva committed Aug 23, 2022
1 parent c966811 commit aed0780
Show file tree
Hide file tree
Showing 4 changed files with 457 additions and 5 deletions.
345 changes: 345 additions & 0 deletions inc/he.js

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions js/zs-background.js
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ window.zeeschuimer = {
* @param tabId ID of the tab in which the request was captured
*/
parse_request: async function (response, source_platform_url, source_url, tabId) {
console.log(source_url);
if (!source_platform_url) {
source_platform_url = source_url;
}
Expand Down Expand Up @@ -126,7 +125,6 @@ window.zeeschuimer = {
let item_list = [];
for (let module in this.modules) {
item_list = this.modules[module](response, source_platform_url, source_url);
console.log(item_list);
if (item_list && item_list.length > 0) {
await Promise.all(item_list.map(async (item) => {
if (!item) {
Expand All @@ -140,6 +138,7 @@ window.zeeschuimer = {
await db.items.add({
"nav_index": nav_index,
"item_id": item_id,
"timestamp_collected": Date.now(),
"source_platform": module,
"source_platform_url": source_platform_url,
"source_url": source_url,
Expand Down
8 changes: 5 additions & 3 deletions manifest.json
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
{

"description": "Collect data while browsing TikTok and upload it for analysis later",
"description": "Collect data while browsing social media platforms and upload it for analysis later",
"manifest_version": 2,
"name": "Zeeschuimer",
"version": "1.1.0",
"version": "1.2.0b1",
"homepage_url": "https://github.com/digitalmethodsinitiative/zeeschuimer",

"icons": {
Expand All @@ -23,9 +23,11 @@
"background": {
"scripts": [
"inc/dexie.js",
"inc/he.js",
"js/zs-background.js",
"modules/tiktok.js",
"modules/instagram.js"
"modules/instagram.js",
"modules/linkedin.js"
]
}
}
106 changes: 106 additions & 0 deletions modules/linkedin.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
zeeschuimer.register_module(
"linkedin.com",
function (response, source_platform_url, source_url) {
let domain = source_platform_url.split("/")[2].toLowerCase().replace(/^www\./, '');
if (domain !== 'linkedin.com') {
return [];
}

const sigil = '{"data":{"metadata"';
let data;
try {
data = JSON.parse(response);
} catch (SyntaxError) {
if(response.indexOf(sigil) >= 0) {
// is html
const embedded_json = sigil + response.split(sigil).pop().split('\n')[0];
if (!embedded_json) {
return [];
}

try {
console.log(he.decode(embedded_json));
data = JSON.parse(he.decode(embedded_json));
} catch (SyntaxError) {
return [];
}
} else {
return [];
}
}

if ("data" in data && "included" in data) {
let item_key = '';
if("results" in data["data"]) {
item_key = 'results';
} else if("*elements" in data["data"]) {
item_key = "*elements"
} else {
return [];
}

// XHR updates
// there is a list of objects, each with an ID
// and a separate list of items to display, a list of those IDs
// so the first step is to map item IDs to objects
let mapped_objects = [];


data["included"].forEach(object => {
mapped_objects[object["entityUrn"]] = object;
});

// then we get the objects with the IDs in the item list
// and that is our result set!
let items = [];
for (let object_ref in data["data"][item_key]) {
let result = data["data"][item_key][object_ref];

if (typeof result !== 'string') {
continue;
}

if (result.indexOf('urn:li:fs_updateV2:(') !== 0) {
continue;
}

let result_object = recursively_enrich(mapped_objects[result], mapped_objects);
result_object["id"] = result;

items.push(result_object);
}

return items;
} else {
return [];
}
}
)

function recursively_enrich(object, mapped_objects) {
if(typeof(object) != 'object') {
return object;
}

for(let field in object) {
if(typeof field === 'string' && field.indexOf('*') === 0) {
if(typeof object[field] === 'string' && object[field].indexOf('urn:') === 0) {
// singular reference
object[field] = recursively_enrich(mapped_objects[object[field]], mapped_objects);
}

else if (typeof object[field] === 'object') {
// list of references
for(let i in object[field]) {
if(typeof object[field][i] === 'string' && object[field][i].indexOf('urn:') === 0) {
object[field][i] = recursively_enrich(mapped_objects[object[field]], mapped_objects);
}
}
}
} else {
object[field] = recursively_enrich(object[field], mapped_objects);
}
}

return object;
}

0 comments on commit aed0780

Please sign in to comment.