-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathapp.js
44 lines (39 loc) · 1.63 KB
/
app.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
const pup = require('puppeteer');
(async () => {
const b = await pup.launch({headless:false});
// Closes the browser if the process is longer then 1 minutes, helps to close crashed headless chrome
setTimeout(async () => {
console.log('CLOSING BROWSER AUTOMATICALLY')
await b.close()
}, 60000);
const p = await b.newPage();
await p.goto('https://myanimelist.net/topanime.php', {waitUntil: 'networkidle0'});
try {
const pages = await p.evaluate(() => {
const els = Array.from(document.querySelectorAll('.top-ranking-table tr td h3 a'));
return els.map(e => { return { name: e.innerText, url: e.href } });
});
await p.close();
const data = {};
for (const [i, ap] of pages.entries()) {
console.log(`Looping entry #${i} : ${ap.name} (${ap.url})`)
if (i < 6) {
console.log('-- New Page --'); const _p = await b.newPage(); await _p.goto(ap.url, {waitUntil: 'networkidle0'})
const _d = await _p.evaluate(() => {return {
englishName: [...document.querySelectorAll('.spaceit_pad')][0].innerText.replace('English: ', ''),
japaneseName: [...document.querySelectorAll('.spaceit_pad')][2].innerText.replace('Japanese: ', ''),
}});
data[ap.name.replace(/[^a-z0-9"<>#%{}|\\^~\[\]`;?:@=&_\-]/gi, '_').replace(/_{2,}/g, '_').toLowerCase()] = _d;
await _p.close(); console.log('-- Page Closed --')
}
}
console.log('Final DATA', data);
console.log('Closing Browser');
await b.close();
// Make sure to close or it will be open forever
} catch (e) {
// Close on errors
await b.close();
throw new Error(e);
}
})();