Update autoconsent (#101)

* Bump up autoconsent

* Fix lint issues

* Fix a test

* Deprecate node 14 and start testing with node 20

* re-enable tsc type-checking

.eslintrc.js

@@ -6,12 +6,23 @@ module.exports = {
  },
  plugins: [
  'mocha',
+ '@typescript-eslint'
  ],
- "extends": "eslint:recommended",
+ "extends": ["eslint:recommended", 'plugin:@typescript-eslint/recommended'],
  "parserOptions": {
- "ecmaVersion": 2020
+ "ecmaVersion": 2021
  },
+ "root": true,
  "rules": {
+ // Not compatible with JSDoc according to @bradzacher and https:/typescript-eslint/typescript-eslint/issues/8955#issuecomment-2097518639
+ '@typescript-eslint/explicit-function-return-type': 'off',
+ '@typescript-eslint/explicit-module-boundary-types': 'off',
+ '@typescript-eslint/parameter-properties': 'off',
+ '@typescript-eslint/typedef': 'off',
+
+ '@typescript-eslint/no-require-imports': 'off',
+ '@typescript-eslint/ban-ts-comment': 'off',
+
  "accessor-pairs": "error",
  "array-bracket-newline": "off",
  "array-bracket-spacing": [

.github/workflows/node.js.yml

@@ -16,7 +16,7 @@ jobs:
 
  strategy:
  matrix:
- node-version: [14.x, 16.x, 18.x]
+ node-version: [16.x, 18.x, 20.x]
 
  steps:
  - uses: actions/checkout@v2

cli/crawl-cli.js

@@ -11,9 +11,9 @@ const {metadataFileExists, createMetadataFile} = require('./metadataFile');
 const crawlConfig = require('./crawlConfig');
 const {createUniqueUrlName} = require('../helpers/hash');
 
-// eslint-disable-next-line no-unused-vars
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const BaseCollector = require('../collectors/BaseCollector');
-// eslint-disable-next-line no-unused-vars
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const BaseReporter = require('../reporters/BaseReporter');
 
 program
@@ -88,7 +88,7 @@ async function run(inputUrls, outputPath, verbose, logPath, numberOfCrawlers, da
 
  try {
  url = new URL(urlString);
- } catch(e) {
+ } catch {
  log(chalk.yellow('Invalid URL:'), urlString);
  return false;
  }

collectors/APICallCollector.js

@@ -166,7 +166,7 @@ class APICallCollector extends BaseCollector {
 
  try {
  url = new URL(urlString);
- } catch (e) {
+ } catch {
  // ignore requests with invalid URL
  return false;
  }

collectors/APICalls/TrackerTracker.js

@@ -253,7 +253,7 @@ class TrackerTracker {
  // calculate absolute URL
  const urlObj = new URL(script, this._mainURL);
  script = urlObj.href;
- } catch(e) {
+ } catch {
  this._log('⚠️ invalid source, assuming global', script);
  script = this._mainURL;
  }
@@ -294,7 +294,7 @@ class TrackerTracker {
 
  try {
  payload = JSON.parse(params.payload);
- } catch(e) {
+ } catch {
  this._log('🚩 invalid breakpoint payload', params.payload);
  return null;
  }

collectors/BaseCollector.js

@@ -9,7 +9,7 @@ class BaseCollector {
  * 
  * @param {CollectorInitOptions} options 
  */
- // eslint-disable-next-line no-unused-vars
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
  init(options) {
  }
 
@@ -18,7 +18,7 @@ class BaseCollector {
  * 
  * @param {{cdpClient: import('puppeteer').CDPSession, url: string, type: import('./TargetCollector').TargetType}} targetInfo 
  */
- // eslint-disable-next-line no-unused-vars
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
  addTarget(targetInfo) {
  }
 
@@ -37,7 +37,7 @@ class BaseCollector {
  * @param {{finalUrl: string, urlFilter?: function(string):boolean}} options
  * @returns {Promise<Object>|Object}
  */
- // eslint-disable-next-line no-unused-vars
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
  getData(options) {
  return Promise.resolve();
  }

collectors/CMPCollector.js

@@ -127,7 +127,7 @@ class CMPCollector extends BaseCollector {
  /**
  * @param {{cdpClient: import('puppeteer').CDPSession, url: string, type: import('./TargetCollector').TargetType}} targetInfo
  */
- // eslint-disable-next-line no-unused-vars
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
  async addTarget(targetInfo) {
  if (targetInfo.type === 'page') {
  this._cdpClient = targetInfo.cdpClient;
@@ -186,14 +186,15 @@ class CMPCollector extends BaseCollector {
  this.receivedMsgs.push(msg);
  switch (msg.type) {
  case 'init': {
- /** @type {AutoconsentConfig} */
+ /** @type {Partial<AutoconsentConfig>} */
  const autoconsentConfig = {
  enabled: true,
  autoAction: null, // we request action explicitly later
  disabledCmps: [],
  enablePrehide: false,
  enableCosmeticRules: true,
  detectRetries: 20,
+ isMainWorld: false
  };
  await this._cdpClient.send('Runtime.evaluate', {
  expression: `autoconsentReceiveMessage({ type: "initResp", config: ${JSON.stringify(autoconsentConfig)} })`,

collectors/RequestCollector.js

@@ -93,7 +93,7 @@ class RequestCollector extends BaseCollector {
  }
 
  return crypto.createHash('sha256').update(body).digest('hex');
- } catch (e) {
+ } catch {
  return null;
  }
  }
@@ -315,7 +315,7 @@ class RequestCollector extends BaseCollector {
 
  try {
  url = new URL(request.url);
- } catch (e) {
+ } catch {
  // ignore requests with invalid URL
  return false;
  }

crawler.js

@@ -48,7 +48,7 @@ function openBrowser(log, proxyHost, executablePath) {
  let url;
  try {
  url = new URL(proxyHost);
- } catch(e) {
+ } catch {
  log('Invalid proxy URL');
  }
 
@@ -257,7 +257,7 @@ async function getSiteData(context, url, {
  try {
  // eslint-disable-next-line no-await-in-loop
  await target.cdpClient.detach();
- } catch (e) {
+ } catch {
  // we don't care that much because in most cases an error here means that target already detached
  }
  }

crawlerConductor.js

@@ -7,7 +7,7 @@ const URL = require('url').URL;
 const {createTimer} = require('./helpers/timer');
 const createDeferred = require('./helpers/deferred');
 const downloadCustomChromium = require('./helpers/downloadCustomChromium');
-// eslint-disable-next-line no-unused-vars
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const BaseCollector = require('./collectors/BaseCollector');
 const notABot = require('./helpers/notABot');
 

helpers/collectorsList.js

@@ -2,7 +2,7 @@
  * @fileoverview Helper that provides IDs of all available collectors (based on the main.js file) and helps creating instances of collectors
  */
 const allExports = require('../main');
-// eslint-disable-next-line no-unused-vars
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const BaseCollector = require('../collectors/BaseCollector');
 const collectorClasses = Object.entries(allExports).filter(([name]) => name.endsWith('Collector')).map(([,collector]) => collector);
 const collectors = collectorClasses.map(CollectorClass => ({

helpers/reportersList.js

@@ -1,4 +1,4 @@
-// eslint-disable-next-line no-unused-vars
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const BaseReporter = require('../reporters/BaseReporter');
 const CLIReporter = require('./../reporters/CLIReporter');
 const FileReporter = require('./../reporters/FileReporter');