fix: adblock caching

package.json

@@ -34,16 +34,15 @@
     "url": "https://github.com/wexond/wexond/issues"
   },
   "scripts": {
-    "dev": "npm run download-filters && cross-env NODE_ENV='dev' node fuse.js",
-    "build": "npm run download-filters && node fuse.js",
+    "dev": "cross-env NODE_ENV='dev' node fuse.js",
+    "build": "node fuse.js",
     "start": "cross-env ENV='dev' electron .",
     "compile-win32": "npm run build && electron-builder -w -p always",
     "compile-darwin": "npm run build && electron-builder -m -p always",
     "compile-linux": "npm run build && electron-builder -l -p always",
     "lint": "tslint \"src/**/*.ts*\"",
     "lint-fix": "prettier --write \"src/**/*.ts*\" && tslint \"src/**/*.ts*\" --fix",
-    "postinstall": "electron-builder install-app-deps",
-    "download-filters": "node tools/download-filters.js"
+    "postinstall": "electron-builder install-app-deps"
   },
   "pre-commit": [
     "lint-fix"
@@ -59,7 +58,6 @@
     "@types/react": "16.8.13",
     "@types/react-dom": "16.8.4",
     "@types/styled-components": "4.1.14",
-    "axios": "0.18.0",
     "concurrently": "4.1.0",
     "cross-env": "5.2.0",
     "electron": "4.1.4",
@@ -81,6 +79,7 @@
     "file-type": "10.11.0",
     "gsap": "2.1.2",
     "icojs": "0.12.3",
+    "axios": "0.18.0",
     "leveldown": "5.0.1",
     "levelup": "4.0.1",
     "mobx": "5.9.4",

src/main/index.ts

@@ -19,16 +19,16 @@ export let appWindow: AppWindow;
 
 registerProtocols();
 
-if (!existsSync(getPath('settings.json'))) {
-  writeFileSync(
-    getPath('settings.json'),
-    JSON.stringify({
-      dialType: 'top-sites',
-    } as Settings),
-  );
-}
-
 app.on('ready', () => {
+  if (!existsSync(getPath('settings.json'))) {
+    writeFileSync(
+      getPath('settings.json'),
+      JSON.stringify({
+        dialType: 'top-sites',
+      } as Settings),
+    );
+  }
+
   // Create our menu entries so that we can use macOS shortcuts
   Menu.setApplicationMenu(
     Menu.buildFromTemplate([

src/main/services/web-request.ts

@@ -3,10 +3,10 @@ import { makeId } from '~/shared/utils/string';
 import { AppWindow } from '../app-window';
 import { matchesPattern } from '~/shared/utils/url';
 import { USER_AGENT } from '~/shared/constants';
-import { existsSync, readFile } from 'fs';
-import console = require('console');
+import { existsSync, readFile, writeFile, mkdirSync } from 'fs';
 import { resolve } from 'path';
 import { appWindow } from '..';
+import Axios from 'axios';
 
 import {
   FiltersEngine,
@@ -16,13 +16,34 @@ import {
 } from '@cliqz/adblocker';
 import { parse } from 'tldts';
 import { requestURL } from '~/renderer/app/utils/network';
+import { getPath } from '~/shared/utils/paths';
+
+const lists: any = {
+  easylist: 'https://easylist.to/easylist/easylist.txt',
+  easyprivacy: 'https://easylist.to/easylist/easyprivacy.txt',
+  malwaredomains: 'http://mirror1.malwaredomains.com/files/justdomains',
+  nocoin:
+    'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/nocoin.txt',
+  'ublock-filters':
+    'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt',
+  'ublock-badware':
+    'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt',
+  'ublock-privacy':
+    'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt',
+  'ublock-unbreak':
+    'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt',
+};
 
 export let engine: FiltersEngine;
 
 const eventListeners: any = {};
 
 export const loadFilters = async () => {
-  const path = resolve(app.getAppPath(), 'filters/default.dat');
+  if (!existsSync(getPath('adblock'))) {
+    mkdirSync(getPath('adblock'));
+  }
+
+  const path = resolve(getPath('adblock/cache.dat'));
 
   /*const { data } = await requestURL(
     'https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt',
@@ -44,6 +65,26 @@ export const loadFilters = async () => {
         newCosmeticFilters: cosmeticFilters,
       });*/
     });
+  } else {
+    const ops = [];
+
+    for (const key in lists) {
+      ops.push(Axios.get(lists[key]));
+    }
+
+    Axios.all(ops).then(res => {
+      let data = '';
+
+      for (const res1 of res) {
+        data += res1.data;
+      }
+
+      engine = FiltersEngine.parse(data);
+
+      writeFile(path, engine.serialize(), err => {
+        if (err) return console.error(err);
+      });
+    });
   }
 };