From 66a6f919c6b939f7875d44e3e83b64aa9c6a6291 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=B3ricz=20Gerg=C5=91?= <mo.geryy@gmail.com>
Date: Wed, 6 Nov 2024 23:55:05 +0100
Subject: [PATCH] fixes

---
 apps/api/src/controllers/v1/types.ts            |  4 +++-
 .../scrapeURL/engines/fire-engine/index.ts      |  1 +
 .../scrapeURL/engines/fire-engine/scrape.ts     |  1 +
 apps/api/src/scraper/scrapeURL/engines/index.ts | 17 +++++++++++------
 apps/api/src/scraper/scrapeURL/index.ts         | 15 +++++++++++++--
 5 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 87e989bad..69ffcdfe1 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -505,8 +505,10 @@ export function fromLegacyScrapeOptions(pageOptions: PageOptions, extractorOptio
     }),
     internalOptions: {
       atsv: pageOptions.atsv,
+      v0DisableJsDom: pageOptions.disableJsDom,
+      v0UseFastMode: pageOptions.useFastMode,
     },
-    // TODO: fallback, fetchPage  Content, replaceAllPathsWithAbsolutePaths, includeLinks, useFastMode, disableJsDom
+    // TODO: fallback, fetchPage  Content, replaceAllPathsWithAbsolutePaths, includeLinks
   }
 }
 
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index bd994cbf8..c8ea6aefa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -176,6 +176,7 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engi
         atsv: meta.internalOptions.atsv,
         geolocation: meta.options.geolocation,
         removeBase64Images: meta.options.removeBase64Images,
+        disableJsDom: meta.internalOptions.v0DisableJsDom,
     };
 
     let response = await performFireEngineScrape(
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index be65c60c0..21e1c8028 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -52,6 +52,7 @@ export type FireEngineScrapeRequestTLSClient = {
     engine: "tlsclient";
     atsv?: boolean; // v0 only, default: false
     geolocation?: { country?: string; languages?: string[]; };
+    disableJsDom?: boolean; // v0 only, default: false
     // blockAds?: boolean; // default: true
 };
 
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 73eacfb24..aadef7fc6 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -33,6 +33,7 @@ export const featureFlags = [
     "location",
     "mobile",
     "skipTlsVerification",
+    "useFastMode",
 ] as const;
 
 export type FeatureFlag = typeof featureFlags[number];
@@ -49,6 +50,7 @@ export const featureFlagOptions: {
     "pdf": { priority: 100 },
     "docx": { priority: 100 },
     "atsv": { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if not
+    "useFastMode": { priority: 90 },
     "location": { priority: 10 },
     "mobile": { priority: 10 },
     "skipTlsVerification": { priority: 10 },
@@ -105,6 +107,7 @@ export const engineOptions: {
             "location": true,
             "mobile": true,
             "skipTlsVerification": true,
+            "useFastMode": false,
         },
         quality: 50,
     },
@@ -120,6 +123,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
+            "useFastMode": false,
         },
         quality: 40,
     },
@@ -135,7 +139,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": false,
         },
         quality: 30,
     },
@@ -151,7 +155,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": false,
         },
         quality: 29,
     },
@@ -167,7 +171,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": false,
         },
         quality: 20,
     },
@@ -183,7 +187,7 @@ export const engineOptions: {
             "location": true,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": true,
         },
         quality: 10,
     },
@@ -199,7 +203,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": true,
         },
         quality: 5,
     },
@@ -215,7 +219,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": true,
         },
         quality: -10,
     },
@@ -231,6 +235,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
+            "useFastMode": true,
         },
         quality: -10,
     },
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0aa22d379..3fde5db34 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -65,6 +65,10 @@ function buildFeatureFlags(url: string, options: ScrapeOptions, internalOptions:
         flags.add("skipTlsVerification");
     }
 
+    if (internalOptions.v0UseFastMode) {
+        flags.add("useFastMode");
+    }
+
     const urlO = new URL(url);
 
     if (urlO.pathname.endsWith(".pdf")) {
@@ -78,6 +82,11 @@ function buildFeatureFlags(url: string, options: ScrapeOptions, internalOptions:
     return flags;
 }
 
+// The meta object contains all required information to perform a scrape.
+// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.
+// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)
+// Having a meta object that is treated as immutable helps the code stay clean and easily tracable,
+// while also retaining the benefits that WebScraper had from its OOP design.
 function buildMetaObject(id: string, url: string, options: ScrapeOptions, internalOptions: InternalOptions): Meta {
     const specParams = urlSpecificParams[new URL(url).hostname.replace(/^www\./, "")];
     if (specParams !== undefined) {
@@ -103,6 +112,8 @@ export type InternalOptions = {
     atsv?: boolean; // anti-bot solver, beta
 
     v0CrawlOnlyUrls?: boolean;
+    v0UseFastMode?: boolean;
+    v0DisableJsDom?: boolean;
 };
 
 export type EngineResultsTracker = { [E in Engine]?: {
@@ -147,8 +158,8 @@ async function scrapeURLLoop(
             const engineResult = _engineResult as EngineScrapeResult & { markdown: string };
 
             // Success factors
-            const isLongEnough = engineResult.markdown.length >= 1;
-            const isGoodStatusCode = engineResult.statusCode < 300;
+            const isLongEnough = engineResult.markdown.length >= 20;
+            const isGoodStatusCode = (engineResult.statusCode >= 200 && engineResult.statusCode < 300) || engineResult.statusCode === 304;
             const hasNoPageError = engineResult.error === undefined;
 
             results[engine] = {