fixes

mendableai · Nov 6, 2024 · 66a6f91 · 66a6f91
1 parent 8616fe6
commit 66a6f91
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 9 deletions.
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
@@ -505,8 +505,10 @@ export function fromLegacyScrapeOptions(pageOptions: PageOptions, extractorOptio
     }),
     internalOptions: {
       atsv: pageOptions.atsv,
+      v0DisableJsDom: pageOptions.disableJsDom,
+      v0UseFastMode: pageOptions.useFastMode,
     },
-    // TODO: fallback, fetchPage  Content, replaceAllPathsWithAbsolutePaths, includeLinks, useFastMode, disableJsDom
+    // TODO: fallback, fetchPage  Content, replaceAllPathsWithAbsolutePaths, includeLinks
   }
 }
 

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -176,6 +176,7 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engi
         atsv: meta.internalOptions.atsv,
         geolocation: meta.options.geolocation,
         removeBase64Images: meta.options.removeBase64Images,
+        disableJsDom: meta.internalOptions.v0DisableJsDom,
     };
 
     let response = await performFireEngineScrape(

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -52,6 +52,7 @@ export type FireEngineScrapeRequestTLSClient = {
     engine: "tlsclient";
     atsv?: boolean; // v0 only, default: false
     geolocation?: { country?: string; languages?: string[]; };
+    disableJsDom?: boolean; // v0 only, default: false
     // blockAds?: boolean; // default: true
 };
 

diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -33,6 +33,7 @@ export const featureFlags = [
     "location",
     "mobile",
     "skipTlsVerification",
+    "useFastMode",
 ] as const;
 
 export type FeatureFlag = typeof featureFlags[number];
@@ -49,6 +50,7 @@ export const featureFlagOptions: {
     "pdf": { priority: 100 },
     "docx": { priority: 100 },
     "atsv": { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if not
+    "useFastMode": { priority: 90 },
     "location": { priority: 10 },
     "mobile": { priority: 10 },
     "skipTlsVerification": { priority: 10 },
@@ -105,6 +107,7 @@ export const engineOptions: {
             "location": true,
             "mobile": true,
             "skipTlsVerification": true,
+            "useFastMode": false,
         },
         quality: 50,
     },
@@ -120,6 +123,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
+            "useFastMode": false,
         },
         quality: 40,
     },
@@ -135,7 +139,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": false,
         },
         quality: 30,
     },
@@ -151,7 +155,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": false,
         },
         quality: 29,
     },
@@ -167,7 +171,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": false,
         },
         quality: 20,
     },
@@ -183,7 +187,7 @@ export const engineOptions: {
             "location": true,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": true,
         },
         quality: 10,
     },
@@ -199,7 +203,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": true,
         },
         quality: 5,
     },
@@ -215,7 +219,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
-
+            "useFastMode": true,
         },
         quality: -10,
     },
@@ -231,6 +235,7 @@ export const engineOptions: {
             "location": false,
             "mobile": false,
             "skipTlsVerification": false,
+            "useFastMode": true,
         },
         quality: -10,
     },

diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
@@ -65,6 +65,10 @@ function buildFeatureFlags(url: string, options: ScrapeOptions, internalOptions:
         flags.add("skipTlsVerification");
     }
 
+    if (internalOptions.v0UseFastMode) {
+        flags.add("useFastMode");
+    }
+
     const urlO = new URL(url);
 
     if (urlO.pathname.endsWith(".pdf")) {
@@ -78,6 +82,11 @@ function buildFeatureFlags(url: string, options: ScrapeOptions, internalOptions:
     return flags;
 }
 
+// The meta object contains all required information to perform a scrape.
+// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.
+// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)
+// Having a meta object that is treated as immutable helps the code stay clean and easily tracable,
+// while also retaining the benefits that WebScraper had from its OOP design.
 function buildMetaObject(id: string, url: string, options: ScrapeOptions, internalOptions: InternalOptions): Meta {
     const specParams = urlSpecificParams[new URL(url).hostname.replace(/^www\./, "")];
     if (specParams !== undefined) {
@@ -103,6 +112,8 @@ export type InternalOptions = {
     atsv?: boolean; // anti-bot solver, beta
 
     v0CrawlOnlyUrls?: boolean;
+    v0UseFastMode?: boolean;
+    v0DisableJsDom?: boolean;
 };
 
 export type EngineResultsTracker = { [E in Engine]?: {
@@ -147,8 +158,8 @@ async function scrapeURLLoop(
             const engineResult = _engineResult as EngineScrapeResult & { markdown: string };
 
             // Success factors
-            const isLongEnough = engineResult.markdown.length >= 1;
-            const isGoodStatusCode = engineResult.statusCode < 300;
+            const isLongEnough = engineResult.markdown.length >= 20;
+            const isGoodStatusCode = (engineResult.statusCode >= 200 && engineResult.statusCode < 300) || engineResult.statusCode === 304;
             const hasNoPageError = engineResult.error === undefined;
 
             results[engine] = {