Merge branch 'mog/webscraper-refactor' of https://github.com/mendable…

…ai/firecrawl into mog/webscraper-refactor
mendableai · Nov 7, 2024 · f9e775a · f9e775a
2 parents a02c42a + 7198a28
commit f9e775a
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 9 deletions.
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
@@ -509,7 +509,7 @@ export function fromLegacyScrapeOptions(pageOptions: PageOptions, extractorOptio
       v0DisableJsDom: pageOptions.disableJsDom,
       v0UseFastMode: pageOptions.useFastMode,
     },
-    // TODO: fallback, fetchPage  Content, replaceAllPathsWithAbsolutePaths, includeLinks
+    // TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
   }
 }
 

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -87,8 +87,7 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engi
         priority: meta.internalOptions.priority,
         geolocation: meta.options.geolocation,
         mobile: meta.options.mobile,
-        removeBase64Images: meta.options.removeBase64Images,
-        // TODO: scrollXPaths, disableJsDom
+        // TODO: scrollXPaths
     };
 
     let response = await performFireEngineScrape(
@@ -137,7 +136,7 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<Eng
         screenshot: meta.options.formats.includes("screenshot"),
         fullPageScreenshot: meta.options.formats.includes("screenshot@fullPage"),
         wait: meta.options.waitFor,
-        removeBase64Images: meta.options.removeBase64Images,
+        geolocation: meta.options.geolocation,
     };
 
     let response = await performFireEngineScrape(
@@ -175,7 +174,6 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engi
 
         atsv: meta.internalOptions.atsv,
         geolocation: meta.options.geolocation,
-        removeBase64Images: meta.options.removeBase64Images,
         disableJsDom: meta.internalOptions.v0DisableJsDom,
     };
 

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -24,16 +24,14 @@ export type FireEngineScrapeRequestCommon = {
     // team_id?: string; // unused
     logRequest?: boolean; // default: true
     instantReturn?: boolean; // default: false
-
-    removeBase64Images?: boolean;
+    geolocation?: { country?: string; languages?: string[]; };
 }
 
 export type FireEngineScrapeRequestChromeCDP = {
     engine: "chrome-cdp";
     skipTlsVerification?: boolean;
     actions?: Action[];
     blockMedia?: true; // cannot be false
-    geolocation?: { country?: string; languages?: string[]; };
     mobile?: boolean;
 };
 
@@ -51,7 +49,6 @@ export type FireEngineScrapeRequestPlaywright = {
 export type FireEngineScrapeRequestTLSClient = {
     engine: "tlsclient";
     atsv?: boolean; // v0 only, default: false
-    geolocation?: { country?: string; languages?: string[]; };
     disableJsDom?: boolean; // v0 only, default: false
     // blockAds?: boolean; // default: true
 };

diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -6,6 +6,7 @@ import { extractLinks } from "../lib/extractLinks";
 import { extractMetadata } from "../lib/extractMetadata";
 import { performLLMExtract } from "./llmExtract";
 import { uploadScreenshot } from "./uploadScreenshot";
+import { removeBase64Images } from "./removeBase64Images";
 
 export type Transformer = (meta: Meta, document: Document) => Document | Promise<Document>;
 
@@ -110,6 +111,7 @@ export const transformerStack: Transformer[] = [
     uploadScreenshot,
     performLLMExtract,
     coerceFieldsToFormats,
+    removeBase64Images, 
 ];
 
 export async function executeTransformers(meta: Meta, document: Document): Promise<Document> {

diff --git a/apps/api/src/scraper/scrapeURL/transformers/removeBase64Images.ts b/apps/api/src/scraper/scrapeURL/transformers/removeBase64Images.ts
@@ -0,0 +1,11 @@
+import { Meta } from "..";
+import { Document } from "../../../controllers/v1/types";
+
+const regex = /(!\[.*?\])\(data:image\/.*?;base64,.*?\)/g;
+
+export function removeBase64Images(meta: Meta, document: Document): Document {
+    if (meta.options.removeBase64Images && document.markdown !== undefined) {
+        document.markdown = document.markdown.replace(regex, '$1(<Base64-Image-Removed>)');
+    }
+    return document;
+}