yeet headers from url specific params

mendableai · Nov 6, 2024 · 7a54291 · 7a54291
1 parent 66a6f91
commit 7a54291
Showing 1 changed file with 3 additions and 30 deletions.
diff --git a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts
@@ -7,22 +7,8 @@ export type UrlSpecificParams = {
 };
 
 const docsParam: UrlSpecificParams = {
-    scrapeOptions: {
-        waitFor: 2000,
-        headers: {
-            "User-Agent":
-                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-            "sec-fetch-site": "same-origin",
-            "sec-fetch-mode": "cors",
-            "sec-fetch-dest": "empty",
-            referer: "https://www.google.com/",
-            "accept-language": "en-US,en;q=0.9",
-            "accept-encoding": "gzip, deflate, br",
-            accept:
-                "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-        },
-    },
-    internalOptions: { forceEngine: "fire-engine;chrome-cdp" },
+    scrapeOptions: { waitFor: 2000 },
+    internalOptions: {},
 }
 
 export const urlSpecificParams: Record<string, UrlSpecificParams> = {
@@ -33,20 +19,7 @@ export const urlSpecificParams: Record<string, UrlSpecificParams> = {
     "rsseau.fr": docsParam,
     "help.salesforce.com": docsParam,
     "scrapethissite.com": {
-        scrapeOptions: {
-            headers: {
-                "User-Agent":
-                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-                "sec-fetch-site": "same-origin",
-                "sec-fetch-mode": "cors",
-                "sec-fetch-dest": "empty",
-                referer: "https://www.google.com/",
-                "accept-language": "en-US,en;q=0.9",
-                "accept-encoding": "gzip, deflate, br",
-                accept:
-                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-            },
-        },
+        scrapeOptions: {},
         internalOptions: { forceEngine: "fetch" },
     },
     // "eonhealth.com": {