Nick:

mendableai · Nov 6, 2024 · 621df86 · 621df86
1 parent 9b271d7
commit 621df86
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 12 deletions.
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
@@ -102,6 +102,7 @@ describe("E2E Tests for v1 API Routes", () => {
   // - TODO: tests for screenshot@fullPage
 
   it.concurrent("should handle 'headers' parameter correctly", async () => {
+    //@ts-ignore
     const scrapeRequest = {
       url: E2E_TEST_SERVER_URL,
       headers: { "e2e-header-test": "firecrawl" }
@@ -297,25 +298,24 @@ describe("E2E Tests for v1 API Routes", () => {
     async () => {
       const scrapeRequest = {
         url: "https://expired.badssl.com/",
+        timeout: 120000
       } as ScrapeRequest;
 
       const response: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
         .post("/v1/scrape")
         .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
         .set("Content-Type", "application/json")
         .send(scrapeRequest);
-
-      expect(response.statusCode).toBe(200);
-      if (!("data" in response.body)) {
-        throw new Error("Expected response body to have 'data' property");
-      }
-      const metadata = response.body.data.metadata as { error: string; statusCode: number };
-      expect(metadata.error).toContain("certificate has expired")
-      expect(metadata.statusCode).toBe(500)
+        console.log("Error1a")
+        console.log(response.body)
+      expect(response.statusCode).toBe(500);
+      console.log("Error?")
 
       const scrapeRequestWithSkipTlsVerification = {
         url: "https://expired.badssl.com/",
-        skipTlsVerification: true
+        skipTlsVerification: true,
+        timeout: 120000
+
       } as ScrapeRequest;
 
       const responseWithSkipTlsVerification: ScrapeResponseRequestTest = await request(FIRECRAWL_API_URL)
@@ -324,6 +324,8 @@ describe("E2E Tests for v1 API Routes", () => {
         .set("Content-Type", "application/json")
         .send(scrapeRequestWithSkipTlsVerification);
 
+      console.log("Error1b")
+      console.log(responseWithSkipTlsVerification.body)
       expect(responseWithSkipTlsVerification.statusCode).toBe(200);
       if (!("data" in responseWithSkipTlsVerification.body)) {
         throw new Error("Expected response body to have 'data' property");

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -79,12 +79,11 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engi
         url: meta.url,
         engine: "chrome-cdp",
         instantReturn: true,
-
+        skipTlsVerification: meta.options.skipTlsVerification,
         headers: meta.options.headers,
         ...(actions.length > 0 ? ({
             actions,
         }) : {}),
-
         priority: meta.internalOptions.priority,
         geolocation: meta.options.geolocation,
         mobile: meta.options.mobile,

diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -30,7 +30,7 @@ export type FireEngineScrapeRequestCommon = {
 
 export type FireEngineScrapeRequestChromeCDP = {
     engine: "chrome-cdp";
-
+    skipTlsVerification?: boolean;
     actions?: Action[];
     blockMedia?: true; // cannot be false
     geolocation?: { country?: string; languages?: string[]; };
@@ -74,6 +74,7 @@ export async function fireEngineScrape<Engine extends FireEngineScrapeRequestChr
             url: request.url,
         },
     }, async span => {
+        console.log(request)
         return await robustFetch(
             {
                 url: `${fireEngineURL}/scrape`,