Fix/incorrect pingqna status

packages/scraper-strategies/README.md

@@ -0,0 +1,3 @@
+# scraper-strategies
+
+Request stategies for [@qnaplus/scraper](https://github.com/qnaplus/scraper)

packages/scraper-strategies/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@qnaplus/scraper-strategies",
-	"version": "3.1.0",
+	"version": "3.1.1",
 	"description": "Request stategies for @qnaplus/scraper",
 	"main": "dist/index.js",
 	"repository": {
@@ -15,7 +15,7 @@
 		"reset": "pnpm clean && rimraf node_modules"
 	},
 	"dependencies": {
-		"@qnaplus/node-curl-impersonate": "^1.0.0",
+		"@qnaplus/node-curl-impersonate": "^1.0.2",
 		"@qnaplus/scraper": "workspace:*",
 		"cycletls": "^1.0.27",
 		"form-data": "^4.0.1",

packages/scraper-strategies/src/curl_impersonate_client.ts

@@ -12,44 +12,62 @@ import {
 import { FetchClient, type FetchClientResponse } from "@qnaplus/scraper";
 
 export class CurlImpersonateScrapingClient extends FetchClient<FetchClientResponse> {
+	async ping(url: string): Promise<boolean> {
+		let ok: boolean | undefined = undefined;
+		await this.forEachPreset(async (preset) => {
+			if (ok !== undefined) {
+				return;
+			}
+			const response = await new RequestBuilder()
+				.url(url)
+				.preset(preset)
+				.send();
+			if (response.stderr !== undefined) {
+				throw new Error(response.stderr); // TODO figure out how to handle this
+			}
+			ok = response.details.response_code === 200;
+		});
+		return ok ?? false;
+	}
+
 	async fetch(url: string): Promise<FetchClientResponse> {
-		const browsers = getCompatibleBrowsers();
-		const badStatusCodes = [403];
+		const badStatusCodes = [-1, 403];
 		let latestResponse: FetchClientResponse = {
 			ok: false,
 			body: "",
 			status: -1,
 			url: "",
 		};
-
-		// TODO: when a bad status code is not recieved, cache the preset that was used.
-		// Continue using that preset until failure, then search again for another working preset.
-		for (const browser of browsers) {
-			for (const version of Object.keys(BrowserPresets[browser.name])) {
-				const { response, details } = await this.doPresetRequest(url, {
-					name: browser.name,
-					version: version as
-						| ChromePresetVersion
-						| FirefoxPresetVersion
-						| SafariPresetVersion,
-				});
-				latestResponse = {
-					body: response,
-					ok: details.response_code === 200,
-					url: details.url_effective,
-					status: details.response_code,
-				};
-				if (!badStatusCodes.includes(details.response_code)) {
-					return latestResponse;
-				}
+		await this.forEachPreset(async (preset) => {
+			if (!badStatusCodes.includes(latestResponse.status)) {
+				return;
+			}
+			const { response, details } = await this.doPresetRequest(url, {
+				name: preset.name,
+				version: preset.version as
+					| ChromePresetVersion
+					| FirefoxPresetVersion
+					| SafariPresetVersion,
+			});
+			latestResponse = {
+				body: response,
+				ok: details.response_code === 200,
+				url: details.url_effective,
+				status: details.response_code,
+			};
+			if (badStatusCodes.includes(latestResponse.status)) {
 				this.logger?.trace(
-					`Request did not return an accepted response code (preset: ${browser.name} v${version})`,
+					`Request did not return an accepted response code (preset: ${preset.name} v${preset.version})`,
 				);
 			}
+		});
+		// TODO: when a bad status code is not recieved, cache the preset that was used.
+		// Continue using that preset until failure, then search again for another working preset.
+		if (badStatusCodes.includes(latestResponse.status)) {
+			this.logger?.trace(
+				`All presets failed (latest response: ${latestResponse}`,
+			);
 		}
-		this.logger?.trace(
-			`All presets failed (latest response: ${latestResponse}`,
-		);
 		return latestResponse;
 	}
 
@@ -60,6 +78,23 @@ export class CurlImpersonateScrapingClient extends FetchClient<FetchClientRespon
 
 	teardown(): Promise<void> | void {}
 
+	private async forEachPreset(
+		cb: (preset: RequestPreset<BrowserType>) => void | Promise<void>,
+	) {
+		const browsers = getCompatibleBrowsers();
+		for (const browser of browsers) {
+			for (const version of Object.keys(BrowserPresets[browser.name])) {
+				await cb({
+					name: browser.name,
+					version: version as
+						| ChromePresetVersion
+						| FirefoxPresetVersion
+						| SafariPresetVersion,
+				});
+			}
+		}
+	}
+
 	private async doPresetRequest<T extends BrowserType>(
 		url: string,
 		preset: RequestPreset<T>,

packages/scraper/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@qnaplus/scraper",
-	"version": "3.1.0",
+	"version": "3.1.1",
 	"description": "Utility package to scrape questions from the VEX Robotics Q&A.",
 	"main": "dist/index.js",
 	"repository": {
@@ -19,13 +19,13 @@
 		"@crawlee/core": "^3.11.5",
 		"cheerio": "^1.0.0",
 		"got-scraping": "^4.0.6",
-		"node-pdf-parser": "^1.0.3",
+		"node-pdf-parser": "^1.0.7",
 		"pino": "catalog:"
 	},
 	"devDependencies": {
 		"@qnaplus/typescript-config": "workspace:1.0.0",
 		"pino-pretty": "catalog:",
-		"vitest": "^3.0.8"
+		"vitest": "^3.1.2"
 	},
 	"volta": {
 		"extends": "../../package.json"