Add support for 'External Sources'

Author: ztpn

src/dev-cli/index.ts

@@ -7,7 +7,7 @@ import { prompt } from 'enquirer';
 import { runScraper } from '@/dev-cli/scraper';
 import { processOptions } from '@/dev-cli/validate';
 
-import { getBuiltinEmbeds, getBuiltinSources } from '..';
+import { getBuiltinEmbeds, getBuiltinExternalSources, getBuiltinSources } from '..';
 
 dotenv.config();
 
@@ -30,7 +30,7 @@ type ShowAnswers = {
   episode: string;
 };
 
-const sourceScrapers = getBuiltinSources().sort((a, b) => b.rank - a.rank);
+const sourceScrapers = [...getBuiltinSources(), ...getBuiltinExternalSources()].sort((a, b) => b.rank - a.rank);
 const embedScrapers = getBuiltinEmbeds().sort((a, b) => b.rank - a.rank);
 const sources = [...sourceScrapers, ...embedScrapers];
 

src/dev-cli/validate.ts

@@ -82,6 +82,7 @@ export async function processOptions(sources: Array<Embed | Sourcerer>, options:
     fetcher,
     target: targets.ANY,
     consistentIpForRequests: true,
+    externalSources: 'all',
   };
 
   return {

src/entrypoint/builder.ts

@@ -1,5 +1,5 @@
 import { ProviderControls, makeControls } from '@/entrypoint/controls';
-import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
+import { getBuiltinEmbeds, getBuiltinExternalSources, getBuiltinSources } from '@/entrypoint/providers';
 import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets';
 import { Fetcher } from '@/fetchers/types';
 import { Embed, Sourcerer } from '@/providers/base';
@@ -26,6 +26,7 @@ export function buildProviders(): ProviderBuilder {
   const embeds: Embed[] = [];
   const sources: Sourcerer[] = [];
   const builtinSources = getBuiltinSources();
+  const builtinExternalSources = getBuiltinExternalSources();
   const builtinEmbeds = getBuiltinEmbeds();
 
   return {
@@ -51,7 +52,7 @@ export function buildProviders(): ProviderBuilder {
         return this;
       }
 
-      const matchingSource = builtinSources.find((v) => v.id === input);
+      const matchingSource = [...builtinSources, ...builtinExternalSources].find((v) => v.id === input);
       if (!matchingSource) throw new Error('Source not found');
       sources.push(matchingSource);
       return this;

src/entrypoint/declare.ts

@@ -1,5 +1,5 @@
 import { makeControls } from '@/entrypoint/controls';
-import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
+import { getBuiltinEmbeds, getBuiltinExternalSources, getBuiltinSources } from '@/entrypoint/providers';
 import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets';
 import { Fetcher } from '@/fetchers/types';
 import { getProviders } from '@/providers/get';
@@ -19,6 +19,9 @@ export interface ProviderMakerOptions {
   // the device that the stream will be played on
   consistentIpForRequests?: boolean;
 
+  // used to add built in sources which aren't used by default aka external sources
+  externalSources?: 'all' | string[];
+
   // This is temporary
   proxyStreams?: boolean;
 }
@@ -29,9 +32,21 @@ export function makeProviders(ops: ProviderMakerOptions) {
     ops.consistentIpForRequests ?? false,
     ops.proxyStreams,
   );
+
+  const sources = [...getBuiltinSources()];
+
+  if (ops.externalSources === 'all') sources.push(...getBuiltinExternalSources());
+  else {
+    ops.externalSources?.forEach((source) => {
+      const matchingSource = getBuiltinExternalSources().find((v) => v.id === source);
+      if (!matchingSource) return;
+      sources.push(matchingSource);
+    });
+  }
+
   const list = getProviders(features, {
     embeds: getBuiltinEmbeds(),
-    sources: getBuiltinSources(),
+    sources,
   });
 
   return makeControls({

src/entrypoint/providers.ts

@@ -2,7 +2,10 @@ import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
 import { Embed, Sourcerer } from '@/providers/base';
 
 export function getBuiltinSources(): Sourcerer[] {
-  return gatherAllSources().filter((v) => !v.disabled);
+  return gatherAllSources().filter((v) => !v.disabled && !v.externalSource);
+}
+export function getBuiltinExternalSources(): Sourcerer[] {
+  return gatherAllSources().filter((v) => v.externalSource && !v.disabled);
 }
 
 export function getBuiltinEmbeds(): Embed[] {

src/index.ts

@@ -15,7 +15,7 @@ export type { SourcererOptions, EmbedOptions } from '@/providers/base';
 export { NotFoundError } from '@/utils/errors';
 export { makeProviders } from '@/entrypoint/declare';
 export { buildProviders } from '@/entrypoint/builder';
-export { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
+export { getBuiltinEmbeds, getBuiltinSources, getBuiltinExternalSources } from '@/entrypoint/providers';
 export { makeStandardFetcher } from '@/fetchers/standardFetch';
 export { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy';
 export { flags, targets } from '@/entrypoint/utils/targets';

src/providers/base.ts

@@ -19,6 +19,9 @@ export type SourcererOptions = {
   name: string; // displayed in the UI
   rank: number; // the higher the number, the earlier it gets put on the queue
   disabled?: boolean;
+  // these sources are built in but not used by default
+  // this can have many uses, we use this for sources that only work on official instances
+  externalSource?: boolean;
   flags: Flags[];
   scrapeMovie?: (input: MovieScrapeContext) => Promise<SourcererOutput>;
   scrapeShow?: (input: ShowScrapeContext) => Promise<SourcererOutput>;
@@ -27,6 +30,7 @@ export type SourcererOptions = {
 export type Sourcerer = SourcererOptions & {
   type: 'source';
   disabled: boolean;
+  externalSource: boolean;
   mediaTypes: MediaScraperTypes[];
 };
 
@@ -38,6 +42,7 @@ export function makeSourcerer(state: SourcererOptions): Sourcerer {
     ...state,
     type: 'source',
     disabled: state.disabled ?? false,
+    externalSource: state.externalSource ?? false,
     mediaTypes,
   };
 }