Merge remote-tracking branch 'upstream/main' into EAI-235

packages/ingest-mongodb-public/src/sources/index.ts

@@ -21,7 +21,6 @@ import {
 import { prismaSourceConstructor } from "./prisma";
 import { wiredTigerSourceConstructor } from "./wiredTiger";
 import { pyMongoSourceConstructor } from "./pyMongo";
-import { cppSourceConstructor } from "./cppDriver";
 import { mongooseSourceConstructor } from "./mongoose";
 import { practicalAggregationsDataSource } from "./practicalAggregations";
 
@@ -229,6 +228,12 @@ export const snootyProjectConfig: LocallySpecifiedSnootyProjectConfig[] = [
     tags: ["docs", "driver", "rust"],
     productName: "Rust Driver",
   },
+  {
+    type: "snooty",
+    name: "cpp-driver",
+    tags: ["docs", "driver", "cpp", "cxx", "c++"],
+    productName: "C++ Driver",
+  },
 ];
 
 export const devCenterProjectConfig: DevCenterProjectConfig = {
@@ -473,7 +478,6 @@ export const sourceConstructors: SourceConstructor[] = [
   pyMongoSourceConstructor,
   mongooseSourceConstructor,
   prismaSourceConstructor,
-  cppSourceConstructor,
   mongoDbCorpDataSource,
   practicalAggregationsDataSource,
   javaReactiveStreamsSourceConstructor,

packages/mongodb-rag-ingest/src/sources/MdOnGithubDataSource.test.ts

@@ -8,78 +8,79 @@ import { Page } from "mongodb-rag-core";
 
 jest.setTimeout(60000);
 
-export function samplePathToPage(pathInRepo: string) {
-  if (pathInRepo.endsWith("_index.md")) {
-    pathInRepo = pathInRepo.replace("_index.md", "index.md");
-  }
-  return pathInRepo
-    .replace(/^docs\/content\/mongocxx-v3/, "https://example/com")
-    .replace(/\.md$/, "/");
-}
-const sampleConf: MakeMdOnGithubDataSourceParams = {
-  name: "sample",
-  repoUrl: "https://github.com/mongodb/mongo-cxx-driver/",
+const baseChatbotRepoConfig: MakeMdOnGithubDataSourceParams = {
+  name: "chatbot",
+  repoUrl: "https://github.com/mongodb/chatbot",
   repoLoaderOptions: {
-    branch: "master",
-    ignoreFiles: [/^(?!^\/docs\/content\/mongocxx-v3\/).*/],
-  },
-  pathToPageUrl: samplePathToPage,
-  metadata: {
-    productName: "C++ Driver (mongocxx)",
+    branch: "main",
   },
+  pathToPageUrl: (path) => path,
+  extractMetadata: () => ({
+    foo: "bar",
+  }),
+};
+
+const mongodbCorpConfig: MakeMdOnGithubDataSourceParams = {
+  ...baseChatbotRepoConfig,
+  name: "mongodb-corp",
   frontMatter: {
     process: true,
-    separator: "+++",
-    format: "toml",
+    separator: "---",
+    format: "yaml",
+  },
+  metadata: {
+    productName: "MongoDB Corp",
   },
+  filter: (path) => path.includes("mongodb-corp"),
   extractTitle: (_, frontmatter) => (frontmatter?.title as string) ?? null,
-  extractMetadata: () => ({
-    foo: "bar",
-  }),
 };
+
+const ingestTestDataConfig: MakeMdOnGithubDataSourceParams = {
+  ...baseChatbotRepoConfig,
+  name: "ingest_testData",
+  metadata: {
+    productName: "Ingest Test Data",
+  },
+  filter: (path) => path.includes("ingest/testData"),
+};
+
 describe("MdOnGithubDataSource", () => {
   let pages: Page[];
+  const samplePages: Record<string, Page | undefined> = {};
+  const getSamplePage = (path: string) => {
+    const samplePage = samplePages[path];
+    assert(samplePage);
+    return samplePage;
+  };
   beforeAll(async () => {
-    const dataSource = await makeMdOnGithubDataSource(sampleConf);
+    const dataSource = await makeMdOnGithubDataSource(mongodbCorpConfig);
     pages = await dataSource.fetchPages();
+    samplePages["mongodb-corp/chatbot/overview.md"] = pages.find((page) => {
+      return page.url.includes("mongodb-corp/chatbot/overview.md");
+    });
   });
   it("loads and processes a real repo of markdown files", async () => {
-    const samplePage = pages.find((page) =>
-      page.title?.includes("Installing the mongocxx driver")
-    );
+    const samplePage = getSamplePage("mongodb-corp/chatbot/overview.md");
     assert(samplePage);
-    expect(samplePage?.body).toContain("install");
+    expect(samplePage?.body).toContain(
+      "The MongoDB AI is an advanced LLM-based chatbot"
+    );
   });
   it("processes metadata", () => {
-    const samplePage = pages[0];
+    const samplePage = getSamplePage("mongodb-corp/chatbot/overview.md");
     expect(samplePage.metadata).toHaveProperty("foo", "bar");
-    expect(samplePage.metadata).toHaveProperty(
-      "productName",
-      "C++ Driver (mongocxx)"
-    );
+    expect(samplePage.metadata).toHaveProperty("productName", "MongoDB Corp");
   });
   it("removes frontmatter from page body", () => {
-    const samplePage = pages[0];
-    expect(samplePage.body).not.toContain("+++");
+    const samplePage = getSamplePage("mongodb-corp/chatbot/overview.md");
+    expect(samplePage.body).not.toContain("---");
   });
   it("extracts title from frontmatter", () => {
-    const samplePage = pages[0];
+    const samplePage = getSamplePage("mongodb-corp/chatbot/overview.md");
     expect(samplePage.title).toBeTruthy();
   });
   it("works with .mdx files", async () => {
-    const sampleConf: MakeMdOnGithubDataSourceParams = {
-      name: "sample",
-      repoUrl: "https://github.com/mongodb/chatbot",
-      repoLoaderOptions: {
-        branch: "main",
-      },
-      pathToPageUrl: (path) => path,
-      metadata: {
-        productName: "C++ Driver (mongocxx)",
-      },
-      filter: (path) => path.includes("ingest/testData"),
-    };
-    const dataSource = await makeMdOnGithubDataSource(sampleConf);
+    const dataSource = await makeMdOnGithubDataSource(ingestTestDataConfig);
     const pages = await dataSource.fetchPages();
     expect(pages.length).toBeGreaterThan(1);
     expect(

packages/mongodb-rag-ingest/testData/sampleMdxFile.mdx

@@ -1 +1,19 @@
-hello mdx!
++++
+title = "Hello, MDX!"
+date = "2024-01-01T00:00:00.000Z"
++++
+
+# Hello, MDX!
+
+This is an MDX file!
+
+It's got standard markdown features like code fences:
+
+```js title="SomeComponent.js"
+  import { sum } from "./math";
+  const three = sum(1, 2);
+```
+
+<SomeComponent />
+
+It renders JSX wow!