Merge pull request #282 from satrong/improve-041712

feat: user selects page parser instead of configuring through env vars

.env.example

@@ -15,6 +15,3 @@ MILVUS_URL=http://localhost:19530
 
 # Cohere API Key - Reranking
 COHERE_API_KEY=
-
-# Jina.ai reader https://jina.ai/reader/#apiform
-NUXT_JINA_READER=false

.vscode/settings.json

@@ -10,6 +10,7 @@
   "cSpell.words": [
     "dexie",
     "groq",
+    "jina",
     "knowledgebase",
     "nuxt"
   ],

components/KnowledgeBaseForm.vue

@@ -51,10 +51,15 @@ const state = reactive({
   name: props.data?.name || '',
   embedding: props.data?.embedding || undefined,
   description: props.data?.description || '',
-  urls: ''
+  urls: '',
+  pageParser: 'cheerio' as 'cheerio' | 'jinaReader'
 })
 const loading = ref(false)
 const isModify = computed(() => props.type === 'update')
+const parserList = [
+  { label: 'Cheerio', value: 'cheerio' },
+  { label: 'Jina Reader', value: 'jinaReader' },
+]
 
 async function onSubmit() {
   loading.value = true
@@ -73,6 +78,7 @@ async function onSubmit() {
   formData.append("name", state.name)
   formData.append("description", state.description)
   formData.append("embedding", state.embedding)
+  formData.append("pageParser", state.pageParser)
 
   if (isModify.value) {
     formData.append('knowledgeBaseId', String(props.data!.id))
@@ -163,6 +169,10 @@ async function submit(formData: FormData) {
           <UTextarea v-model="state.urls" autoresize :maxrows="6" placeholder="One per line" />
         </UFormGroup>
 
+        <UFormGroup label="URL page parser" name="pageParser" class="mb-4">
+          <USelectMenu v-model="state.pageParser" :options="parserList" value-attribute="value" />
+        </UFormGroup>
+
         <div class="flex justify-end">
           <UButton color="gray" class="mr-2" @click="onClose()">Cancel</UButton>
           <UButton type="submit" :loading="loading">

nuxt.config.ts

@@ -43,9 +43,4 @@ export default defineNuxtConfig({
       ]
     }
   },
-  runtimeConfig: {
-    jina: {
-      reader: false
-    }, // can be overridden by NUXT_JINA_READER environment variable
-  }
 })

server/types/index.ts

@@ -1,10 +1,13 @@
 import { MultiPartData } from 'h3'
 
+export type PageParser = 'cheerio' | 'jinaReader'
+
 export type KnowledgeBaseFormData = {
   name: string
   description: string
   embedding: string
   knowledgeBaseId: number | null
   uploadedFiles: MultiPartData[]
   urls: string[]
+  pageParser: PageParser
 }

server/utils/http.ts

@@ -1,5 +1,5 @@
 import { MultiPartData, type H3Event } from 'h3'
-import { KnowledgeBaseFormData } from '@/server/types'
+import type { KnowledgeBaseFormData, PageParser } from '@/server/types'
 
 export const parseKnowledgeBaseFormRequest = async (event: H3Event): Promise<KnowledgeBaseFormData> => {
   const items = await readMultipartFormData(event)
@@ -10,6 +10,7 @@ export const parseKnowledgeBaseFormRequest = async (event: H3Event): Promise<Kno
   let _name = ''
   let _description = ''
   let _embedding = ''
+  let _pageParser: PageParser = 'cheerio'
   const urls: string[] = []
   const _knowledgeBaseId = event?.context?.params?.id
   items?.forEach((item) => {
@@ -32,6 +33,9 @@ export const parseKnowledgeBaseFormRequest = async (event: H3Event): Promise<Kno
     if (key === 'embedding') {
       _embedding = decodedData
     }
+    if (key === 'pageParser') {
+      _pageParser = decodedData as PageParser
+    }
   })
 
   const formData: KnowledgeBaseFormData = {
@@ -40,7 +44,8 @@ export const parseKnowledgeBaseFormRequest = async (event: H3Event): Promise<Kno
     embedding: _embedding,
     knowledgeBaseId: _knowledgeBaseId ? parseInt(_knowledgeBaseId) : null,
     uploadedFiles,
-    urls: urls
+    urls: urls,
+    pageParser: _pageParser,
   }
 
   return formData

server/utils/rag.ts

@@ -8,6 +8,7 @@ import { RecursiveUrlLoader } from "langchain/document_loaders/web/recursive_url
 import { compile } from "html-to-text"
 import { MultiPartData, H3Event } from 'h3'
 import { createRetriever } from '@/server/retriever'
+import type { PageParser } from '@/server/types'
 
 export const loadDocuments = async (file: MultiPartData) => {
   const Loaders = {
@@ -27,16 +28,19 @@ export const loadDocuments = async (file: MultiPartData) => {
   return new Loaders[ext](blob).load()
 }
 
-export const loadURL = async (url: string, jinaReader: boolean) => {
+export const loadURL = async (url: string, pageParser: PageParser) => {
   console.log("URL: ", url)
-  if (jinaReader) {
+  if (pageParser === 'jinaReader') {
     console.log("Using Jina reader to load URL")
     const jinaUrl = `https://r.jina.ai/${url}`
     const response = await fetch(jinaUrl)
     const data = await response.text()
-    return [new Document({
-      pageContent: data
-    })]
+    return [
+      new Document({
+        pageContent: data,
+        metadata: { source: url }
+      })
+    ]
   } else {
     /*console.log("Using CheerioWebBaseLoader to load URL")
     const loader = new CheerioWebBaseLoader(url)
@@ -84,10 +88,10 @@ export const ingestURLs = async (
   event: H3Event
 ) => {
   const docs = []
-  const config = useRuntimeConfig(event)
+  const { pageParser } = await parseKnowledgeBaseFormRequest(event)
 
   for (const url of urls) {
-    const loadedDocs = await loadURL(url, config?.jina?.reader)
+    const loadedDocs = await loadURL(url, pageParser)
     docs.push(...loadedDocs)
   }