Upgrade llama-index

.github/workflows/ci.yml

@@ -8,6 +8,10 @@ on:
       - feature
       - "releases/**"
 
+concurrency:
+  group: pairag-ci-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
   pull-requests: write

.github/workflows/main.yml

@@ -8,6 +8,10 @@ on:
       - feature
       - "releases/**"
 
+concurrency:
+  group: pr-build-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
   pull-requests: write

.github/workflows/main_gpu.yml

@@ -8,6 +8,10 @@ on:
       - feature
       - "releases/**"
 
+concurrency:
+  group: pr-build-gpu-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
   pull-requests: write

pyproject.toml

@@ -13,7 +13,7 @@ readme = "README.md"
 python = ">=3.11.0,<3.12"
 fastapi = "0.115.2"
 uvicorn = "^0.29.0"
-llama-index-core = "^0.12.2"
+llama-index-core = "0.12.11"
 llama-index-embeddings-openai = "^0.3.0"
 llama-index-embeddings-azure-openai = "^0.3.0"
 llama-index-embeddings-dashscope = "^0.3.0"
@@ -31,7 +31,6 @@ llama-index-readers-file = "^0.4.3"
 llama-index-postprocessor-flag-embedding-reranker = "^0.3.0"
 llama-index-storage-chat-store-redis = "^0.4.0"
 llama-index-tools-google = "^0.3.0"
-llama-index-tools-duckduckgo = "^0.3.0"
 llama-index-llms-paieas = "^0.3.0"
 llama-index-agent-openai = "^0.4.1"
 llama-index-multi-modal-llms-openai = "^0.4.2"
@@ -59,14 +58,7 @@ torchvision = [
 ]
 transformers = "^4.42.4"
 openpyxl = "^3.1.2"
-pdf2image = "^1.17.0"
-python-bidi = "0.4.2"
-easyocr = "^1.7.1"
-opencv-python = "^4.6.0.66"
 llama-parse = "0.4.2"
-pypdf2 = "^3.0.1"
-pdfplumber = "^0.11.0"
-pdfminer-six = "^20231228"
 openinference-semantic-conventions = "^0.1.9"
 openinference-instrumentation = "^0.1.12"
 pytest-asyncio = "^0.23.7"
@@ -85,16 +77,14 @@ pre-commit = "^3.8.0"
 cn-clip = "^1.5.1"
 pymysql = "^1.1.1"
 milvus-lite = "^2.4.9"
-rapidocr-onnxruntime = "^1.3.24"
-rapid-table = "^0.1.3"
 bs4 = "^0.0.2"
 httpx = "^0.27.0"
 detectron2 = [
   {markers = "sys_platform == 'linux'", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/detectron2-0.6%2B864913fpt2.3.0cpu-cp311-cp311-linux_x86_64.whl"},
   {markers = "sys_platform == 'win32'", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/detectron2-0.6%2B864913fpt2.3.0cpu-cp311-cp311-win_amd64.whl"},
   {markers = "sys_platform != 'win32' and sys_platform != 'linux' ", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/detectron2-0.6%2B864913fpt2.2.2cpu-cp311-cp311-macosx_10_9_universal2.whl"}
 ]
-magic-pdf = {version = "0.10.6", extras = ["full"]}
+magic-pdf = {version = "1.0.1", extras = ["full"]}
 peft = "^0.12.0"
 duckduckgo-search = "^6.2.12"
 aliyun-bootstrap = "^1.0.2"

pyproject_gpu.toml

@@ -13,7 +13,7 @@ readme = "README.md"
 python = ">=3.11.0,<3.12"
 fastapi = "0.115.2"
 uvicorn = "^0.29.0"
-llama-index-core = "^0.12.2"
+llama-index-core = "^0.12.11"
 llama-index-embeddings-openai = "^0.3.0"
 llama-index-embeddings-azure-openai = "^0.3.0"
 llama-index-embeddings-dashscope = "^0.3.0"
@@ -30,7 +30,6 @@ llama-index-embeddings-huggingface = "^0.4.0"
 llama-index-postprocessor-flag-embedding-reranker = "^0.3.0"
 llama-index-storage-chat-store-redis = "^0.4.0"
 llama-index-tools-google = "^0.3.0"
-llama-index-tools-duckduckgo = "^0.3.0"
 llama-index-multi-modal-llms-openai = "^0.4.2"
 llama-index-vector-stores-alibabacloud-opensearch = "^0.3.0"
 llama-index-llms-paieas = "^0.3.0"
@@ -52,14 +51,7 @@ torch = "2.2.2"
 torchvision = "0.17.2"
 transformers = "4.42.4"
 openpyxl = "^3.1.2"
-pdf2image = "^1.17.0"
-python-bidi = "0.4.2"
-easyocr = "^1.7.1"
-opencv-python = "^4.6.0.66"
 llama-parse = "0.4.2"
-pypdf2 = "^3.0.1"
-pdfplumber = "^0.11.0"
-pdfminer-six = "^20231228"
 openinference-semantic-conventions = "^0.1.9"
 openinference-instrumentation = "^0.1.12"
 pytest-asyncio = "^0.23.7"
@@ -78,16 +70,14 @@ pre-commit = "^3.8.0"
 cn-clip = "^1.5.1"
 pymysql = "^1.1.1"
 milvus-lite = "^2.4.9"
-rapidocr-onnxruntime = "^1.3.24"
-rapid-table = "^0.1.3"
 bs4 = "^0.0.2"
 httpx = "0.27.0"
 detectron2 = [
   {markers = "sys_platform == 'linux'", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/detectron2-0.6%2B864913fpt2.3.0cu121-cp311-cp311-linux_x86_64.whl"},
   {markers = "sys_platform == 'win32'", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/detectron2-0.6%2B864913fpt2.3.0cu121-cp311-cp311-win_amd64.whl"},
   {markers = "sys_platform != 'win32' and sys_platform != 'linux' ", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/detectron2-0.6%2B864913fpt2.2.2cpu-cp311-cp311-macosx_10_9_universal2.whl"}
 ]
-magic-pdf = {version = "0.10.6", extras = ["full"]}
+magic-pdf = {version = "1.0.1", extras = ["full"]}
 paddlepaddle-gpu = [
   {markers = "sys_platform == 'linux'", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/paddlepaddle_gpu-3.0.0b1-cp311-cp311-linux_x86_64.whl"},
   {markers = "sys_platform != 'linux'", url = "https://pai-rag.oss-cn-hangzhou.aliyuncs.com/packages/python_wheels/paddlepaddle_gpu-3.0.0b1-cp311-cp311-win_amd64.whl"}

src/pai_rag/app/web/event_listeners.py

@@ -209,7 +209,8 @@ def save_config(input_elements: List[Any]):
                 value=input_oss_ak_sk(value_sk), type="text" if value_sk else "password"
             ),
             gr.update(
-                value=f"[{datetime.datetime.now()}] Snapshot configuration saved successfully!"
+                value=f"[{datetime.datetime.now()}] Snapshot configuration saved successfully!",
+                visible=True,
             ),
         ]
     except RagApiError as api_error:

src/pai_rag/app/web/tabs/settings_tab.py

@@ -317,7 +317,9 @@ def create_setting_tab() -> Dict[str, Any]:
             )
 
             save_btn = gr.Button("Save Llm Setting", variant="primary")
-            save_state = gr.Textbox(label="Connection Info: ", container=False)
+            save_state = gr.Textbox(
+                label="Connection Info: ", container=False, visible=False
+            )
             save_btn.click(
                 fn=ev_listeners.save_config,
                 inputs=set(llm_components),

src/pai_rag/integrations/nodeparsers/pai/pai_markdown_parser.py

@@ -76,7 +76,7 @@ def _format_tree_nodes(
                 image_url=node.content,
                 excluded_embed_metadata_keys=doc_node.excluded_embed_metadata_keys,
                 excluded_llm_metadata_keys=doc_node.excluded_llm_metadata_keys,
-                metadata_seperator=doc_node.metadata_seperator,
+                metadata_separator=doc_node.metadata_separator,
                 metadata_template=doc_node.metadata_template,
                 text_template=doc_node.text_template,
                 metadata={
@@ -110,7 +110,7 @@ def _create_text_node(
                 embedding=doc_node.embedding,
                 excluded_embed_metadata_keys=doc_node.excluded_embed_metadata_keys,
                 excluded_llm_metadata_keys=doc_node.excluded_llm_metadata_keys,
-                metadata_seperator=doc_node.metadata_seperator,
+                metadata_separator=doc_node.metadata_separator,
                 metadata_template=doc_node.metadata_template,
                 text_template=doc_node.text_template,
                 metadata={
@@ -125,7 +125,7 @@ def _create_text_node(
                 embedding=doc_node.embedding,
                 excluded_embed_metadata_keys=doc_node.excluded_embed_metadata_keys,
                 excluded_llm_metadata_keys=doc_node.excluded_llm_metadata_keys,
-                metadata_seperator=doc_node.metadata_seperator,
+                metadata_separator=doc_node.metadata_separator,
                 metadata_template=doc_node.metadata_template,
                 text_template=doc_node.text_template,
                 meta_data=doc_node.extra_info,
@@ -237,7 +237,7 @@ def traverse_tree(
                             image_url=child.content,
                             excluded_embed_metadata_keys=doc_node.excluded_embed_metadata_keys,
                             excluded_llm_metadata_keys=doc_node.excluded_llm_metadata_keys,
-                            metadata_seperator=doc_node.metadata_seperator,
+                            metadata_separator=doc_node.metadata_separator,
                             metadata_template=doc_node.metadata_template,
                             text_template=doc_node.text_template,
                             metadata={

src/pai_rag/integrations/readers/pai_pdf_reader.py

@@ -12,6 +12,7 @@
 import tempfile
 from PIL import Image
 import os
+import traceback
 from loguru import logger
 from magic_pdf.data.dataset import PymuDocDataset
 from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze
@@ -294,9 +295,9 @@ def parse_pdf(
 
             return md_content
 
-        except Exception as e:
-            logger.error(e)
-            return None
+        except Exception:
+            logger.error(traceback.format_exc())
+            raise
 
     def load_data(
         self,

tests/integrations/test_nl2pandas_retriever.py

@@ -62,7 +62,7 @@ def test_data_analysis_synthesizer():
                 end_char_idx=None,
                 text_template="{metadata_str}\n\n{content}",
                 metadata_template="{key}: {value}",
-                metadata_seperator="\n",
+                metadata_separator="\n",
             ),
             score=1.0,
         )