Merge pull request #425 from LlmKira/dev

telegram: super-long text support

app/receiver/telegram/__init__.py

@@ -3,12 +3,14 @@
 # @Author  : sudoskys
 # @File    : __init__.py.py
 # @Software: PyCharm
+import asyncio
 from typing import List
 
 import telebot
+import telegramify_markdown
 from loguru import logger
 from telebot.async_telebot import AsyncTeleBot
-from telegramify_markdown import convert
+from telegramify_markdown import markdownify, customize
 
 from app.middleware.llm_task import OpenaiMiddleware
 from app.receiver import function
@@ -23,6 +25,8 @@
 
 from llmkira.task.schema import Location, EventMessage
 
+customize.latex_escape = True
+
 
 class TelegramSender(BaseSender):
     """
@@ -107,7 +111,7 @@ async def forward(
                 continue
             await self.bot.send_message(
                 chat_id=receiver.chat_id,
-                text=convert(item.text),
+                text=markdownify(item.text),
                 reply_to_message_id=receiver.message_id,
                 parse_mode="MarkdownV2",
                 disable_web_page_preview=True,
@@ -142,19 +146,44 @@ async def reply(
             if not event.text:
                 continue
             try:
-                await self.bot.send_message(
-                    chat_id=receiver.chat_id,
-                    text=convert(event.text),
-                    reply_to_message_id=receiver.message_id
-                    if reply_to_message
-                    else None,
-                    parse_mode="MarkdownV2",
-                    disable_web_page_preview=True,
+                cell_stack = telegramify_markdown.telegramify(
+                    content=event.text, max_word_count=4050
                 )
+                for cell in cell_stack:
+                    if cell.content_type == telegramify_markdown.ContentTypes.TEXT:
+                        await self.bot.send_message(
+                            chat_id=receiver.chat_id,
+                            text=cell.content,
+                            reply_to_message_id=receiver.message_id
+                            if reply_to_message
+                            else None,
+                            parse_mode="MarkdownV2",
+                            disable_web_page_preview=True,
+                        )
+                    elif cell.content_type == telegramify_markdown.ContentTypes.PHOTO:
+                        await self.bot.send_photo(
+                            chat_id=receiver.chat_id,
+                            photo=(cell.file_name, cell.file_data),
+                            reply_to_message_id=receiver.message_id
+                            if reply_to_message
+                            else None,
+                            caption=cell.caption,
+                        )
+                    elif cell.content_type == telegramify_markdown.ContentTypes.FILE:
+                        await self.bot.send_document(
+                            chat_id=receiver.chat_id,
+                            document=(cell.file_name, cell.file_data),
+                            reply_to_message_id=receiver.message_id
+                            if reply_to_message
+                            else None,
+                        )
+                    else:
+                        raise ValueError(f"Unknown content type {cell.content_type}")
+                    await asyncio.sleep(3)
             except telebot.apihelper.ApiTelegramException as e:
                 if "message to reply not found" in str(e):
                     await self.bot.send_message(
-                        chat_id=receiver.chat_id, text=convert(event.text)
+                        chat_id=receiver.chat_id, text=markdownify(event.text)
                     )
                 else:
                     logger.error(f"User {receiver.user_id} send message error")
@@ -164,7 +193,7 @@ async def reply(
     async def error(self, receiver: Location, text):
         await self.bot.send_message(
             chat_id=receiver.chat_id,
-            text=convert(text),
+            text=markdownify(text),
             reply_to_message_id=receiver.message_id,
             parse_mode="MarkdownV2",
         )

app/sender/discord/__init__.py

@@ -14,7 +14,7 @@
 from hikari.impl import ProxySettings
 from loguru import logger
 from telebot import formatting
-from telegramify_markdown import convert
+from telegramify_markdown import markdownify
 
 from app.setting.discord import BotSetting
 from llmkira.kv_manager.env import EnvManager
@@ -274,7 +274,7 @@ async def listen_learn_command(ctx: crescent.Context, instruction: str):
             reply = await learn_instruction(
                 uid=uid_make(__sender__, ctx.user.id), instruction=instruction
             )
-            return await ctx.respond(content=convert(reply), ephemeral=True)
+            return await ctx.respond(content=markdownify(reply), ephemeral=True)
 
         @client.include
         @crescent.command(
@@ -398,7 +398,7 @@ async def listen_tool_command(ctx: crescent.Context):
                 f"# {tool_item.name}\n{tool_item.get_function_string}\n```{tool_item.usage}```"
                 for tool_item in _tool
             ]
-            reply_message_text = convert("\n".join(_paper))
+            reply_message_text = markdownify("\n".join(_paper))
             await ctx.respond(
                 ephemeral=True,
                 content=reply_message_text,
@@ -418,7 +418,7 @@ async def listen_env_command(ctx: crescent.Context, env_string: str):
                     "**🧊 Env parse failed...O_o**\n", separator="\n"
                 )
             else:
-                text = convert(dict2markdown(env_map))
+                text = markdownify(dict2markdown(env_map))
             await ctx.respond(
                 ephemeral=True,
                 content=text,

app/sender/kook/__init__.py

@@ -11,7 +11,7 @@
 from khl import Bot, Message, Cert, MessageTypes, PrivateMessage, PublicMessage
 from loguru import logger
 from telebot import formatting
-from telegramify_markdown import convert
+from telegramify_markdown import markdownify
 
 from app.setting.kook import BotSetting
 from llmkira.kv_manager.env import EnvManager
@@ -287,7 +287,7 @@ async def listen_learn_command(
                 instruction=instruction,
             )
             return await msg.reply(
-                content=convert(reply),
+                content=markdownify(reply),
                 is_temp=True,
                 type=MessageTypes.KMD,
             )
@@ -336,7 +336,7 @@ async def listen_login_command(
         async def listen_logout_command(msg: Message):
             reply = await logout(uid=uid_make(__sender__, msg.author_id))
             return await msg.reply(
-                content=convert(reply),
+                content=markdownify(reply),
                 is_temp=True,
                 type=MessageTypes.KMD,
             )
@@ -380,7 +380,7 @@ async def listen_tool_command(msg: Message):
                 f"# {tool_item.name}\n{tool_item.get_function_string}\n```{tool_item.usage}```"
                 for tool_item in _tool
             ]
-            reply_message_text = convert("\n".join(_paper))
+            reply_message_text = markdownify("\n".join(_paper))
             await msg.reply(
                 is_temp=True,
                 type=MessageTypes.KMD,
@@ -422,7 +422,7 @@ async def listen_env_command(msg: Message, env_string: str):
                     "**🧊 Env parse failed...O_o**\n", separator="\n"
                 )
             else:
-                text = convert(dict2markdown(env_map))
+                text = markdownify(dict2markdown(env_map))
             await msg.reply(
                 is_temp=True,
                 type=MessageTypes.KMD,

app/sender/slack/__init__.py

@@ -15,7 +15,7 @@
 from slack_bolt.context.respond.async_respond import AsyncRespond
 from slack_sdk.web.async_client import AsyncWebClient
 from telebot import formatting
-from telegramify_markdown import convert
+from telegramify_markdown import markdownify
 
 from app.sender.util_func import (
     is_command,
@@ -266,7 +266,7 @@ async def listen_env_command(ack: AsyncAck, respond: AsyncRespond, command):
             _manager = EnvManager(user_id=uid_make(__sender__, command.user_id))
             if not command.text:
                 env_map = await _manager.read_env()
-                text = convert(dict2markdown(env_map))
+                text = markdownify(dict2markdown(env_map))
                 return await respond(text=text)
             _arg = command.text
             try:
@@ -277,7 +277,7 @@ async def listen_env_command(ack: AsyncAck, respond: AsyncRespond, command):
                 logger.exception(f"[213562]env update failed {e}")
                 text = formatting.mbold("🧊 Failed")
             else:
-                text = convert(dict2markdown(env_map))
+                text = markdownify(dict2markdown(env_map))
             await respond(text=text)
 
         @bot.command(command="/clear")
@@ -310,7 +310,7 @@ async def listen_tool_command(ack: AsyncAck, respond: AsyncRespond, command):
                 f"# {tool_item.name}\n{tool_item.get_function_string}\n```{tool_item.usage}```"
                 for tool_item in _tool
             ]
-            reply_message_text = convert("\n".join(_paper))
+            reply_message_text = markdownify("\n".join(_paper))
             return await respond(text=reply_message_text)
 
         async def auth_chain(uuid, user_id):

app/sender/telegram/__init__.py

@@ -11,7 +11,7 @@
 from telebot import types
 from telebot.async_telebot import AsyncTeleBot
 from telebot.asyncio_storage import StateMemoryStorage
-from telegramify_markdown import convert
+from telegramify_markdown import markdownify
 
 from app.sender.util_func import (
     parse_command,
@@ -279,7 +279,7 @@ async def listen_env_command(message: types.Message):
                 env_map = await _manager.read_env()
                 return await bot.reply_to(
                     message,
-                    text=convert(dict2markdown(env_map)),
+                    text=markdownify(dict2markdown(env_map)),
                     parse_mode="MarkdownV2",
                 )
             try:
@@ -292,7 +292,7 @@ async def listen_env_command(message: types.Message):
                     formatting.mbold("🧊 Failed"), separator="\n"
                 )
             else:
-                text = convert(dict2markdown(env_map))
+                text = markdownify(dict2markdown(env_map))
             await bot.reply_to(message, text=text, parse_mode="MarkdownV2")
 
         @bot.message_handler(
@@ -319,7 +319,7 @@ async def listen_help_command(message: types.Message):
             _message = await bot.reply_to(
                 message,
                 text=formatting.format_text(
-                    telegramify_markdown.convert(help_message()),
+                    telegramify_markdown.markdownify(help_message()),
                     separator="\n",
                 ),
                 parse_mode="MarkdownV2",
@@ -338,7 +338,7 @@ async def listen_tool_command(message: types.Message):
             if len(reply_message_text) > 4096:
                 reply_message_text = reply_message_text[:4096]
             return await bot.reply_to(
-                message, text=convert(reply_message_text), parse_mode="MarkdownV2"
+                message, text=markdownify(reply_message_text), parse_mode="MarkdownV2"
             )
 
         @bot.message_handler(
@@ -364,7 +364,7 @@ async def listen_auth_command(message: types.Message):
                     auth_result = "🪄 Snapshot released"
                 else:
                     auth_result = "You dont have this snapshot"
-            return await bot.reply_to(message, text=convert(auth_result))
+            return await bot.reply_to(message, text=markdownify(auth_result))
 
         @bot.message_handler(
             content_types=["text", "photo", "document"], chat_types=["private"]

app/sender/util_func.py

@@ -167,7 +167,7 @@ async def login(uid: str, arg_string) -> str:
     :param arg_string: input string
     :return: str message
     """
-    error = telegramify_markdown.convert(
+    error = telegramify_markdown.markdownify(
         "🔑 **Incorrect format.**\n"
         "You can set it via `/login https://<something api.openai.com>/v1$<api key>"
         "$<model such as gpt-4-turbo>$<tool_model such as gpt-4o-mini>` format, "
@@ -183,16 +183,16 @@ async def login(uid: str, arg_string) -> str:
                 token=settings[0], provider_url=settings[1]
             )
         except ProviderError as e:
-            return telegramify_markdown.convert(f"Login failed, website return {e}")
+            return telegramify_markdown.markdownify(f"Login failed, website return {e}")
         except Exception as e:
             logger.error(f"Login failed {e}")
-            return telegramify_markdown.convert(f"Login failed, because {type(e)}")
+            return telegramify_markdown.markdownify(f"Login failed, because {type(e)}")
         else:
             await save_credential(
                 uid=uid,
                 credential=credential,
             )
-            return telegramify_markdown.convert(
+            return telegramify_markdown.markdownify(
                 "Login success as provider! Welcome master!"
             )
     elif len(settings) == 3 or len(settings) == 4:
@@ -213,7 +213,7 @@ async def login(uid: str, arg_string) -> str:
             uid=uid,
             credential=credential,
         )
-        return telegramify_markdown.convert(
+        return telegramify_markdown.markdownify(
             f"Login success as {settings[2]}! Welcome master!"
         )
     else:
@@ -229,7 +229,7 @@ async def logout(uid: str) -> str:
     user = await USER_MANAGER.read(user_id=uid)
     user.credential = None
     await USER_MANAGER.save(user_model=user)
-    return telegramify_markdown.convert("Logout success! Welcome back master!")
+    return telegramify_markdown.markdownify("Logout success! Welcome back master!")
 
 
 class TimerObjectContainer:

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     "pytelegrambotapi<5.0.0,>=4.14.0",
     "ffmpeg-python<1.0.0,>=0.2.0",
     "duckduckgo-search>=6.2.0",
-    "telegramify-markdown>=0.1.2",
+    "telegramify-markdown>=0.2.0",
     "json-repair>=0.13.0",
     "curl-cffi>=0.6.2",
     "deprecated>=1.2.14",