feat(web): add dev tools add format code (#247)

Author: tpoisonooo

README_zh.md

@@ -112,7 +112,6 @@ Web 版视频教程见 [BiliBili](https://www.bilibili.com/video/BV1S2421N7mn) 
 
 如果你只有 2G 显存，或追求性价比💰，[看这个知乎文档](https://zhuanlan.zhihu.com/p/685205206)。
 
-
 # 🔥 运行
 
 我们将以 mmpose 和一些 `word`/`excel`/`pdf`/`ppt` 测试文档为例，介绍如何把知识助手部署到飞书群

docs/rag_annotate_sft_data_zh.md

@@ -2,8 +2,8 @@
 
 RAG 标注训练数据是否有用，请参考论文：
 
-* [Reformatted Alignment](https://arxiv.org/abs/2402.12219)
-* [CherryLLM](https://arxiv.org/abs/2308.12032)
+- [Reformatted Alignment](https://arxiv.org/abs/2402.12219)
+- [CherryLLM](https://arxiv.org/abs/2308.12032)
 
 本文介绍如何使用 HuixiangDou 在脏数据上标注 openmmlab+internlm 知识类问题，使用 10 个 repo（见配置文件）的文档做知识库。
 
@@ -55,10 +55,10 @@ python3 -m huixiangdou.rag --config_path config-alignment-example.ini
   }
 ```
 
-* code 是错误码，所有错误见 service/helper.py
-* direct_reply 是 LLM 原本回复，当 code 为 0 时才调用
-* reason 是错误的原因
-* refs 是参考文档
+- code 是错误码，所有错误见 service/helper.py
+- direct_reply 是 LLM 原本回复，当 code 为 0 时才调用
+- reason 是错误的原因
+- refs 是参考文档
 
 下面是个示例，`rag_reply` 更具体，不会像原始的 `direct_reply` 更“务虚”。可以进入下一阶段清洗流程。
 

huixiangdou/__init__.py

@@ -1,11 +1,11 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 """import module."""
 # only import frontend when needed, not here
-from .version import __version__
 from .service import ChatClient  # noqa E401
 from .service import ErrorCode  # noqa E401
 from .service import FeatureStore  # noqa E401
 from .service import HybridLLMServer  # noqa E401
 from .service import WebSearch  # noqa E401
 from .service import Worker  # noqa E401
 from .service import llm_serve  # noqa E401
+from .version import __version__

huixiangdou/rag.py

@@ -78,12 +78,14 @@ def rag(process_id: int, task: list, output_dir: str):
     assistant = Worker(work_dir=args.work_dir, config_path=args.config_path)
 
     # assistant.TOPIC_TEMPLATE = '告诉我这句话的关键字和主题，直接说主题不要解释：“{}”'
-    
+
     output_path = os.path.join(output_dir, 'output{}.json'.format(process_id))
     for item in task:
         query = item.query
 
-        code, response, refs = assistant.generate(query=query, history=[], groupname='')
+        code, response, refs = assistant.generate(query=query,
+                                                  history=[],
+                                                  groupname='')
 
         item.rag_reply = response
         item.code = int(code)
@@ -99,7 +101,7 @@ def rag(process_id: int, task: list, output_dir: str):
 
 
 def split_tasks(json_path: str, processes: int):
-    """Split queries for multiple prcesses"""
+    """Split queries for multiple processes."""
     queries = []
     tasks = []
     _all = []
@@ -133,7 +135,8 @@ def split_tasks(json_path: str, processes: int):
     else:
         pool = Pool(args.processes)
         for process_id in range(args.processes):
-            pool.apply_async(rag, (process_id, tasks[process_id], args.output_dir))
+            pool.apply_async(rag,
+                             (process_id, tasks[process_id], args.output_dir))
         pool.close()
         logger.debug('waiting for preprocess read finish..')
         pool.join()

huixiangdou/service/helper.py

@@ -47,7 +47,6 @@ class ErrorCode(Enum):
     SG_SEARCH_FAIL = 16, 'SourceGraph search fail, please check TOKEN'
     LLM_NOT_RESPONSE_SG = 17, 'LLM not response query with sg search'
 
-
     def __new__(cls, value, description):
         """Create new instance of ErrorCode."""
         obj = object.__new__(cls)

huixiangdou/service/llm_server_hybrid.py

@@ -291,7 +291,7 @@ def call_kimi(self, prompt, history):
             elif prompt_len <= int(32768 * 1.5) - 1024:
                 model = 'moonshot-v1-32k'
             else:
-                prompt = prompt[0: int(128000 * 1.5) -1024]
+                prompt = prompt[0:int(128000 * 1.5) - 1024]
                 model = 'moonshot-v1-128k'
 
         logger.info('choose kimi model {}'.format(model))
@@ -432,7 +432,7 @@ def generate_response(self, prompt, history=[], backend='local'):
             prompt (str): The prompt to send to the LLM.
             history (list, optional): List of previous interactions. Defaults to [].  # noqa E501
             remote (bool, optional): Flag to determine whether to use a remote server. Defaults to False.  # noqa E501
-            backend (str): LLM type to call. Support 'local', 'remote' and specfied LLM name ('kimi', 'deepseek' and so on) 
+            backend (str): LLM type to call. Support 'local', 'remote' and specified LLM name ('kimi', 'deepseek' and so on)
 
         Returns:
             str: Generated response from the LLM.

huixiangdou/service/web_search.py

@@ -220,7 +220,6 @@ def logging_search_query(self, query: str):
             f.write(query)
             f.write('\n')
 
-
     def get(self, query: str, max_article=1):
         """Executes a google search with cache.
 
@@ -230,10 +229,10 @@ def get(self, query: str, max_article=1):
         """
         query = query.strip()
         query = query[0:32]
-        
+
         try:
             self.logging_search_query(query=query)
-            
+
             articles = self.google(query=query, max_article=max_article)
             self.save_search_result(query=query, articles=articles)
             return articles, None

huixiangdou/service/worker.py

@@ -81,7 +81,7 @@ def __init__(self, work_dir: str, config_path: str, language: str = 'zh'):
             self.GENERATE_TEMPLATE = 'Background Information: "{}"\n Question: "{}"\n Please read the reference material carefully and answer the question.'  # noqa E501
 
     def direct_chat(self, query: str):
-        """"Generate reply with LLM"""
+        """"Generate reply with LLM."""
         return self.llm.generate_response(prompt=query, backend='remote')
 
     def single_judge(self, prompt, tracker, throttle: int, default: int):
@@ -219,7 +219,8 @@ def generate(self, query, history, groupname):
             search_prompt = self.llm.generate_response(prompt)
             tracker.log('search prompt', search_prompt)
 
-            articles, error = web_search.get(query=search_prompt, max_article=2)
+            articles, error = web_search.get(query=search_prompt,
+                                             max_article=2)
             if error is not None:
                 return ErrorCode.SEARCH_FAIL, response, references
 
@@ -280,9 +281,15 @@ def generate(self, query, history, groupname):
                     return ErrorCode.SG_SEARCH_FAIL, response, references
 
                 if sg_context is not None and len(sg_context) > 2:
-                    prompt, _ = self.llm.build_prompt(instruction=query, context=sg_context, history_pair=history, template=self.GENERATE_TEMPLATE)
-
-                    response = self.llm.generate_response(prompt=prompt, history=history, backend='remote')
+                    prompt, _ = self.llm.build_prompt(
+                        instruction=query,
+                        context=sg_context,
+                        history_pair=history,
+                        template=self.GENERATE_TEMPLATE)
+
+                    response = self.llm.generate_response(prompt=prompt,
+                                                          history=history,
+                                                          backend='remote')
                     tracker.log('source graph', [sg_context, response])
 
                     if response is None or len(response) < 1:

resource/rag_example_input.json

@@ -84,4 +84,4 @@
 "想请教下大佬，",
 "Better models of human high-level visual cortex emerge from natural language supervision with a large and diverse dataset\n有大佬， 可以帮忙下载一下这篇nature论文吗[可怜][可怜][可怜][可怜]",
 "大佬居然不知道？"
-]
+]

setup.py

@@ -19,6 +19,7 @@ def get_version():
         exec(compile(f.read(), version_file, 'exec'))
     return locals()['__version__']
 
+
 def read_requirements():
     lines = []
     with open('requirements.txt', 'r') as f:
@@ -31,6 +32,7 @@ def read_requirements():
                 lines.append(line)
     return lines
 
+
 install_packages = read_requirements()
 
 if __name__ == '__main__':

web/model/qalib.py

@@ -65,6 +65,6 @@ class AddDocError(BaseModel):
 
 
 class AddDocsRes(BaseModel):
-    docBase: Optional[str] = ""
+    docBase: Optional[str] = ''
     docs: Optional[List[str]] = []
     errors: Optional[List[AddDocError]] = []

web/proxy/main.py

@@ -147,7 +147,7 @@ def chat_with_featue_store(cache: CacheRetriever,
     worker = WebWorker(work_dir=workdir, config_path=configpath)
 
     history = format_history(payload.history)
-    query_log = "{} {}\n".format(fs_id, payload.content)
+    query_log = '{} {}\n'.format(fs_id, payload.content)
     with open('query.log', 'a') as f:
         f.write(query_log)
     error, response, references = worker.generate(query=payload.content,

web/proxy/web_worker.py

@@ -239,7 +239,7 @@ def generate(self, query, history, retriever, groupname):
         tracker.log('feature store doc', [chunk, response])
         if response is not None and len(response) < 1:
             # llm error
-            return ErrorCode.INTERNAL_ERROR, "LLM API 没给回复，见 https://github.com/InternLM/HuixiangDou/issues/214 ", retrieve_ref
+            return ErrorCode.INTERNAL_ERROR, 'LLM API 没给回复，见 https://github.com/InternLM/HuixiangDou/issues/214 ', retrieve_ref
 
         if response is not None and len(response) > 0:
             prompt = self.PERPLESITY_TEMPLATE.format(query, response)

web/service/agent.py

@@ -1,21 +1,25 @@
 import json
 import re
 from enum import Enum
-from typing import Union, List
+from typing import List, Union
 
 import lark_oapi as lark
 import requests
 from fastapi import Request, Response
-from lark_oapi import RawRequest, RawResponse, UTF_8, USER_AGENT, AUTHORIZATION, X_TT_LOGID, X_REQUEST_ID, CONTENT_TYPE, \
-    Content_Disposition, LARK_REQUEST_TIMESTAMP, LARK_REQUEST_NONCE, LARK_REQUEST_SIGNATURE
-from lark_oapi.api.im.v1 import GetChatRequest, P2ImMessageReceiveV1, MentionEvent, ReplyMessageRequest, \
-    ReplyMessageRequestBody, GetMessageResourceRequest
+from lark_oapi import (AUTHORIZATION, CONTENT_TYPE, LARK_REQUEST_NONCE,
+                       LARK_REQUEST_SIGNATURE, LARK_REQUEST_TIMESTAMP,
+                       USER_AGENT, UTF_8, X_REQUEST_ID, X_TT_LOGID,
+                       Content_Disposition, RawRequest, RawResponse)
+from lark_oapi.api.im.v1 import (GetChatRequest, GetMessageResourceRequest,
+                                 MentionEvent, P2ImMessageReceiveV1,
+                                 ReplyMessageRequest, ReplyMessageRequestBody)
 
 from web.config.env import HuixiangDouEnv
 from web.constant import biz_constant
-from web.model.base import standard_error_response, BaseBody
-from web.model.chat import ChatRequestBody, ChatType, LarkChatDetail, ChatQueryInfo, WechatRequest, WechatType, \
-    WechatPollItem, WechatResponse
+from web.model.base import BaseBody, standard_error_response
+from web.model.chat import (ChatQueryInfo, ChatRequestBody, ChatType,
+                            LarkChatDetail, WechatPollItem, WechatRequest,
+                            WechatResponse, WechatType)
 from web.service import qalib
 from web.service.cache import ChatCache
 from web.service.chat import ChatService
@@ -79,7 +83,7 @@ def get_event_handler(cls):
             HuixiangDouEnv.get_lark_encrypt_key(),
             HuixiangDouEnv.get_lark_verification_token(),
             lark.LogLevel.DEBUG).register_p2_im_message_receive_v1(
-            cls._on_im_message_received).build()
+                cls._on_im_message_received).build()
 
     @classmethod
     def _on_im_message_received(cls, data: P2ImMessageReceiveV1):
@@ -123,7 +127,9 @@ def _on_im_message_received(cls, data: P2ImMessageReceiveV1):
         ChatCache.mark_agent_used(app_id, ChatType.LARK)
 
         if msg.root_id or msg.parent_id:
-            logger.debug(f"[lark] app_id: {app_id}, name: {chat_name} got reply message, omit")
+            logger.debug(
+                f'[lark] app_id: {app_id}, name: {chat_name} got reply message, omit'
+            )
             return
 
         # parse lark content
@@ -142,8 +148,11 @@ def _on_im_message_received(cls, data: P2ImMessageReceiveV1):
         if len(lark_content.images) > 0:
             query_id = chat_svc.generate_query_id(lark_content.content)
             for index in range(len(lark_content.images)):
-                image_store_path = chat_svc.gen_image_store_path(query_id, str(index), ChatType.LARK)
-                if cls._store_image(client, message_id, lark_content.images[index], image_store_path):
+                image_store_path = chat_svc.gen_image_store_path(
+                    query_id, str(index), ChatType.LARK)
+                if cls._store_image(client, message_id,
+                                    lark_content.images[index],
+                                    image_store_path):
                     # replace image_key with actually store path
                     lark_content.images[index] = image_store_path
 
@@ -153,7 +162,7 @@ def _on_im_message_received(cls, data: P2ImMessageReceiveV1):
         chat_detail = LarkChatDetail(appId=app_id,
                                      appSecret=app_secret,
                                      messageId=msg.message_id)
-        unique_id = data.event.sender.sender_id.open_id + "@" + chat_id
+        unique_id = data.event.sender.sender_id.open_id + '@' + chat_id
         chat_svc.chat_by_agent(lark_content, ChatType.LARK, chat_detail,
                                unique_id, query_id)
 
@@ -241,8 +250,10 @@ def _get_content_type_when_at_user_exists(
         return LarkContentType.AT_OTHER_PERSON_TEXT
 
     @classmethod
-    def _store_image(cls, client: lark.client, message_id: str, image_key: str, path: str) -> bool:
-        body = GetMessageResourceRequest.builder().message_id(message_id).file_key(image_key).build()
+    def _store_image(cls, client: lark.client, message_id: str, image_key: str,
+                     path: str) -> bool:
+        body = GetMessageResourceRequest.builder().message_id(
+            message_id).file_key(image_key).build()
         response = client.im.v1.message_resource.get(body)
         if not response.success():
             logger.error(
@@ -329,7 +340,7 @@ def action(cls, body: WechatRequest,
             chat_request_body = ChatRequestBody(content=body.query.content)
 
         # push into chat queue
-        unique_id = body.username + "@" + body.groupname
+        unique_id = body.username + '@' + body.groupname
         chat_svc.chat_by_agent(chat_request_body, ChatType.WECHAT, body,
                                unique_id, query_id)
         # record query_id
@@ -369,7 +380,7 @@ def _fetch_response(cls, feature_store_id: str) -> WechatResponse:
                 l.append(
                     WechatPollItem(req=WechatRequest.model_validate_json(
                         json.dumps(item.detail)),
-                        rsp=item.response))
+                                   rsp=item.response))
                 complete_query_id_list.append(item.queryId)
         ret.root = l
 

web/service/qalib.py

@@ -13,8 +13,9 @@
 from web.model.huixiangdou import (HxdTask, HxdTaskPayload, HxdTaskType,
                                    HxdToken)
 from web.model.integrate import IntegrateLarkBody, IntegrateWebSearchBody
-from web.model.qalib import (Lark, QalibInfo, QalibPositiveNegative,
-                             QalibSample, WebSearch, Wechat, AddDocsRes, AddDocError)
+from web.model.qalib import (AddDocError, AddDocsRes, Lark, QalibInfo,
+                             QalibPositiveNegative, QalibSample, WebSearch,
+                             Wechat)
 from web.mq.hxd_task import HuixiangDouTask
 from web.orm.redis import r
 from web.util.log import log
@@ -108,9 +109,13 @@ async def add_docs(self, files: List[UploadFile] = File(...)):
         write_size = 0
         # store files
         for file in files:
-            if file.filename and len(file.filename.encode("utf-8")) > 255:
-                logger.error(f"filename: {file.filename} too long, maximum 255 bytes, omit current filename")
-                ret.errors.append(AddDocError(fileName=file.filename, reason="filename is too long"))
+            if file.filename and len(file.filename.encode('utf-8')) > 255:
+                logger.error(
+                    f'filename: {file.filename} too long, maximum 255 bytes, omit current filename'
+                )
+                ret.errors.append(
+                    AddDocError(fileName=file.filename,
+                                reason='filename is too long'))
                 continue
 
             with open(os.path.join(store_dir, file.filename), 'wb') as f:

web/tools/README.md

@@ -0,0 +1,5 @@
+# 运维工具
+
+- dump_redis_query.py   保存所有问题，无答复
+- get_puyu_model_list.py  获取 puyu 所有 model list
+- test_update_fs_max_len.py  更新所有用户的 remote LLM 最大长度

tests/tool_dump_redis_query.py web/tools/dump_redis_query.py

@@ -1,7 +1,9 @@
-from redis import Redis
+import json
 import os
+
 from loguru import logger
-import json
+from redis import Redis
+
 
 def redis_host():
     host = os.getenv('REDIS_HOST')
@@ -29,7 +31,11 @@ def feature_store_base_dir():
     return 'feature_stores'
 
 
-db = Redis(host=redis_host(), port=redis_port(), password=redis_passwd(), charset='utf-8', decode_responses=True)
+db = Redis(host=redis_host(),
+           port=redis_port(),
+           password=redis_passwd(),
+           charset='utf-8',
+           decode_responses=True)
 keys = db.keys('HuixiangDou:query:*')
 
 with open('query.jsonl', 'a') as f: