Merge pull request #90 from camaze/redis_persist_cjn

【CodeFuse】redis data persistence

Author: lightislost

docker-compose.yaml

@@ -114,9 +114,13 @@ services:
       - "8001:8001"
     volumes:
       - ./logs/redis:/var/lib/redis/logs
+      - ./examples/redis-stack.conf:/redis-stack.conf
+      - ./data/redis:/data
     networks:
       - ekg-net
     restart: always
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
 
 
   ollama:
@@ -201,7 +205,7 @@ services:
     restart: on-failure
     networks:
       - ekg-net
-    command: sh -c "uvicorn start:app --reload --reload-dir /home/user/muagent/examples --reload-dir /home/user/muagent/muagent --port=3737 --host=0.0.0.0"
+    command: sh -c "uvicorn start:app --reload --reload-dir /home/user/muagent/examples --reload-dir /home/user/muagent/muagent --reload-exclude */test_config.py --port=3737 --host=0.0.0.0"
 
   ekgfrontend:
     build:

examples/ekg_examples/start.py

@@ -273,12 +273,14 @@ def embed_query(self, text: str) -> List[float]:
 # )
 embed_config = None
 
+clear_history_data = os.environ.get('clear_history_data', 'False') == 'True'
 
 ekg_construct_service = EKGConstructService(
     embed_config=embed_config,
     llm_config=llm_config,
     tb_config=tb_config,
     gb_config=gb_config,
+    clear_history_data=clear_history_data
 )
 
 

examples/redis-stack.conf

@@ -0,0 +1,9 @@
+# https://redis.io/docs/latest/operate/oss_and_stack/management/config-file/
+
+# disable rdb
+save ""
+
+# aof config
+appendonly yes
+appendfsync everysec
+auto-aof-rewrite-min-size 8mb

examples/test_config.py.example

@@ -41,6 +41,8 @@ os.environ["tb_index_name"] = "ekg_migration_new"
 os.environ['tb_definition_value'] = 'message_test_new'
 os.environ['tb_expire_time'] = '604800' #86400*7
 
+# clear history data in tb and gb
+os.environ['clear_history_data'] = 'True'
 
 
 ########################################

muagent/service/ekg_construct/ekg_construct_base.py

@@ -62,7 +62,8 @@ def __init__(
             intention_router: Optional[IntentionRouter] = None,
             do_init: bool = False,
             kb_root_path: str = KB_ROOT_PATH,
-            initialize_space=True
+            initialize_space=True,
+            clear_history_data=True
         ):
 
         self.db_config = db_config
@@ -83,6 +84,7 @@ def __init__(
             self.model, embed_config=self.embed_config)
         # init db handler
         self.initialize_space = initialize_space
+        self.clear_history_data = clear_history_data
         self.init_handler()
         # load custom keywords
         if os.path.exists(EXTRA_KEYWORDS_PATH):
@@ -138,7 +140,6 @@ def init_tb(self, do_init: bool=None):
             TextField("ekg_type",),
         ]
 
-
         if self.tb_config:
             tb_dict = {"TbaseHandler": TbaseHandler}
             tb_class =  tb_dict.get(self.tb_config.tb_type, TbaseHandler)
@@ -148,6 +149,11 @@ def init_tb(self, do_init: bool=None):
                 definition_value=self.tb_config.extra_kwargs.get(
                     "definition_value", "muagent_ekg")
             )
+
+            if self.clear_history_data:
+                self.tb.drop_index(self.node_indexname)
+                self.tb.drop_index(self.edge_indexname)
+
             # # create index
             if not self.tb.is_index_exists(self.node_indexname):
                 res = self.tb.create_index(
@@ -176,8 +182,11 @@ def init_gb(self, do_init: bool=None):
                 self.gb.add_hosts('storaged0', 9779)
                 print('增加NebulaGraph Storage主机中，等待20秒')
                 time.sleep(20)
-                # 初始化space
-                self.gb.drop_space('client')
+
+                if self.clear_history_data:
+                    # 初始化space
+                    self.gb.drop_space('client')
+
                 self.gb.create_space('client')
 
                 # 创建node tags和edge types