fixed the tokenizer for the claude models. Using the same one as for gpt-4-turbo since anthropic doesnt have their tokenizer public. Also added the possibility to enable or disable the logging in the console

Flamingo Fiesta · Flamingo Fiesta · commit 6c9ec9afc665 · 2024-05-19T23:46:37.000+02:00
diff --git a/bot/bot.py b/bot/bot.py
@@ -44,12 +44,13 @@
 
 # setup
 db = database.Database()
+
 logger = logging.getLogger(__name__)
 
 user_semaphores = {}
 user_tasks = {}
 
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s') #logging error
+
 HELP_MESSAGE = """Commands:
 
 ⚪ /new – Start new dialog 
@@ -92,11 +93,19 @@ def update_user_roles_from_config(db, roles):
             )
     print("User roles updated from config.")
 
-
 def split_text_into_chunks(text, chunk_size):
     for i in range(0, len(text), chunk_size):
         yield text[i:i + chunk_size]
 
+def configure_logging():
+    # Configure logging based on the enable_detailed_logging value
+    if config.enable_detailed_logging:
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+    else:
+        logging.basicConfig(level=logging.CRITICAL, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+
+    # Set the logger level based on configuration
+    logger.setLevel(logging.getLogger().level)
 
 async def register_user_if_not_exists(update: Update, context: CallbackContext, user: User):
     user_registered_now = False
@@ -1663,7 +1672,7 @@ async def model_settings_handler(update: Update, context: CallbackContext):
 
     elif data.startswith('claude-model-set_settings|'):
         # Check for API key
-        if config.claude_api_key is None or config.claude_api_key == "":
+        if config.anthropic_api_key is None or config.anthropic_api_key == "":
             await context.bot.send_message(
                 chat_id=user_id,
                 text="This bot does not have the Anthropic models available :(",
@@ -1678,7 +1687,7 @@ async def model_settings_handler(update: Update, context: CallbackContext):
     elif data.startswith('model-set_settings|'):
         _, model_key = data.split("|")
         # Prevent Claude models from being set without API key
-        if "claude" in model_key.lower() and (config.claude_api_key is None or config.claude_api_key == ""):
+        if "claude" in model_key.lower() and (config.anthropic_api_key is None or config.anthropic_api_key == ""):
             await context.bot.send_message(
                 chat_id=user_id,
                 text="This bot does not have the Anthropic models available :(",
@@ -2148,6 +2157,7 @@ def run_bot() -> None:
     bot_instance = application.bot
 
     update_user_roles_from_config(db, config.roles)
+    configure_logging()
 
     application = (
         ApplicationBuilder()
diff --git a/bot/config.py b/bot/config.py
@@ -14,9 +14,10 @@
 # config parameters
 telegram_token = config_yaml["telegram_token"]
 openai_api_key = config_yaml["openai_api_key"]
-claude_api_key = config_yaml.get("claude_api_key", None)
+anthropic_api_key = config_yaml.get("anthropic_api_key", None)
 developer_username =  config_yaml.get("developer_username", "")
 timezone = config_yaml.get("database_timezone", "UTC")
+enable_detailed_logging = config_yaml.get("enable_detailed_logging", True)
 stripe_secret_key = config_yaml.get("stripe_secret_key", None)
 stripe_webhook_secret = config_yaml.get("stripe_webhook_secret", None)  
 openai_api_base = config_yaml.get("openai_api_base", None)
diff --git a/bot/openai_utils.py b/bot/openai_utils.py
@@ -10,14 +10,14 @@
 
 import json #logging error
 
-from tokenizers import Tokenizer, models, pre_tokenizers, trainers
+#from tokenizers import Tokenizer, models, pre_tokenizers, trainers # other tokenizer module
+
 # setup openai
 openai.api_key = config.openai_api_key
+anthropic.api_key = config.anthropic_api_key
+
 if config.openai_api_base is not None:
     openai.api_base = config.openai_api_base
-logger = logging.getLogger(__name__)
-
-anthropic.api_key = config.claude_api_key
 
 OPENAI_COMPLETION_OPTIONS = {
     "temperature": 0.7,
@@ -28,24 +28,36 @@
     "request_timeout": 60.0,
 }
 
-#GPT HELP 2
+logger = logging.getLogger(__name__)
+
+def configure_logging():
+    # Configure logging based on the enable_detailed_logging value
+    if config.enable_detailed_logging:
+        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+    else:
+        logging.basicConfig(level=logging.CRITICAL, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+
+    # Set the logger level based on configuration
+    logger.setLevel(logging.getLogger().level)
+
+configure_logging()
+
 def validate_payload(payload): #maybe comment out
     # Example validation: Ensure all messages have content that is a string
     for message in payload.get("messages", []):
         if not isinstance(message.get("content"), str):
             logger.error("Invalid message content: Not a string")
             raise ValueError("Message content must be a string")
-#GPT HELP 2
-        
 
+        
 class ChatGPT:
     def __init__(self, model="gpt-4-1106-preview"):
         assert model in {"text-davinci-003", "gpt-3.5-turbo-16k", "gpt-3.5-turbo", "gpt-4", "gpt-4-1106-preview", "gpt-4-vision-preview", "gpt-4-turbo-2024-04-09", "gpt-4o", "claude-3-opus-20240229", "claude-3-sonnet-20240229", "claude-3-haiku-20240307"}, f"Unknown model: {model}"
         self.model = model
         self.is_claude_model = model.startswith("claude")
         self.logger = logging.getLogger(__name__)
         self.headers = {
-            "Authorization": f"Bearer {config.claude_api_key if self.is_claude_model else config.openai_api_key}",
+            "Authorization": f"Bearer {config.anthropic_api_key if self.is_claude_model else config.openai_api_key}",
             "Content-Type": "application/json",
         }
 
@@ -64,7 +76,7 @@ async def send_message(self, message, dialog_messages=[], chat_mode="assistant")
                     if not prompt.strip():
                         raise ValueError("Generated prompt is empty")
 
-                    client = anthropic.AsyncAnthropic(api_key=config.claude_api_key)
+                    client = anthropic.AsyncAnthropic(api_key=config.anthropic_api_key)
                     response = await client.completions.create(
                         model=self.model,
                         messages=[{"role": "user", "content": prompt}],
@@ -147,7 +159,7 @@ async def send_message_stream(self, message, dialog_messages=[], chat_mode="assi
                     if not prompt.strip():
                         raise ValueError("Generated prompt is empty")
                     
-                    client = anthropic.AsyncAnthropic(api_key=config.claude_api_key)
+                    client = anthropic.AsyncAnthropic(api_key=config.anthropic_api_key)
 
                     async with client.messages.stream(
                         model=self.model,
@@ -393,52 +405,31 @@ def _generate_claude_prompt(self, message, dialog_messages, chat_mode, image_buf
         combined_prompt += "\n\nAssistant:"
         return combined_prompt
 
-
     def _postprocess_answer(self, answer):
         self.logger.debug(f"Pre-processed answer: {answer}")
         answer = answer.strip()
         self.logger.debug(f"Post-processed answer: {answer}")
         return answer
 
     def _count_tokens_from_messages(self, messages, answer, model="gpt-4-1106-preview"):
+
         if model.startswith("claude"):
-            tokenizer = self._get_claude_tokenizer()
+            encoding = tiktoken.encoding_for_model("gpt-4-turbo-2024-04-09")
         else:
             encoding = tiktoken.encoding_for_model(model)
 
         tokens_per_message = 3
         tokens_per_name = 1
 
-        if model == "gpt-3.5-turbo-16k":
-            tokens_per_message = 4  # every message follows <im_start>{role/name}\n{content}<im_end>\n
-            tokens_per_name = -1  # if there's a name, the role is omitted
-        elif model == "gpt-3.5-turbo":
-            tokens_per_message = 4
-            tokens_per_name = -1
-        elif model == "gpt-4":
-            tokens_per_message = 3
-            tokens_per_name = 1
-        elif model == "gpt-4-1106-preview":
-            tokens_per_message = 3
-            tokens_per_name = 1
-        elif model == "gpt-4-vision-preview":
-            tokens_per_message = 3
-            tokens_per_name = 1 
-        elif model == "gpt-4-turbo-2024-04-09": 
-            tokens_per_message = 3
-            tokens_per_name = 1
-        elif model == "gpt-4o": 
+        if model.startswith("gpt-3"):
+            tokens_per_message = 4 # every message follows <im_start>{role/name}\n{content}<im_end>\n
+            tokens_per_name = -1 # if there's a name, the role is omitted
+        elif model.startswith("gpt-4"):
             tokens_per_message = 3
             tokens_per_name = 1 
-        elif model == "claude-3-opus-20240229": 
-            tokens_per_message = 3
-            tokens_per_name = 1
-        elif model == "claude-3-sonnet-20240229": 
+        elif model.startswith("claude"):
             tokens_per_message = 3
             tokens_per_name = 1 
-        elif model == "claude-3-haiku-20240307": 
-            tokens_per_message = 3
-            tokens_per_name = 1  
         else:
             raise ValueError(f"Unknown model: {model}")
 
@@ -450,49 +441,33 @@ def _count_tokens_from_messages(self, messages, answer, model="gpt-4-1106-previe
                 for sub_message in message["content"]:
                     if "type" in sub_message:
                         if sub_message["type"] == "text":
-                            if model.startswith("claude"):
-                                n_input_tokens += len(tokenizer.encode(sub_message["text"]).tokens)
-                            else:
-                                n_input_tokens += len(encoding.encode(sub_message["text"]))
+                            n_input_tokens += len(encoding.encode(sub_message["text"]))
                         elif sub_message["type"] == "image_url":
                             pass
             else:
                 if "type" in message:
                     if message["type"] == "text":
-                        if model.startswith("claude"):
-                            n_input_tokens += len(tokenizer.encode(message["text"]).tokens)
-                        else:
-                            n_input_tokens += len(encoding.encode(message["text"]))
+                        n_input_tokens += len(encoding.encode(message["text"]))
                     elif message["type"] == "image_url":
                         pass
 
-
         n_input_tokens += 2
 
         # output
-        if model.startswith("claude"):
-            n_output_tokens = 1 + len(tokenizer.encode(answer).tokens)
-        else:
-            n_output_tokens = 1 + len(encoding.encode(answer))
+        n_output_tokens = 1 + len(encoding.encode(answer))
 
         return n_input_tokens, n_output_tokens
 
     def _count_tokens_from_prompt(self, prompt, answer, model="text-davinci-003"):
         if model.startswith("claude"):
-            tokenizer = self._get_claude_tokenizer()
-            n_input_tokens = len(tokenizer.encode(prompt).tokens) + 1
-            n_output_tokens = len(tokenizer.encode(answer).tokens)
+            encoding = tiktoken.encoding_for_model("gpt-4-turbo-2024-04-09")
         else:
             encoding = tiktoken.encoding_for_model(model)
-            n_input_tokens = len(encoding.encode(prompt)) + 1
-            n_output_tokens = len(encoding.encode(answer))
 
-        return n_input_tokens, n_output_tokens
+        n_input_tokens = len(encoding.encode(prompt)) + 1
+        n_output_tokens = len(encoding.encode(answer))
 
-    def _get_claude_tokenizer(self):
-        tokenizer = Tokenizer(models.BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
-        return tokenizer
+        return n_input_tokens, n_output_tokens
     
 async def transcribe_audio(audio_file) -> str:
     r = await openai.Audio.atranscribe("whisper-1", audio_file)
@@ -501,6 +476,7 @@ async def transcribe_audio(audio_file) -> str:
 
 async def generate_images(prompt, model="dall-e-2", n_images=4, size="1024x1024", quality="standard"):
     """Generate images using OpenAI's specified model, including DALL-E 3."""
+    #redundancy to make sure the api call isnt made wrong
     if model=="dalle-2":
         model="dall-e-2"
         quality="standard"
diff --git a/config/config.example.yml b/config/config.example.yml
@@ -1,6 +1,6 @@
 telegram_token: ""
 openai_api_key: ""
-claude_api_key: ""
+anthropic_api_key: ""
 stripe_secret_key: "" 
 stripe_webhook_secret: "" # to recieve the payment updates
 openai_api_base: null  # leave null to use default api base or you can put your own base url here
@@ -10,6 +10,7 @@ return_n_generated_images: 1
 n_chat_modes_per_page: 5
 image_size: "1024x1024" #Can be configured within the bot menu, its initialized here to have a default
 enable_message_streaming: true  # if set, messages will be shown to user word-by-word
+enable_detailed_logging: true # if set to true, youll get constant logs of what is happening in the bot
 developer_username: [""] #will be included in certain errors given to users so they can contact the developer easier
 database_timezone: "" #so that the user_roles command give you accurate time of when the users last used the bot
 
@@ -30,4 +31,4 @@ roles:
   beta_tester: [] 
   friend: [] #unlimited usage of the bot, no admin privileges
   regular_user: []
-  trial_user: []
+  trial_user: [] #initial role for every user that registers, automatically upgrades to regular_user after first payment
diff --git a/config/models.yml b/config/models.yml
@@ -1,4 +1,4 @@
-#available_text_models: ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4-1106-preview", "gpt-4-vision-preview", "gpt-4-turbo-2024-04-09", "gpt-4", "text-davinci-003"] #Big menu
+#available_text_models: ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4-1106-preview", "gpt-4-vision-preview", "gpt-4-turbo-2024-04-09", "gpt-4", "text-davinci-003", "claude-3-opus-20240229", "claude-3-sonnet-20240229", "claude-3-haiku-20240307"] #Big menu
 available_text_models: ["gpt-3.5-turbo", "gpt-4-vision-preview", "gpt-4-turbo-2024-04-09", "gpt-4o", "claude-3-opus-20240229", "claude-3-sonnet-20240229", "claude-3-haiku-20240307"] #small menu with all needed models
 available_image_models: ["dalle-2", "dalle-3"]
 
@@ -136,7 +136,7 @@ info:
 
     scores:
       smart: 3
-      fast: 5
+      fast: 4
       cheap: 4
 
   text-davinci-003:
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,6 @@
 python-telegram-bot[rate-limiter]==20.1
 openai==0.28.1 #chatgpt library
-tiktoken>=0.3.0 #tokenizer for openai
+tiktoken>=0.3.0 #tokenizer 
 PyYAML==6.0 #configs 
 pymongo==4.3.3 #database
 python-dotenv==0.21.0 #.env files
@@ -11,4 +11,4 @@ redis>=4.0.2 #payment recieve notif
 aioredis>=2.0.0 #payment recieve notif
 pytz==2023.3 #timezone management
 anthropic #claude library
-tokenizers #tokenizer for anthropic
+#tokenizers #other tokenizer if you want to also use the one in the utils file
