Fix: setup google-storage client (langgenius#4296)

Co-authored-by: kotamat <[email protected]>
Co-authored-by: crazywoola <[email protected]>

Author: 3 people

api/extensions/storage/google_storage.py

@@ -1,4 +1,5 @@
 import base64
+import io
 from collections.abc import Generator
 from contextlib import closing
 
@@ -15,14 +16,19 @@ def __init__(self, app: Flask):
         super().__init__(app)
         app_config = self.app.config
         self.bucket_name = app_config.get('GOOGLE_STORAGE_BUCKET_NAME')
-        service_account_json = base64.b64decode(app_config.get('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')).decode(
-            'utf-8')
-        self.client = GoogleCloudStorage.Client().from_service_account_json(service_account_json)
+        service_account_json_str = app_config.get('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
+        # if service_account_json_str is empty, use Application Default Credentials
+        if service_account_json_str:
+            service_account_json = base64.b64decode(service_account_json_str).decode('utf-8')
+            self.client = GoogleCloudStorage.Client.from_service_account_info(service_account_json)
+        else:
+            self.client = GoogleCloudStorage.Client()
 
     def save(self, filename, data):
         bucket = self.client.get_bucket(self.bucket_name)
         blob = bucket.blob(filename)
-        blob.upload_from_file(data)
+        with io.BytesIO(data) as stream:
+            blob.upload_from_file(stream)
 
     def load_once(self, filename: str) -> bytes:
         bucket = self.client.get_bucket(self.bucket_name)

docker/docker-compose.yaml

@@ -88,6 +88,7 @@ services:
       AZURE_BLOB_ACCOUNT_URL: 'https://<your_account_name>.blob.core.windows.net'
       # The Google storage configurations, only available when STORAGE_TYPE is `google-storage`.
       GOOGLE_STORAGE_BUCKET_NAME: 'yout-bucket-name'
+      # if you want to use Application Default Credentials, you can leave GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 empty.
       GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: 'your-google-service-account-json-base64-string'
       # The type of vector store to use. Supported values are `weaviate`, `qdrant`, `milvus`, `relyt`.
       VECTOR_STORE: weaviate
@@ -226,6 +227,7 @@ services:
       AZURE_BLOB_ACCOUNT_URL: 'https://<your_account_name>.blob.core.windows.net'
       # The Google storage configurations, only available when STORAGE_TYPE is `google-storage`.
       GOOGLE_STORAGE_BUCKET_NAME: 'yout-bucket-name'
+      # if you want to use Application Default Credentials, you can leave GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 empty.
       GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: 'your-google-service-account-json-base64-string'
       # The type of vector store to use. Supported values are `weaviate`, `qdrant`, `milvus`, `relyt`, `pgvector`.
       VECTOR_STORE: weaviate