patch20250130: make the concurrency of uploading jobsbs configurable (#200)

* make the concurrency of uploading jobsbs configurable

* also make upload_chunk_size configurable

Author: colorzzr

app/configs/app_config.py

@@ -13,7 +13,7 @@ class Env:
         # NOTE: there is a limitation on minio that
         # the multipart number is 10000. so we set
         # the chunk_size as 20MB -> total 200GB
-        chunk_size = 1024 * 1024 * 20  # MB
+        chunk_size = ConfigClass.upload_chunk_size  # MB
         resilient_retry = 3
         resilient_backoff = 1
         resilient_retry_interval = 1  # seconds
@@ -31,7 +31,7 @@ class Env:
         # set hard limit for pending jobs, otherwise cli will consume all memory
         # to cache jobs. If later on the speed of chunk deliver become faster, we
         # can increase the concurrency number.
-        num_of_jobs = 20
+        num_of_jobs = ConfigClass.concurrent_job_limit
 
         github_url = 'PilotDataPlatform/cli'
 

app/configs/config.py

@@ -37,6 +37,8 @@ class Settings(BaseSettings):
     apikey_endpoint: str = 'api-key'
 
     upload_batch_size: int = 100
+    concurrent_job_limit: int = 10
+    upload_chunk_size: int = 1024 * 1024 * 20  # 20MB
 
     def __init__(self, **data):
         super().__init__(**data)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "3.12.0"
+version = "3.12.1"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]