Mise à jour de l'importation des données***

Author: RICHARD-Quentin

main.py

@@ -66,14 +66,18 @@
 
 
 async def main():
-    logger.info("Starting Taiga crawler...")
-    await a_moins_b(url, 0, -1, headers)
-    collection_tasks = [col.get('function')(url, col, headers) for col in collections]
-    await asyncio.gather(*collection_tasks)
-    print("Taiga crawler ended successful !!!")
+    # logger.info("Starting Taiga crawler...")
+    # await a_moins_b(url, 0, -1, headers)
+    # collection_tasks = [col.get('function')(url, col, headers) for col in collections]
+    # await asyncio.gather(*collection_tasks)
+    # print("Taiga crawler ended successful !!!")
 
-    print("Process Data")
+    print("Starting import_ind...")
+    start_time = datetime.now()
     await import_ind()
+    end_time = datetime.now()
+    execution_time = end_time - start_time
+    print(f"import_ind completed in {execution_time}")
 
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()

src/import_ind.py

@@ -7,7 +7,6 @@
 
 dotenv.load_dotenv()
 sesame_api_baseurl = os.getenv('SESAME_API_BASEURL')
-sem = asyncio.Semaphore(50)
 
 configs = {
     "taiga_etd.json": {
@@ -134,7 +133,7 @@ async def process_data(data, config, file, session):
     with open(f'./data/{file}', 'w', encoding='utf-8') as fichier:
         json.dump(result, fichier, ensure_ascii=False)
     tasks = [send_request(session, f'{sesame_api_baseurl}/management/identities/upsert', entry) for entry in result]
-    await gather_with_concurrency(10, tasks)
+    await gather_with_concurrency(25, tasks)
     print(f"Processed {file}")
 
 async def import_ind():
@@ -145,5 +144,7 @@ async def import_ind():
             datas[file] = json.load(fichier).get('data')
 
     async with aiohttp.ClientSession() as session:
-        for file in cache_files:
-            await process_data(datas[file], configs[file], file, session)
+        # for file in cache_files:
+        #     await process_data(datas[file], configs[file], file, session)
+        tasks = [process_data(datas[file], configs[file], file, session) for file in cache_files]
+        await gather_with_concurrency(10, tasks)