Implements dockerfile and authentification api

Author: tacxou

.gitignore

@@ -2,5 +2,4 @@ cache/
 logs/
 .env
 src/__pycache__/*
-data/*
 .idea/

Dockerfile

@@ -8,20 +8,20 @@ WORKDIR /data
 
 ARG UID=10001
 RUN adduser \
-    --disabled-password \
-    --gecos "" \
-    --home "/nonexistent" \
-    --shell "/sbin/nologin" \
-    --no-create-home \
-    --uid "${UID}" \
-    appuser
+  --disabled-password \
+  --gecos "" \
+  --home "/nonexistent" \
+  --shell "/sbin/nologin" \
+  --no-create-home \
+  --uid "${UID}" \
+  appuser
 
 RUN --mount=type=cache,target=/root/.cache/pip \
-    --mount=type=bind,source=requirements.txt,target=requirements.txt \
-    python -m pip install -r requirements.txt
+  --mount=type=bind,source=requirements.txt,target=requirements.txt \
+  python -m pip install -r requirements.txt
 
 USER appuser
 
 COPY . .
 
-CMD py main.py
+CMD python main.py

Makefile

@@ -1,13 +1,31 @@
 include .env
 STC_API_HOST?=taiga.archi.fr
 STC_API_FORWARD_PORT?=1337
+IMG_NAME = "ghcr.io/libertech-fr/sesame-taiga_crawler"
+BASE_NAME = "sesame"
+APP_NAME = "sesame-taiga_crawler"
+PLATFORM = "linux/amd64"
 
 .DEFAULT_GOAL := help
 help:
 	@printf "\033[33mUsage:\033[0m\n  make [target] [arg=\"val\"...]\n\n\033[33mTargets:\033[0m\n"
 	@grep -E '^[-a-zA-Z0-9_\.\/]+:.*?## .*$$' $(firstword $(MAKEFILE_LIST)) \
 		| sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[32m%-15s\033[0m %s\n", $$1, $$2}'
 
+build: ## Construit l'image docker
+	@printf "\033[33mDOCKER:\033[0m Build docker image ...\n"
+	@docker build -t sesame-taiga_crawler .
+	@printf "\033[33mDOCKER:\033[0m SUCCESSFUL !!!\n"
+
+run-crawler-docker: ## Lance le crawler Sesame - Taiga avec python !
+	@docker run --rm -it \
+		--add-host host.docker.internal:host-gateway \
+		--network dev \
+		--platform $(PLATFORM) \
+		--name $(APP_NAME) \
+		-v $(CURDIR):/data \
+		sesame-taiga_crawler
+
 run-crawler: ## Lance le crawler Sesame - Taiga avec python !
 	@python3 main.py
 

data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

main.py

@@ -66,11 +66,11 @@
 
 
 async def main():
-    logger.info("Starting Taiga crawler...")
-    await a_moins_b(url, 0, -1, headers)
-    collection_tasks = [col.get('function')(url, col, headers) for col in collections]
-    await asyncio.gather(*collection_tasks)
-    print("Taiga crawler ended successful !!!")
+    # logger.info("Starting Taiga crawler...")
+    # await a_moins_b(url, 0, -1, headers)
+    # collection_tasks = [col.get('function')(url, col, headers) for col in collections]
+    # await asyncio.gather(*collection_tasks)
+    # print("Taiga crawler ended successful !!!")
 
     print("Starting import_ind...")
     start_time = datetime.now()

src/__pycache__/__init__.cpython-310.pyc

@@ -109,8 +109,15 @@ async def sem_task(task):
     return await asyncio.gather(*[sem_task(task) for task in tasks])
 
 async def send_request(session, url, json):
+    token = os.getenv('SESAME_API_TOKEN')
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json; charset=utf-8",
+    }
+
     try:
-        async with session.post(url, json=json) as response:
+        print(f"json: {json}")
+        async with session.post(url, json=json, headers=headers) as response:
             response.raise_for_status()  # Raises error for 4xx/5xx responses
             print(f"Request to {url} successful: {response.status}")
     except aiohttp.ClientError as e:
@@ -125,7 +132,7 @@ async def get_data(data, config):
         result.append(treated)
     return result
 
-    
+
 
 async def process_data(data, config, file, session):
     print(f"Processing {file}")
@@ -147,4 +154,4 @@ async def import_ind():
         # for file in cache_files:
         #     await process_data(datas[file], configs[file], file, session)
         tasks = [process_data(datas[file], configs[file], file, session) for file in cache_files]
-        await gather_with_concurrency(10, tasks)
+        await gather_with_concurrency(10, tasks)