Merge pull request #238 from GispoCoding/Fix_lambda_libraries

tmiosmauli · web-flow · commit 80dee3d5179c · 2023-06-13T16:09:03.000+03:00
Fix lambda libraries
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,51 +1,51 @@
-exclude: ".enc.json$"
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
-    hooks:
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-      - id: mixed-line-ending
-  - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: v8.5.0
-    hooks:
-      - id: eslint
-        additional_dependencies:
-          - eslint@8.5.0
-          - eslint-plugin-import@2.25.3
-          - eslint-import-resolver-typescript@2.5.0
-          - eslint-plugin-react@7.27.1
-          - eslint-plugin-react-hooks@4.3.0
-          - eslint-webpack-plugin@3.1.1
-          - typescript-eslint@0.0.1-alpha.0
-          - "@typescript-eslint/eslint-plugin@5.7.0"
-          - "@typescript-eslint/parser@5.7.0"
-        files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx
-        types: [file]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.5.1
-    hooks:
-      - id: prettier
-        types_or: [css, javascript]
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
-    hooks:
-      - id: isort
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
-    hooks:
-      - id: mypy
-        additional_dependencies:
-          - types-requests
-  - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-bugbear~=21.9.2
-          - pep8-naming~=0.12.1
+exclude: ".enc.json$"
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v8.5.0
+    hooks:
+      - id: eslint
+        additional_dependencies:
+          - eslint@8.5.0
+          - eslint-plugin-import@2.25.3
+          - eslint-import-resolver-typescript@2.5.0
+          - eslint-plugin-react@7.27.1
+          - eslint-plugin-react-hooks@4.3.0
+          - eslint-webpack-plugin@3.1.1
+          - typescript-eslint@0.0.1-alpha.0
+          - "@typescript-eslint/eslint-plugin@5.7.0"
+          - "@typescript-eslint/parser@5.7.0"
+        files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx
+        types: [file]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.5.1
+    hooks:
+      - id: prettier
+        types_or: [css, javascript]
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910-1
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - types-requests
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear~=21.9.2
+          - pep8-naming~=0.12.1
diff --git a/backend/arcgis.Dockerfile b/backend/arcgis.Dockerfile
@@ -1,14 +1,15 @@
-FROM public.ecr.aws/lambda/python:3.8
-
+FROM public.ecr.aws/lambda/python:3.10
 # common code
 COPY lambda_functions/base_loader/__init__.py ${LAMBDA_TASK_ROOT}/app/__init__.py
 COPY lambda_functions/base_loader/base_loader.py ${LAMBDA_TASK_ROOT}/app/base_loader.py
 # common deps
 RUN pip3 install  \
     psycopg2-binary \
     geoalchemy2 \
+    sqlalchemy==1.4 \
     requests \
     shapely==1.8.0  \
+    urllib3==1.26 \
     --target "${LAMBDA_TASK_ROOT}"
 
 # this code
diff --git a/backend/db_manager.Dockerfile b/backend/db_manager.Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.8
+FROM public.ecr.aws/lambda/python:3.10
 
 # Copy function code
 COPY lambda_functions/db_manager/db_manager.py ${LAMBDA_TASK_ROOT}/db_manager.py
diff --git a/backend/lambda_functions/lipas_loader/lipas_loader.py b/backend/lambda_functions/lipas_loader/lipas_loader.py
@@ -1,3 +1,4 @@
+import time
 from typing import Any, Dict, List, Optional, Tuple
 
 import requests
@@ -50,6 +51,7 @@ def __init__(
         **kwargs,
     ) -> None:
         super().__init__(connection_string, **kwargs)
+        self.feature_counter: int = 0  # Sorry but this had to be done :-)
 
         self.type_codes_all_year = (
             type_codes_all_year
@@ -86,7 +88,6 @@ def get_features(self, only_page: Optional[int] = None) -> List[int]:  # type: i
             r = requests.get(url, params=params, headers=self.HEADERS)
             r.raise_for_status()
             data = r.json()
-
             if data:
                 ids += [item[ID_FIELD] for item in data if "location" in item]
                 current_page += 1
@@ -160,7 +161,10 @@ def get_feature(self, sports_place_id: int):  # type: ignore[override]
             "deleted": False,
             "tarmo_category": tarmo_category,
         }
-
+        #  LOGGER.info(f"Features loaded: {len(flattened)}")
+        self.feature_counter = self.feature_counter + 1
+        if self.feature_counter % 10 == 0:
+            time.sleep(1)
         return flattened
 
     def save_feature(self, sport_place: Dict[str, Any], session: Session) -> bool:
@@ -218,7 +222,7 @@ def _sport_places_url_and_params(self, page: int) -> Tuple[str, Dict[str, Any]]:
         return main_url, params
 
     def _sport_place_url(self, sports_place_id: int):
-        print(self.api_url)
+        #  print(self.api_url)
         return "/".join((self.api_url, LipasLoader.SPORT_PLACES, str(sports_place_id)))
 
 
diff --git a/backend/lipas.Dockerfile b/backend/lipas.Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.8
+FROM public.ecr.aws/lambda/python:3.10
 
 # common code
 COPY lambda_functions/base_loader/__init__.py ${LAMBDA_TASK_ROOT}/app/__init__.py
@@ -7,8 +7,10 @@ COPY lambda_functions/base_loader/base_loader.py ${LAMBDA_TASK_ROOT}/app/base_lo
 RUN pip3 install  \
     psycopg2-binary \
     geoalchemy2 \
+    sqlalchemy==1.4 \
     requests \
     shapely==1.8.0  \
+    urllib3==1.26 \
     --target "${LAMBDA_TASK_ROOT}"
 
 # this code
diff --git a/backend/notifier.Dockerfile b/backend/notifier.Dockerfile
@@ -1,10 +1,11 @@
-FROM public.ecr.aws/lambda/python:3.8
+FROM public.ecr.aws/lambda/python:3.10
 
 # Copy function code
 COPY lambda_functions/notifier/notifier.py ${LAMBDA_TASK_ROOT}/app/notifier.py
 
 RUN pip3 install  \
     requests \
+    urllib3==1.26 \
     --target "${LAMBDA_TASK_ROOT}"
 
 CMD [ "app.notifier.handler" ]
diff --git a/backend/osm.Dockerfile b/backend/osm.Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.8
+FROM public.ecr.aws/lambda/python:3.10
 
 # common code
 COPY lambda_functions/base_loader/__init__.py ${LAMBDA_TASK_ROOT}/app/__init__.py
@@ -7,8 +7,10 @@ COPY lambda_functions/base_loader/base_loader.py ${LAMBDA_TASK_ROOT}/app/base_lo
 RUN pip3 install  \
     psycopg2-binary \
     geoalchemy2 \
+    sqlalchemy==1.4 \
     requests \
     shapely==1.8.0  \
+    urllib3==1.26 \
     --target "${LAMBDA_TASK_ROOT}"
 
 # this code
diff --git a/backend/wfs.Dockerfile b/backend/wfs.Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.8
+FROM public.ecr.aws/lambda/python:3.10
 
 # common code
 COPY lambda_functions/base_loader/__init__.py ${LAMBDA_TASK_ROOT}/app/__init__.py
@@ -7,8 +7,10 @@ COPY lambda_functions/base_loader/base_loader.py ${LAMBDA_TASK_ROOT}/app/base_lo
 RUN pip3 install  \
     psycopg2-binary \
     geoalchemy2 \
+    sqlalchemy==1.4 \
     requests \
     shapely==1.8.0  \
+    urllib3==1.26 \
     --target "${LAMBDA_TASK_ROOT}"
 
 # this code
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
 #    pip-compile requirements-dev.in
 #
diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
 #    pip-compile requirements.in
 #
@@ -14,7 +14,7 @@ charset-normalizer==2.0.10
     #   requests
 geoalchemy2==0.10.0
     # via -r requirements.in
-greenlet==1.1.2
+greenlet==2.0.2
     # via sqlalchemy
 idna==3.3
     # via requests
@@ -34,7 +34,7 @@ python-slugify==6.1.1
     # via -r requirements.in
 requests==2.27.0
     # via -r requirements.in
-shapely==1.8.0
+shapely==1.8.5
     # via -r requirements.in
 sqlalchemy==1.4.29
     # via
diff --git a/web/public/map-styles/nls-style.json b/web/public/map-styles/nls-style.json
diff --git a/web/yarn.lock b/web/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM public.ecr.aws/lambda/python:3.8`
	`1`	`+FROM public.ecr.aws/lambda/python:3.10`
`2`	`2`
`3`	`3`	`# Copy function code`
`4`	`4`	`COPY lambda_functions/db_manager/db_manager.py ${LAMBDA_TASK_ROOT}/db_manager.py`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`#`
`2`		`-# This file is autogenerated by pip-compile with python 3.10`
`3`		`-# To update, run:`
	`2`	`+# This file is autogenerated by pip-compile with Python 3.11`
	`3`	`+# by the following command:`
`4`	`4`	`#`
`5`	`5`	`# pip-compile requirements-dev.in`
`6`	`6`	`#`