Merge pull request #924 from mapswipe/dev

Backend Workers Refactoring

Author: Hagellach37

.github/workflows/actions.yml

@@ -63,6 +63,7 @@ jobs:
       env:
         FIREBASE_API_KEY: ${{ secrets.FIREBASE_API_KEY }}
         FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
+        FIREBASE_STORAGE_BUCKET: ${{ secrets.FIREBASE_STORAGE_BUCKET }}
         POSTGRES_PASSWORD: postgres
         POSTGRES_USER: postgres
         POSTGRES_DB: postgres

Makefile

@@ -24,7 +24,9 @@ update_firebase_functions_and_db_rules:
 	docker compose run --rm firebase_deploy
 
 deploy_latest_workers_version:
-	git pull; docker compose build postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy; docker compose up -d postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy
-
+	git pull; docker compose build postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy; docker compose up -d postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy; docker compose restart nginx
 update_tms_api_key:
 	@echo "first update the .env file with the new api key and then run this script with tms_name=your_tms_provider_name to update API key"; docker compose run --rm mapswipe_workers_creation python python_scripts/change_tms_api_key_for_projects.py $(tms_name)
+
+list_all_project_managers:
+	docker compose run --rm mapswipe_workers_creation python python_scripts/list_all_project_managers.py

README.md

@@ -21,6 +21,10 @@ Please refer to the documentation for more information: https://mapswipe-workers
 - MapSwipe OSM-Wiki: https://wiki.openstreetmap.org/wiki/MapSwipe
 
 
+## Development Setup 
+
+Please see here: https://mapswipe-workers.readthedocs.io/en/master/dev_setup.html
+
 ## Contributing Guidelines
 
 ### Feature Branch

docker-compose.development.yaml

@@ -13,14 +13,14 @@ services:
     manager_dashboard:
         container_name: manager_dashboard
         build:
-            context: manager_dashboard/
+            context: manager-dashboard/
         restart: unless-stopped
         ports:
             - '5000:80'
         expose:
             - "80"
         volumes:
-            - ./manager_dashboard/manager_dashboard:/usr/share/nginx/html/manager_dashboard
+            - ./manager-dashboard:/usr/share/nginx/html/manager_dashboard
         networks:
             - manager_dashboard
 
@@ -39,6 +39,6 @@ services:
           - ./postgres-data:/var/lib/postgresql/mapswipe
         restart: unless-stopped
         ports:
-          - "5431:5432"
+          - "${POSTGRES_PORT}:5432"
         networks:
-            - postgres
+            - postgres

docker-compose.yaml

@@ -58,6 +58,7 @@ x-mapswipe-workers: &base_mapswipe_workers
         FIREBASE_DB: '${FIREBASE_DB}'
         FIREBASE_API_KEY: '${FIREBASE_API_KEY}'
         FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
+        FIREBASE_STORAGE_BUCKET: '${FIREBASE_STORAGE_BUCKET}'
         GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
         POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
         POSTGRES_USER: '${POSTGRES_USER}'

docs/source/configuration.md

@@ -3,7 +3,7 @@
 Most of the configuration is stored in environment variables.
 At the root of the GitHub repository (in the same directory as `docker-compose.yml`) an example file (`example.env`) with all possible configuration variables exists. To get started copy this file to `.env` (no name is required) and fill in missing variables. The Docker Compose file will access those variables when needed.
 
-> Note: If you want those variables to be accessible as Environment Variables in your current shell (Eg. Inside a Python virtual environment for development.) your need to parse the file and export the variables: `export $(cat .env | xargs)`
+> Note: If you want those variables to be accessible as Environment Variables in your current shell (E.g. Inside a Python virtual environment for development.) your need to parse the file and export the variables: `export $(cat .env | xargs)`
 
 In following chapters configuration values and keys are discussed for each part of the MapSwipe Back-end.
 
@@ -17,6 +17,7 @@ Required environment variables are:
 - FIREBASE_API_KEY
 - FIREBASE_DB
 - FIREBASE_TOKEN
+- FIREBASE_STORAGE_BUCKET
 - GOOGLE_APPLICATION_CREDENTIALS
 
 ### Postgres DB

docs/source/dev_setup.md

@@ -46,17 +46,6 @@ The path to the Service Account Key is defined in the `GOOGLE_APPLICATION_CREDEN
 You could also set up your own Firebase instance. However, this is not recommended. 
 If you still want to do it, get your Service Account Key from Firebase from [Google Cloud Service Accounts](https://console.cloud.google.com/iam-admin/serviceaccounts).
 
-#### Directories
-
-MapSwipe Workers needs access to a data directory for logs and data for the API:
-
-To create this directories run:
-```
-mkdir --parents ~/.local/share/mapswipe_workers
-```
-
-> Note: XDG Base Directory Specification is respected
-
 
 ### Database
 
@@ -74,21 +63,33 @@ Or set up Postgres using the `initdb.sql` file in the `postgres/` folder.
 
 ### Mapswipe-Workers Python Package
 
-1. Export environment variables to current shell.
-2. Create a Python virtual environment with `system-site-packages` option enabled to get access to GDAL/OGR Python packages
-3. Activate the virtual environment.
-5. Install MapSwipe Workers using pip.
-6. Run it.
+#### Installation
+
+The Python virtual environment needs to be created with `system-site-packages` option enabled to get access to GDAL/OGR Python packages.
+Also the MapSwipe Workers Python Package needs access to a data directory for logs and data for the API.
 
 ```bash
-export $(cat .env | xargs)
 cd mapswipe_workers
 python -m venv --system-site-packages venv
 source venv/bin/activate
-pip install --editable mapswipe_workers/
+pip install --editable .
+pre-commit install
+mkdir --parents ~/.local/share/mapswipe_workers
+```
+
+#### Usage
+
+```bash
+export $(cat .env | xargs)  # See section on Configuration above
 mapswipe_workers --help
 ```
 
+#### Tests
+
+```bash
+python -m unittest
+```
+
 > Yeah! If you reached this point, you are ready to get into coding. Below you find some more information on Logging, Firebase Functions and Database Backup. However, you don't need this to get started for now.
 
 # Further Information

example.env

@@ -6,6 +6,7 @@ FIREBASE_API_KEY=
 FIREBASE_TOKEN=
 GOOGLE_APPLICATION_CREDENTIALS="$HOME/.config/mapswipe_workers/serviceAccountKey.json"
 FIREBASE_AUTH_SITE=
+FIREBASE_STORAGE_BUCKET=
 
 # postgres configuration
 POSTGRES_USER=mapswipe_workers

firebase/Dockerfile

@@ -3,4 +3,4 @@ RUN npm install firebase-functions@latest firebase-admin@latest --save
 RUN npm install -g [email protected]
 COPY . /firebase
 RUN cd firebase/functions && npm install
-WORKDIR /firebase/
+WORKDIR /firebase/

firebase/README.md

@@ -7,7 +7,7 @@ Create a docker image from the latest data.
 * `docker-compose build firebase_image`
 
 Alternatively you could do
-* `sudo docker build -t . firebase_image`
+* `sudo docker build -t firebase_image .`
 
 Then run the container interactively and open a bash shell.
 * `sudo docker run -it firebase_image /bin/bash`

mapswipe_workers/.pre-commit-config.yaml

@@ -4,11 +4,11 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/PyCQA/flake8/
     rev: 3.8.3
     hooks:
       - id: flake8
   - repo: https://github.com/PyCQA/isort
-    rev: 5.5.2
+    rev: 5.12.0
     hooks:
       - id: isort

mapswipe_workers/mapswipe_workers/auth.py

@@ -29,18 +29,25 @@ def get_tileserver_url(tileserver: str) -> str:
 
 
 def firebaseDB() -> db:
+    """Initialize the app.
+
+    Credentials will be retrieved from of following environment variable:
+    `GOOGLE_APPLICATION_CREDENTIALS` which contains the path to service account
+    key.
+    """
+
     try:
         # Is an App instance already initialized?
         firebase_admin.get_app()
         # Return the imported Firebase Realtime Database module
         return db
     except ValueError:
-        databaseURL = f"https://{FIREBASE_DB}.firebaseio.com"
-        # Initialize the app.
-        # Credentials will be retrieved from of following environment variable:
-        # GOOGLE_APPLICATION_CREDENTIALS (Path to service account key in json format)
-        firebase_admin.initialize_app(options={"databaseURL": databaseURL})
-        # Return the imported Firebase Realtime Database module
+        if "https://" in FIREBASE_DB:
+            firebase_admin.initialize_app(options={"databaseURL": FIREBASE_DB})
+        else:
+            databaseURL = f"https://{FIREBASE_DB}.firebaseio.com"
+            firebase_admin.initialize_app(options={"databaseURL": databaseURL})
+
         return db
 
 

mapswipe_workers/mapswipe_workers/config.py

@@ -2,6 +2,7 @@
 
 FIREBASE_API_KEY = os.environ["FIREBASE_API_KEY"]
 FIREBASE_DB = os.getenv("FIREBASE_DB", default="mapswipe")
+FIREBASE_STORAGE_BUCKET = os.getenv("FIREBASE_STORAGE_BUCKET")
 
 POSTGRES_DB = os.getenv("POSTGRES_DB", default="mapswipe")
 POSTGRES_HOST = os.getenv("POSTGRES_HOST", "postgres")

mapswipe_workers/mapswipe_workers/definitions.py

@@ -132,44 +132,45 @@ class ProjectType(Enum):
     FOOTPRINT = 2
     CHANGE_DETECTION = 3
     COMPLETENESS = 4
+    MEDIA_CLASSIFICATION = 5
+    DIGITIZATION = 6
 
     @property
     def constructor(self):
         # Imports are first made once this method get called to avoid circular imports.
-        from mapswipe_workers.project_types.arbitrary_geometry.project import (
-            Project as ag_project,
-        )
-        from mapswipe_workers.project_types.tile_map_service_grid.project import (
-            Project as tmsg_project,
+        from mapswipe_workers.project_types import (
+            ChangeDetectionProject,
+            ClassificationProject,
+            CompletenessProject,
+            DigitizationProject,
+            FootprintProject,
+            MediaClassificationProject,
         )
 
         project_type_classes = {
-            1: tmsg_project,
-            2: ag_project,
-            3: tmsg_project,
-            4: tmsg_project,
+            1: ClassificationProject,
+            2: FootprintProject,
+            3: ChangeDetectionProject,
+            4: CompletenessProject,
+            5: MediaClassificationProject,
+            6: DigitizationProject,
         }
         return project_type_classes[self.value]
 
     @property
     def tutorial(self):
         # Imports are first made once this method get called to avoid circular imports.
-        from mapswipe_workers.project_types.arbitrary_geometry.tutorial import (
-            Tutorial as ArbitraryGeometryTutorial,
-        )
-        from mapswipe_workers.project_types.tile_map_service_grid.tutorial import (
-            Tutorial as tmsg_tutorial,
+        from mapswipe_workers.project_types import (
+            ChangeDetectionTutorial,
+            ClassificationTutorial,
+            CompletenessTutorial,
+            FootprintTutorial,
         )
 
-        # TODO: implement for arbitrary geometries
-        # from mapswipe_workers.project_types.arbitrary_geometries.arbitrary_geometries_tutorial import ( # noqa E501
-        #    ArbitraryGeometriesTutorial,
-        # )
-
         project_type_classes = {
-            1: tmsg_tutorial,
-            2: ArbitraryGeometryTutorial,
-            3: tmsg_tutorial,
-            4: tmsg_tutorial,
+            1: ClassificationTutorial,
+            2: FootprintTutorial,
+            3: ChangeDetectionTutorial,
+            4: CompletenessTutorial,
         }
         return project_type_classes[self.value]