Fix incomplete urls in lipas and osm data

Author: Rikuoja

backend/lambda_functions/lipas_loader/lipas_loader.py

@@ -145,6 +145,10 @@ def get_feature(self, sports_place_id: int):  # type: ignore[override]
             season = Season.ALL_YEAR.value
         tarmo_category = self.category_from_code[type_code]
 
+        # validate/fix URL!
+        if "www" in data and data["www"] and not data["www"].startswith("http"):
+            data["www"] = "https://" + data["www"]
+
         flattened = {
             **data,
             **props,

backend/lambda_functions/osm_loader/osm_loader.py

@@ -149,6 +149,12 @@ def get_feature(self, element: Dict[str, Any]) -> Optional[dict]:  # type: ignor
         if geom.is_empty:
             return None
 
+        # validate/fix URL!
+        if "website" in tags:
+            tags["www"] = tags.pop("website")
+            if tags["www"] and not tags["www"].startswith("http"):
+                tags["www"] = "https://" + tags["www"]
+
         # OSM ids are *only* unique *within* each type!
         # SQLAlchemy merge() doesn't know how to handle unique constraints that are not
         # pk. Therefore, we will have to specify the primary key here (not generated in

backend/test/test_lipas_loader.py

@@ -157,6 +157,15 @@ def test_get_sport_place_talviuintipaikka(loader):
     assert sport_place["tarmo_category"] == "Talviuinti"
 
 
+def test_get_sport_place_incomplete_url_fixed(loader):
+    sport_place = loader.get_feature(506521)
+    assert sport_place["geom"] == "MULTIPOINT (23.7443451904576 61.3050219780947)"
+    assert sport_place["season"] == "Koko vuosi"
+    assert sport_place["table"] == "lahiliikuntapaikka"
+    assert sport_place["tarmo_category"] == "Ulkoilupaikat"
+    assert sport_place["www"] == "https://www.lempaala.fi"
+
+
 def assert_data_is_imported(main_db_params):
     conn = psycopg2.connect(**main_db_params)
     try:

backend/test/test_osm_loader.py

@@ -83,6 +83,7 @@
                 "amenity": "parking",
                 "capacity": "5",
                 "parking": "surface",
+                "website": "www.parkkipaikka.fi",
             },
         },
         {
@@ -273,6 +274,7 @@ def test_get_customer_parking_feature(parking_loader, parking_data):
     assert customer_parking["osm_type"] == "node"
     assert customer_parking["geom"].startswith("POINT")
     assert customer_parking["tags"]
+    assert customer_parking["tags"]["www"] == "https://www.parkkipaikka.fi"
     assert customer_parking["type_name"] == "Asiakaspysäköinti"