Merge pull request #520 from Police-Data-Accessibility-Project/mc_519_get_data_sources_bug

Fix bug in data sources and meta URL GET queries

Author: maxachis

src/api/endpoints/data_source/get/query.py

@@ -37,7 +37,6 @@ async def run(self, session: AsyncSession) -> DataSourceGetOuterResponse:
                 # Required Attributes
                 URL.name,
                 URLRecordType.record_type,
-                URL.confirmed_agencies,
 
                 # Optional Attributes
                 URL.description,
@@ -102,7 +101,7 @@ async def run(self, session: AsyncSession) -> DataSourceGetOuterResponse:
 
             url_description: str | None = mapping[URL.description]
             link_batch_url_batch_id: int | None = mapping[LinkBatchURL.batch_id]
-            url_record_formats: list[str] | None = mapping[URLOptionalDataSourceMetadata.record_formats]
+            url_record_formats: list[str] = mapping[URLOptionalDataSourceMetadata.record_formats] or []
             url_data_portal_type: str | None = mapping[URLOptionalDataSourceMetadata.data_portal_type]
             url_supplying_entity: str | None = mapping[URLOptionalDataSourceMetadata.supplying_entity]
             url_coverage_start: date | None = mapping[URLOptionalDataSourceMetadata.coverage_start]
@@ -118,7 +117,7 @@ async def run(self, session: AsyncSession) -> DataSourceGetOuterResponse:
             url_scraper_url: str | None = mapping[URLOptionalDataSourceMetadata.scraper_url]
             url_submission_notes: str | None = mapping[URLOptionalDataSourceMetadata.submission_notes]
             url_access_notes: str | None = mapping[URLOptionalDataSourceMetadata.access_notes]
-            url_access_types: list[AccessTypeEnum] | None = mapping[URLOptionalDataSourceMetadata.access_types]
+            url_access_types: list[AccessTypeEnum] = mapping[URLOptionalDataSourceMetadata.access_types] or []
 
             responses.append(
                 DataSourceGetResponse(

src/api/endpoints/meta_url/get/query.py

@@ -30,7 +30,6 @@ async def run(self, session: AsyncSession) -> MetaURLGetOuterResponse:
 
                 # Required Attributes
                 URL.name,
-                URL.confirmed_agencies,
 
                 # Optional Attributes
                 URL.description,

tests/automated/integration/readonly/api/agencies/get/test_root.py

@@ -12,7 +12,7 @@ async def test_agency_get(
     responses_raw: list[dict] = readonly_helper.api_test_helper.request_validator.get_v3(
         url=f"/agencies",
     )
-    assert len(responses_raw) == 1
+    assert len(responses_raw) == 2
     response_raw = responses_raw[0]
     assert response_raw["id"] == readonly_helper.agency_1_id
     assert response_raw["name"] == "Agency 1"

tests/automated/integration/readonly/api/data_sources/test_get.py

@@ -18,13 +18,13 @@ async def test_get(readonly_helper: ReadOnlyTestHelper):
     )
     outer_response = DataSourceGetOuterResponse(**raw_json)
 
-    assert len(outer_response.results) == 1
+    assert len(outer_response.results) == 2
     response: DataSourceGetResponse = outer_response.results[0]
 
     diff = DeepDiff(
         response.model_dump(mode='json'),
         DataSourceGetResponse(
-            url_id=readonly_helper.url_data_source_id,
+            url_id=readonly_helper.maximal_data_source,
             url="read-only-ds.com",
 
             name="Read only URL name",

tests/automated/integration/readonly/api/meta_urls/agencies/test_forbid.py

@@ -8,7 +8,7 @@
 @pytest.mark.asyncio
 async def test_forbid(readonly_helper: ReadOnlyTestHelper):
     check_forbidden_url_type(
-        route=f"/meta-urls/{readonly_helper.url_data_source_id}/agencies",
+        route=f"/meta-urls/{readonly_helper.minimal_data_source}/agencies",
         api_test_helper=readonly_helper.api_test_helper,
         method="GET"
     )

tests/automated/integration/readonly/conftest.py

@@ -6,7 +6,6 @@
 from sqlalchemy import Engine
 from starlette.testclient import TestClient
 
-from src.db.helpers.connect import get_postgres_connection_string
 from tests.automated.integration.api._helpers.RequestValidator import RequestValidator
 from tests.automated.integration.readonly.helper import ReadOnlyTestHelper
 from tests.automated.integration.readonly.setup import setup_readonly_data
@@ -45,6 +44,8 @@ async def readonly_helper(
         db_data_creator=db_data_creator,
     )
 
-    helper: ReadOnlyTestHelper = await setup_readonly_data(api_test_helper=api_test_helper)
+    helper: ReadOnlyTestHelper = await setup_readonly_data(
+        api_test_helper=api_test_helper
+    )
 
     yield helper

tests/automated/integration/readonly/helper.py

@@ -13,6 +13,9 @@ class Config:
 
     agency_1_id: int
     agency_1_location_id: int
+    agency_2_id: int
+    agency_2_location_id: int
 
-    url_data_source_id: int
+    minimal_data_source: int
+    maximal_data_source: int
     url_meta_url_id: int

tests/automated/integration/readonly/setup.py

@@ -15,7 +15,6 @@
 from src.db.models.impl.url.record_type.sqlalchemy import URLRecordType
 from tests.automated.integration.readonly.helper import ReadOnlyTestHelper
 from tests.helpers.api_test_helper import APITestHelper
-from tests.helpers.counter import next_int
 from tests.helpers.data_creator.core import DBDataCreator
 from tests.helpers.data_creator.models.creation_info.county import CountyCreationInfo
 from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo
@@ -33,7 +32,6 @@ async def setup_readonly_data(
         name="Pennsylvania",
         iso="PA"
     )
-
     allegheny_county: CountyCreationInfo = await db_data_creator.create_county(
         state_id=pennsylvania.us_state_id,
         name="Allegheny"
@@ -46,10 +44,18 @@ async def setup_readonly_data(
 
 
     # Add Agencies
-    agency_1_id: int = await add_agency(adb_client, pittsburgh)
+    agency_1_id: int = await add_agency(adb_client, pittsburgh.location_id)
+    agency_2_id: int = await add_agency(adb_client, allegheny_county.location_id)
 
     # Add Data Source With Linked Agency
-    url_data_source_id: int = await add_data_source(agency_1_id, db_data_creator)
+    maximal_data_source: int = await add_maximal_data_source(
+        agency_1_id=agency_1_id,
+        db_data_creator=db_data_creator
+    )
+    minimal_data_source: int = await add_minimal_data_source(
+        agency_1_id=agency_1_id,
+        db_data_creator=db_data_creator
+    )
 
     # Add Meta URL with Linked Agency
     url_meta_url_id: int = await add_meta_url(agency_1_id, db_data_creator)
@@ -61,7 +67,11 @@ async def setup_readonly_data(
         agency_1_id=agency_1_id,
         agency_1_location_id=pittsburgh.location_id,
 
-        url_data_source_id=url_data_source_id,
+        agency_2_id=agency_2_id,
+        agency_2_location_id=allegheny_county.location_id,
+
+        maximal_data_source=maximal_data_source,
+        minimal_data_source=minimal_data_source,
         url_meta_url_id=url_meta_url_id,
     )
 
@@ -93,7 +103,7 @@ async def add_meta_url(
     return url_id
 
 
-async def add_data_source(
+async def add_maximal_data_source(
     agency_1_id: int,
     db_data_creator: DBDataCreator
 ) -> int:
@@ -150,10 +160,41 @@ async def add_data_source(
     )
     return url_id
 
+async def add_minimal_data_source(
+    agency_1_id: int,
+    db_data_creator: DBDataCreator
+) -> int:
+    adb_client: AsyncDatabaseClient = db_data_creator.adb_client
+    url = URL(
+        scheme="https",
+        url="minimal-ds.com",
+        name="Minimal name",
+        trailing_slash=False,
+        collector_metadata={},
+        status=URLStatus.OK,
+        source=URLSource.ROOT_URL,
+    )
+    url_id: int = await adb_client.add(url, return_id=True)
+    await db_data_creator.create_validated_flags(
+        url_ids=[url_id],
+        validation_type=URLType.DATA_SOURCE
+    )
+    record_type = URLRecordType(
+        url_id=url_id,
+        record_type=RecordType.POLICIES_AND_CONTRACTS
+    )
+    await adb_client.add(record_type)
+
+    await db_data_creator.create_url_agency_links(
+        url_ids=[url_id],
+        agency_ids=[agency_1_id]
+    )
+    return url_id
+
 
 async def add_agency(
     adb_client: AsyncDatabaseClient,
-    pittsburgh: LocalityCreationInfo
+    location_id: int
 ) -> int:
     agency_1 = Agency(
         name="Agency 1",
@@ -164,7 +205,7 @@ async def add_agency(
     # Add Agency location
     agency_1_location = LinkAgencyLocation(
         agency_id=agency_id,
-        location_id=pittsburgh.location_id,
+        location_id=location_id,
     )
     await adb_client.add(agency_1_location)
     return agency_id