fix: refactor to keep database pgstac and ingestor pypgstac in sync (#137)

* fix: refactor to keep database pgstac and ingestor pypgstac in sync

* add default value for optional jwks_url settings parameter

* add ingestor with no authentication to integration test deployment

Co-authored-by: Jamison French <[email protected]>

---------

Co-authored-by: Jamison French <[email protected]>

Author: hrodmn

integration_tests/cdk/app.py

@@ -1,13 +1,16 @@
-from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_rds
+from aws_cdk import App, RemovalPolicy, Stack, aws_ec2, aws_iam, aws_rds
 from config import AppConfig, build_app_config
 from constructs import Construct
 from eoapi_cdk import (
     PgStacApiLambda,
     PgStacDatabase,
+    StacIngestor,
     TiPgApiLambda,
     TitilerPgstacApiLambda,
 )
 
+PGSTAC_VERSION = "0.9.5"
+
 
 class VpcStack(Stack):
     def __init__(
@@ -81,7 +84,7 @@ def __init__(
             instance_type=aws_ec2.InstanceType(app_config.db_instance_type),
             add_pgbouncer=True,
             removal_policy=RemovalPolicy.DESTROY,
-            pgstac_version="0.9.5",
+            pgstac_version=PGSTAC_VERSION,
         )
 
         assert pgstac_db.security_group
@@ -137,6 +140,31 @@ def __init__(
             },
         )
 
+        s3_read_only_role = aws_iam.Role(
+            self,
+            "S3ReadOnlyRole",
+            assumed_by=aws_iam.ServicePrincipal("lambda.amazonaws.com"),
+            description="Role with read-only access to S3 buckets",
+        )
+
+        s3_read_only_role.add_managed_policy(
+            aws_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3ReadOnlyAccess")
+        )
+
+        StacIngestor(
+            self,
+            "ingestor",
+            data_access_role=s3_read_only_role,
+            stac_db_secret=pgstac_db.pgstac_secret,
+            stac_db_security_group=pgstac_db.security_group,
+            stac_url=stac_api.url,
+            stage="test",
+            pgstac_version=PGSTAC_VERSION,
+            api_env={
+                "JWKS_URL": "",  # no authentication!
+            },
+        )
+
 
 app = App()
 

lib/database/index.ts

@@ -10,11 +10,10 @@ import {
   aws_logs,
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
-import { CustomLambdaFunctionProps } from "../utils";
+import { CustomLambdaFunctionProps, DEFAULT_PGSTAC_VERSION } from "../utils";
 import { PgBouncer } from "./PgBouncer";
 
 const instanceSizes: Record<string, number> = require("./instance-memory.json");
-const DEFAULT_PGSTAC_VERSION = "0.9.5";
 
 let defaultPgSTACCustomOptions: { [key: string]: any } = {
   context: "FALSE",

lib/ingestor-api/index.ts

@@ -13,7 +13,7 @@ import {
   Stack,
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
-import { CustomLambdaFunctionProps } from "../utils";
+import { CustomLambdaFunctionProps, DEFAULT_PGSTAC_VERSION } from "../utils";
 
 export class StacIngestor extends Construct {
   table: dynamodb.Table;
@@ -39,10 +39,10 @@ export class StacIngestor extends Construct {
       assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
       managedPolicies: [
         iam.ManagedPolicy.fromAwsManagedPolicyName(
-          "service-role/AWSLambdaBasicExecutionRole",
+          "service-role/AWSLambdaBasicExecutionRole"
         ),
         iam.ManagedPolicy.fromAwsManagedPolicyName(
-          "service-role/AWSLambdaVPCAccessExecutionRole",
+          "service-role/AWSLambdaVPCAccessExecutionRole"
         ),
       ],
     });
@@ -57,6 +57,7 @@ export class StacIngestor extends Construct {
       dbSecurityGroup: props.stacDbSecurityGroup,
       subnetSelection: props.subnetSelection,
       lambdaFunctionOptions: props.apiLambdaFunctionOptions,
+      pgstacVersion: props.pgstacVersion,
     });
 
     this.buildApiEndpoint({
@@ -74,7 +75,8 @@ export class StacIngestor extends Construct {
       dbVpc: props.vpc,
       dbSecurityGroup: props.stacDbSecurityGroup,
       subnetSelection: props.subnetSelection,
-      lambdaFunctionOptions: props.ingestorLambdaFunctionOptions
+      lambdaFunctionOptions: props.ingestorLambdaFunctionOptions,
+      pgstacVersion: props.pgstacVersion,
     });
 
     this.registerSsmParameter({
@@ -110,20 +112,23 @@ export class StacIngestor extends Construct {
     dbSecret: secretsmanager.ISecret;
     dbVpc: undefined | ec2.IVpc;
     dbSecurityGroup: ec2.ISecurityGroup;
-    subnetSelection: undefined | ec2.SubnetSelection
+    subnetSelection: undefined | ec2.SubnetSelection;
     lambdaFunctionOptions?: CustomLambdaFunctionProps;
+    pgstacVersion?: string;
   }): lambda.Function {
-
     const handler = new lambda.Function(this, "api-handler", {
       // defaults
       runtime: lambda.Runtime.PYTHON_3_11,
       handler: "src.handler.handler",
       memorySize: 2048,
       logRetention: aws_logs.RetentionDays.ONE_WEEK,
       timeout: Duration.seconds(30),
-      code:lambda.Code.fromDockerBuild(__dirname, {
+      code: lambda.Code.fromDockerBuild(__dirname, {
         file: "runtime/Dockerfile",
-        buildArgs: { PYTHON_VERSION: '3.11' },
+        buildArgs: {
+          PYTHON_VERSION: "3.11",
+          PGSTAC_VERSION: props.pgstacVersion || DEFAULT_PGSTAC_VERSION,
+        },
       }),
       allowPublicSubnet: true,
       vpc: props.dbVpc,
@@ -139,7 +144,7 @@ export class StacIngestor extends Construct {
 
     // Allow handler to connect to DB
 
-    if (props.dbVpc){
+    if (props.dbVpc) {
       props.dbSecurityGroup.addIngressRule(
         handler.connections.securityGroups[0],
         ec2.Port.tcp(5432),
@@ -160,10 +165,9 @@ export class StacIngestor extends Construct {
     dbSecurityGroup: ec2.ISecurityGroup;
     subnetSelection: undefined | ec2.SubnetSelection;
     lambdaFunctionOptions?: CustomLambdaFunctionProps;
+    pgstacVersion?: string;
   }): lambda.Function {
-
-
-    const handler = new lambda.Function(this, "stac-ingestor",{
+    const handler = new lambda.Function(this, "stac-ingestor", {
       // defaults
       runtime: lambda.Runtime.PYTHON_3_11,
       handler: "src.ingestor.handler",
@@ -172,7 +176,10 @@ export class StacIngestor extends Construct {
       timeout: Duration.seconds(180),
       code: lambda.Code.fromDockerBuild(__dirname, {
         file: "runtime/Dockerfile",
-        buildArgs: { PYTHON_VERSION: '3.11' },
+        buildArgs: {
+          PYTHON_VERSION: "3.11",
+          PGSTAC_VERSION: props.pgstacVersion || DEFAULT_PGSTAC_VERSION,
+        },
       }),
       vpc: props.dbVpc,
       vpcSubnets: props.subnetSelection,
@@ -187,15 +194,15 @@ export class StacIngestor extends Construct {
     props.dbSecret.grantRead(handler);
 
     // Allow handler to connect to DB
-    if (props.dbVpc){
+    if (props.dbVpc) {
       props.dbSecurityGroup.addIngressRule(
         handler.connections.securityGroups[0],
         ec2.Port.tcp(5432),
         "Allow connections from STAC Ingestor"
       );
     }
 
-    // Allow handler to write results back to DBƒ
+    // Allow handler to write results back to DB
     props.table.grantWriteData(handler);
 
     // Trigger handler from writes to DynamoDB table
@@ -221,7 +228,6 @@ export class StacIngestor extends Construct {
     endpointConfiguration?: apigateway.EndpointConfiguration;
     ingestorDomainNameOptions?: apigateway.DomainNameOptions;
   }): apigateway.LambdaRestApi {
-
     return new apigateway.LambdaRestApi(
       this,
       `${Stack.of(this).stackName}-ingestor-api`,
@@ -236,10 +242,12 @@ export class StacIngestor extends Construct {
         endpointConfiguration: props.endpointConfiguration,
         policy: props.policy,
 
-        domainName:  props.ingestorDomainNameOptions ? {
-          domainName: props.ingestorDomainNameOptions.domainName,
-          certificate: props.ingestorDomainNameOptions.certificate,
-        } : undefined,
+        domainName: props.ingestorDomainNameOptions
+          ? {
+              domainName: props.ingestorDomainNameOptions.domainName,
+              certificate: props.ingestorDomainNameOptions.certificate,
+            }
+          : undefined,
       }
     );
   }
@@ -316,20 +324,26 @@ export interface StacIngestorProps {
   /**
    * Custom Domain Name Options for Ingestor API
    */
-   readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions;
+  readonly ingestorDomainNameOptions?: apigateway.DomainNameOptions;
 
   /**
-     * Can be used to override the default lambda function properties.
-     *
-     * @default - default settings are defined in the construct.
-     */
+   * Can be used to override the default lambda function properties.
+   *
+   * @default - default settings are defined in the construct.
+   */
   readonly apiLambdaFunctionOptions?: CustomLambdaFunctionProps;
 
   /**
-     * Can be used to override the default lambda function properties.
-     *
-     * @default - default settings are defined in the construct.
-     */
-readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionProps;
+   * Can be used to override the default lambda function properties.
+   *
+   * @default - default settings are defined in the construct.
+   */
+  readonly ingestorLambdaFunctionOptions?: CustomLambdaFunctionProps;
 
+  /**
+   * pgstac version - must match the version installed on the pgstac database
+   *
+   * @default - default settings are defined in the construct
+   */
+  readonly pgstacVersion?: string;
 }

lib/ingestor-api/runtime/Dockerfile

@@ -1,4 +1,5 @@
 ARG PYTHON_VERSION
+
 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION}
 
 WORKDIR /tmp
@@ -8,7 +9,10 @@ RUN yum install -y git
 RUN python -m pip install pip -U
 
 COPY runtime/requirements.txt requirements.txt
-RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
+
+ARG PGSTAC_VERSION
+RUN echo "pypgstac==${PGSTAC_VERSION}" > constraints.txt
+RUN python -m pip install -r requirements.txt -c constraints.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic
 
 RUN mkdir -p /asset/src
 COPY runtime/src/*.py /asset/src/

lib/ingestor-api/runtime/requirements.txt

@@ -5,6 +5,6 @@ orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
 pydantic_ssm_settings>=1.0,<2.0
 pydantic>=1.9.0
-pypgstac==0.8.5
+pypgstac
 requests>=2.27.1
 stac-pydantic>=3.0,<4.0

lib/ingestor-api/runtime/src/config.py

@@ -22,7 +22,8 @@ class Settings(BaseSettings):
     root_path: Optional[str] = Field(description="Path from where to serve this URL.")
 
     jwks_url: Optional[HttpUrlString] = Field(
-        description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json"  # noqa
+        default=None,
+        description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json",  # noqa
     )
 
     stac_url: HttpUrlString = Field(description="URL of STAC API")

lib/utils/index.ts

@@ -1,5 +1,4 @@
-import {
-    aws_lambda as lambda,
-  } from "aws-cdk-lib";
+import { aws_lambda as lambda } from "aws-cdk-lib";
 
 export type CustomLambdaFunctionProps = lambda.FunctionProps | any;
+export const DEFAULT_PGSTAC_VERSION = "0.9.5";