feat: move to query parameters for pipeline selection (#3136)

#### Motivation

inline parameters in the URL were hard to parse and did not allow for
future expansion of pipeline configuration, by moving to a query
parameter the pipline can be configured easier and has opportunities for
expansion later.

#### Modification

switches from a inline url to a query parameter for pipline selection

#### Checklist

_If not applicable, provide explanation of why._

- [ ] Tests updated
- [ ] Docs updated
- [ ] Issue linked in Title

Author: blacha

packages/_infra/src/edge/index.ts

@@ -103,7 +103,7 @@ export class EdgeStack extends cdk.Stack {
           forwardedValues: {
             /** Forward all query strings but do not use them for caching */
             queryString: true,
-            queryStringCacheKeys: ['config', 'exclude'].map(encodeURIComponent),
+            queryStringCacheKeys: ['config', 'exclude', 'pipeline'].map(encodeURIComponent),
           },
           lambdaFunctionAssociations: [],
         },
@@ -118,6 +118,7 @@ export class EdgeStack extends cdk.Stack {
               'exclude',
               'tileMatrix',
               'style',
+              'pipeline',
               // Deprecated single character query params for style and projection
               's',
               'p',

packages/config-loader/src/json/tiff.config.ts

@@ -2,6 +2,8 @@ import {
   ConfigImagery,
   ConfigProviderMemory,
   ConfigTileSetRaster,
+  DefaultColorRampOutput,
+  DefaultTerrainRgbOutput,
   ImageryDataType,
   sha256base58,
   TileSetType,
@@ -446,29 +448,11 @@ export async function initConfigFromUrls(
   const elevationTileSet: ConfigTileSetRaster = {
     id: 'ts_elevation',
     name: 'elevation',
-    title: 'Basemaps',
+    title: 'Elevation Basemap',
     category: 'Basemaps',
     type: TileSetType.Raster,
     layers: [],
-    outputs: [
-      {
-        title: 'TerrainRGB',
-        name: 'terrain-rgb',
-        pipeline: [{ type: 'terrain-rgb' }],
-        output: {
-          type: 'webp',
-          lossless: true,
-          background: { r: 1, g: 134, b: 160, alpha: 1 },
-          resizeKernel: { in: 'nearest', out: 'nearest' },
-        },
-      },
-      {
-        title: 'Color ramp',
-        name: 'color-ramp',
-        pipeline: [{ type: 'color-ramp' }],
-        output: { type: 'webp' },
-      },
-    ],
+    outputs: [DefaultTerrainRgbOutput, DefaultColorRampOutput],
   };
 
   provider.put(aerialTileSet);
@@ -489,11 +473,13 @@ export async function initConfigFromUrls(
         elevationTileSet.layers.push(existingLayer);
       }
       existingLayer[cfg.projection] = cfg.id;
-      provider.put(elevationTileSet);
-      tileSets.push(elevationTileSet);
+      if (!provider.objects.has(elevationTileSet.id)) {
+        provider.put(elevationTileSet);
+        tileSets.push(elevationTileSet);
+      }
     }
   }
-  // FIXME: this should return all the tile sets that were created
+  // FIXME: tileSet should be removed now that we are returning all tilesets
   return { tileSet: aerialTileSet, tileSets, imagery: configs };
 }
 

packages/config/src/config/tile.set.output.ts

@@ -0,0 +1,23 @@
+import { ConfigTileSetRasterOutput } from './tile.set.js';
+
+export const DefaultTerrainRgbOutput: ConfigTileSetRasterOutput = {
+  title: 'TerrainRGB',
+  name: 'terrain-rgb',
+  pipeline: [{ type: 'terrain-rgb' }],
+  output: {
+    // terrain rgb cannot be resampled after it has been made
+    lossless: true,
+    // Zero encoded as a TerrainRGB
+    background: { r: 1, g: 134, b: 160, alpha: 1 },
+    resizeKernel: { in: 'nearest', out: 'nearest' },
+  },
+} as const;
+
+export const DefaultColorRampOutput: ConfigTileSetRasterOutput = {
+  title: 'Color ramp',
+  name: 'color-ramp',
+  pipeline: [{ type: 'color-ramp' }],
+  output: {
+    background: { r: 1, g: 134, b: 160, alpha: 1 },
+  },
+} as const;

packages/config/src/config/tile.set.ts

@@ -92,10 +92,20 @@ export interface ConfigTileSetRasterOutput {
    */
   pipeline?: ConfigRasterPipeline[];
 
-  /** Raster output format */
-  output: {
-    /** Output file format to use */
-    type: ImageFormat;
+  /**
+   * Raster output format
+   * if none is provided it is assumed to be a RGBA output allowing all image formats
+   */
+  output?: {
+    /**
+     * Allowed output file format to use
+     *
+     * Will default to all image formats
+     * if "lossless" is set then lossless image formats
+     *
+     * @default ImageFormat[] - All Image formats
+     */
+    type?: ImageFormat[];
     /**
      * should the output be lossless
      *
@@ -111,7 +121,7 @@ export interface ConfigTileSetRasterOutput {
     /**
      * When scaling tiles in the rendering process what kernel to use
      *
-     * will fall back to  {@link ConfigTileSetRaster.background} if not defined
+     * will fall back to {@link ConfigTileSetRaster.background} if not defined
      */
     resizeKernel?: { in: TileResizeKernel; out: TileResizeKernel };
   };

packages/config/src/index.ts

@@ -22,6 +22,7 @@ export {
   TileResizeKernel,
   TileSetType,
 } from './config/tile.set.js';
+export { DefaultColorRampOutput, DefaultTerrainRgbOutput } from './config/tile.set.output.js';
 export { ConfigVectorStyle, Layer, Sources, StyleJson } from './config/vector.style.js';
 export { ConfigBundled, ConfigProviderMemory } from './memory/memory.config.js';
 export { standardizeLayerName } from './name.convertor.js';

packages/config/src/memory/memory.config.ts

@@ -10,6 +10,7 @@ import { ConfigPrefix } from '../config/prefix.js';
 import { ConfigProvider } from '../config/provider.js';
 import { ConfigLayer, ConfigTileSet, ConfigTileSetRaster, TileSetType } from '../config/tile.set.js';
 import { ConfigVectorStyle } from '../config/vector.style.js';
+import { DefaultColorRampOutput, DefaultTerrainRgbOutput } from '../index.js';
 import { standardizeLayerName } from '../name.convertor.js';
 
 interface DuplicatedImagery {
@@ -144,10 +145,15 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
     const layerByName = new Map<string, ConfigLayer>();
     // Set all layers as minZoom:32
     for (const l of layers) {
+      // Ignore any tileset that has defined pipelines
+      const tileSet = this.objects.get(this.TileSet.id(l.name)) as ConfigTileSetRaster;
+      if (tileSet.outputs) continue;
+
       const newLayer = { ...l, minZoom: 32 };
       delete newLayer.maxZoom; // max zoom not needed when minzoom is 32
       layerByName.set(newLayer.name, { ...layerByName.get(l.name), ...newLayer });
     }
+
     const allTileset: ConfigTileSet = {
       type: TileSetType.Raster,
       id: 'ts_all',
@@ -180,14 +186,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
 
       // FIXME: should we store output types here
       if (i.bands?.length === 1) {
-        existing.outputs = [
-          {
-            title: 'Color ramp',
-            name: 'color-ramp',
-            pipeline: [{ type: 'color-ramp' }],
-            output: { type: 'webp' },
-          },
-        ];
+        existing.outputs = [DefaultTerrainRgbOutput, DefaultColorRampOutput];
       }
     }
     // The latest imagery overwrite the earlier ones.
@@ -219,14 +218,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
 
     // FIXME: should we store output types here
     if (i.bands?.length === 1) {
-      ts.outputs = [
-        {
-          title: 'Color ramp',
-          name: 'color-ramp',
-          pipeline: [{ type: 'color-ramp' }],
-          output: { type: 'webp' },
-        },
-      ];
+      ts.outputs = [DefaultTerrainRgbOutput, DefaultColorRampOutput];
     }
     return ts;
   }

packages/lambda-tiler/src/cli/render.preview.ts

@@ -39,7 +39,7 @@ async function main(): Promise<void> {
     tileSet,
     location,
     z,
-    output: { title: outputFormat, output: { type: outputFormat }, name: 'rgba' },
+    output: { title: outputFormat, output: { type: [outputFormat] }, name: 'rgba' },
   });
   const previewFile = fsa.toUrl(`./z${z}_${location.lon}_${location.lat}.${outputFormat}`);
   await fsa.write(previewFile, Buffer.from(res.body, 'base64'));

packages/lambda-tiler/src/index.ts

@@ -73,7 +73,7 @@ handler.router.get('/v1/tiles/:tileSet/:tileMatrix/style/:styleName.json', style
 handler.router.get('/v1/tiles/:tileSet/:tileMatrix/tile.json', tileJsonGet);
 
 // Tiles
-handler.router.get('/v1/tiles/:tileSet/:tileMatrix/:z(^\\d+)/:x(^\\d+)/:y(^\\d+):tileType', tileXyzGet);
+handler.router.get('/v1/tiles/:tileSet/:tileMatrix/:z/:x/:y.:tileType', tileXyzGet);
 
 // Preview
 handler.router.get('/v1/preview/:tileSet/:tileMatrix/:z/:lon/:lat', tilePreviewGet);

packages/lambda-tiler/src/routes/__tests__/xyz.test.ts

@@ -6,7 +6,7 @@ import { LogConfig } from '@basemaps/shared';
 import { round } from '@basemaps/test/build/rounding.js';
 
 import { FakeData } from '../../__tests__/config.data.js';
-import { Api, mockRequest } from '../../__tests__/xyz.util.js';
+import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
 import { handler } from '../../index.js';
 import { ConfigLoader } from '../../util/config.loader.js';
 import { Etag } from '../../util/etag.js';
@@ -53,8 +53,8 @@ describe('/v1/tiles', () => {
 
     const request = mockRequest('/v1/tiles/aerial/3857/0/0/0.webp', 'get', Api.header);
     const res = await handler.router.handle(request);
-    console.log(res.statusDescription);
-    assert.equal(res.status, 200);
+
+    assert.equal(res.status, 200, res.statusDescription);
     assert.equal(res.header('content-type'), 'image/webp');
     assert.equal(res.header('eTaG'), 'fakeEtag');
     // o(res.body).equals(rasterMockBuffer.toString('base64'));
@@ -68,13 +68,13 @@ describe('/v1/tiles', () => {
     it(`should 200 with empty ${fmt} if a tile is out of bounds`, async (t) => {
       t.mock.method(ConfigLoader, 'getDefaultConfig', () => Promise.resolve(config));
 
-      const res = await handler.router.handle(
-        mockRequest(`/v1/tiles/aerial/global-mercator/0/0/0.${fmt}`, 'get', Api.header),
-      );
-      assert.equal(res.status, 200);
+      const request = mockRequest(`/v1/tiles/aerial/global-mercator/0/0/0.${fmt}`, 'get', Api.header);
+      const res = await handler.router.handle(request);
+      assert.equal(res.status, 200, res.statusDescription);
       assert.equal(res.header('content-type'), `image/${fmt}`);
       assert.notEqual(res.header('etag'), undefined);
       assert.equal(res.header('cache-control'), 'public, max-age=604800, stale-while-revalidate=86400');
+      assert.deepEqual(request.logContext['pipeline'], 'rgba');
     });
   });
 
@@ -114,7 +114,7 @@ describe('/v1/tiles', () => {
     const req = mockRequest('/v1/tiles/🦄 🌈/global-mercator/0/0/0.png', 'get', Api.header);
     assert.equal(req.path, '/v1/tiles/%F0%9F%A6%84%20%F0%9F%8C%88/global-mercator/0/0/0.png');
     const res = await handler.router.handle(req);
-    assert.equal(res.status, 200);
+    assert.equal(res.status, 200, res.statusDescription);
     assert.equal(res.header('content-type'), 'image/png');
     assert.notEqual(res.header('etag'), undefined);
     assert.equal(res.header('cache-control'), 'public, max-age=604800, stale-while-revalidate=86400');
@@ -129,21 +129,52 @@ describe('/v1/tiles', () => {
     });
   });
 
+  it('should 404 if pipelines are defined but one is not requested', async (t) => {
+    t.mock.method(ConfigLoader, 'getDefaultConfig', () => Promise.resolve(config));
+
+    const elevation = FakeData.tileSetRaster('elevation');
+
+    elevation.outputs = [{ title: 'Terrain RGB', name: 'terrain-rgb', output: { lossless: true } }];
+    config.put(elevation);
+
+    const request = mockRequest('/v1/tiles/elevation/3857/11/2022/1283.webp', 'get', Api.header);
+
+    const res = await handler.router.handle(request);
+
+    assert.equal(res.status, 404, res.statusDescription);
+  });
+
   it('should generate a terrain-rgb 11/2022/1283 in webp', async (t) => {
     t.mock.method(ConfigLoader, 'getDefaultConfig', () => Promise.resolve(config));
 
     const elevation = FakeData.tileSetRaster('elevation');
 
-    elevation.outputs = [{ title: 'Terrain RGB', name: 'terrain-rgb', output: { type: 'webp', lossless: true } }];
+    elevation.outputs = [{ title: 'Terrain RGB', name: 'terrain-rgb', output: { lossless: true } }];
     config.put(elevation);
 
-    const request = mockRequest('/v1/tiles/elevation/3857/11/2022/1283-terrain-rgb.webp', 'get', Api.header);
+    const request = mockUrlRequest('/v1/tiles/elevation/3857/11/2022/1283.webp', '?pipeline=terrain-rgb', Api.header);
 
     const res = await handler.router.handle(request);
 
-    assert.equal(res.status, 200);
+    assert.equal(res.status, 200, res.statusDescription);
     // Validate the session information has been set correctly
     assert.deepEqual(request.logContext['xyz'], { x: 2022, y: 1283, z: 11 });
+    assert.deepEqual(request.logContext['pipeline'], 'terrain-rgb');
     assert.deepEqual(round(request.logContext['location']), { lat: -41.44272638, lon: 175.51757812 });
   });
+
+  it('should validate lossless if pipelines are defined but one is not requested', async (t) => {
+    t.mock.method(ConfigLoader, 'getDefaultConfig', () => Promise.resolve(config));
+
+    const elevation = FakeData.tileSetRaster('elevation');
+
+    elevation.outputs = [{ title: 'Terrain RGB', name: 'terrain-rgb', output: { lossless: true } }];
+    config.put(elevation);
+
+    // JPEG is not lossless
+    const res = await handler.router.handle(
+      mockUrlRequest('/v1/tiles/elevation/3857/11/2022/1283.jpeg', '?pipeline=terrain-rgb', Api.header),
+    );
+    assert.equal(res.status, 400, res.statusDescription);
+  });
 });