From 59f09b14a0d50a327b14e1d515e779581ad67f98 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 12 Sep 2023 17:07:10 -0400 Subject: [PATCH 01/17] Add alarm and map.html --- README.md | 4 ++-- infrastructure/aws/cdk/app.py | 31 +++++++++++++++++++++++++++++++ titiler/xarray/factory.py | 8 +++++++- titiler/xarray/reader.py | 2 +- 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 315c2d8..07bd396 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Tests use data generated locally by using `tests/fixtures/generate_test_*.py` sc To run all the tests: ```bash -python -m pip install -e .["tests"] +python -m pip install -e ".[tests]" python -m pytest --cov titiler.xarray --cov-report term-missing -s -vv ``` @@ -80,7 +80,7 @@ An example of Cloud Stack is available for AWS STACK_STAGE=staging npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-staging # Deploy in specific region - AWS_DEFAULT_REGION=eu-central-1 AWS_REGION=eu-central-1 AWS_PROFILE=myprofile STACK_STAGE=staging npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-staging + AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=smce-veda STACK_STAGE=production npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-production ``` diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index 57072be..a28871b 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -9,6 +9,12 @@ from aws_cdk import aws_lambda from aws_cdk import aws_logs as logs from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration +from aws_cdk import ( + aws_cloudwatch as cloudwatch, + aws_sns as sns, + aws_sns_subscriptions as subscriptions, + aws_cloudwatch_actions as cloudwatch_actions, +) from config import StackSettings from constructs import Construct @@ -77,6 +83,31 @@ def __init__( f"{id}-integration", lambda_function ), ) + + # Create an SNS Topic + topic = sns.Topic(self, "DevTitilerXarray500Errors", + display_name="Dev TitilerXarray Gateway 500 Errors", + topic_name="DevTitilerXarray500Errors") + # Subscribe email to the topic + email_address = "aimee@developmentseed.org" + topic.add_subscription(subscriptions.EmailSubscription(email_address)) + + # Create CloudWatch Alarm + alarm = cloudwatch.Alarm(self, "MyAlarm", + metric=cloudwatch.Metric( + namespace="AWS/ApiGateway", + metric_name="5XXError", + dimensions_map={ + "ApiName": f"{id}-endpoint" + }, + period=Duration.minutes(1)), + evaluation_periods=1, + threshold=1, + alarm_description="Alarm if 500 errors are detected", + alarm_name="ApiGateway500Alarm", + actions_enabled=True + ) + alarm.add_alarm_action(cloudwatch_actions.SnsAction(topic)) CfnOutput(self, "Endpoint", value=api.url) diff --git a/titiler/xarray/factory.py b/titiler/xarray/factory.py index 48f34ca..f5f8828 100644 --- a/titiler/xarray/factory.py +++ b/titiler/xarray/factory.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from typing import Dict, List, Literal, Optional, Tuple, Type from urllib.parse import urlencode +from starlette.templating import Jinja2Templates +import jinja2 from fastapi import Depends, Path, Query from rio_tiler.models import Info @@ -384,7 +386,11 @@ def map_viewer( tilejson_url += f"?{urlencode(request.query_params._list)}" tms = self.supported_tms.get(TileMatrixSetId) - return self.templates.TemplateResponse( + templates = Jinja2Templates( + directory="", + loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, ".")]), + ) + return templates.TemplateResponse( name="map.html", context={ "request": request, diff --git a/titiler/xarray/reader.py b/titiler/xarray/reader.py index f4fb663..bcde5be 100644 --- a/titiler/xarray/reader.py +++ b/titiler/xarray/reader.py @@ -18,7 +18,7 @@ def xarray_open_dataset( src_path: str, group: Optional[Any] = None, reference: Optional[bool] = False, - decode_times: Optional[bool] = True, + decode_times: Optional[bool] = False, ) -> xarray.Dataset: """Open dataset.""" xr_open_args: Dict[str, Any] = { From adf95a831be15f19ed7ac6d15e836af4a2ab67cb Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 12 Sep 2023 17:07:32 -0400 Subject: [PATCH 02/17] Use different template --- titiler/xarray/map.html | 120 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 titiler/xarray/map.html diff --git a/titiler/xarray/map.html b/titiler/xarray/map.html new file mode 100644 index 0000000..b7eca39 --- /dev/null +++ b/titiler/xarray/map.html @@ -0,0 +1,120 @@ + + + + + TiTiler Map Viewer + + + + + + + + + +
+ + + + From 434c82f03dfcedf1f6d0475c3d7c3c6349c7aaba Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 13 Sep 2023 15:11:48 -0400 Subject: [PATCH 03/17] Update map template --- infrastructure/aws/cdk/app.py | 47 ++++---- infrastructure/aws/cdk/config.py | 1 + titiler/xarray/factory.py | 75 ++++++++---- titiler/xarray/map-form.html | 196 +++++++++++++++++++++++++++++++ titiler/xarray/map.html | 190 +++++++++++++++--------------- 5 files changed, 370 insertions(+), 139 deletions(-) create mode 100644 titiler/xarray/map-form.html diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index a28871b..4500dc9 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -85,29 +85,30 @@ def __init__( ) # Create an SNS Topic - topic = sns.Topic(self, "DevTitilerXarray500Errors", - display_name="Dev TitilerXarray Gateway 500 Errors", - topic_name="DevTitilerXarray500Errors") - # Subscribe email to the topic - email_address = "aimee@developmentseed.org" - topic.add_subscription(subscriptions.EmailSubscription(email_address)) - - # Create CloudWatch Alarm - alarm = cloudwatch.Alarm(self, "MyAlarm", - metric=cloudwatch.Metric( - namespace="AWS/ApiGateway", - metric_name="5XXError", - dimensions_map={ - "ApiName": f"{id}-endpoint" - }, - period=Duration.minutes(1)), - evaluation_periods=1, - threshold=1, - alarm_description="Alarm if 500 errors are detected", - alarm_name="ApiGateway500Alarm", - actions_enabled=True - ) - alarm.add_alarm_action(cloudwatch_actions.SnsAction(topic)) + if settings.alarm_email: + topic = sns.Topic(self, "DevTitilerXarray500Errors", + display_name="Dev TitilerXarray Gateway 500 Errors", + topic_name="DevTitilerXarray500Errors") + # Subscribe email to the topic + email_address = settings.alarm_email + topic.add_subscription(subscriptions.EmailSubscription(email_address)) + + # Create CloudWatch Alarm + alarm = cloudwatch.Alarm(self, "MyAlarm", + metric=cloudwatch.Metric( + namespace="AWS/ApiGateway", + metric_name="5XXError", + dimensions_map={ + "ApiName": f"{id}-endpoint" + }, + period=Duration.minutes(1)), + evaluation_periods=1, + threshold=1, + alarm_description="Alarm if 500 errors are detected", + alarm_name="ApiGateway500Alarm", + actions_enabled=True + ) + alarm.add_alarm_action(cloudwatch_actions.SnsAction(topic)) CfnOutput(self, "Endpoint", value=api.url) diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index ef8690d..4a48fc8 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -32,6 +32,7 @@ class StackSettings(pydantic.BaseSettings): # The maximum of concurrent executions you want to reserve for the function. # Default: - No specific limit - account limit. max_concurrent: Optional[int] + alarm_email: Optional[str] = "" class Config: """model config""" diff --git a/titiler/xarray/factory.py b/titiler/xarray/factory.py index f5f8828..562b182 100644 --- a/titiler/xarray/factory.py +++ b/titiler/xarray/factory.py @@ -18,6 +18,7 @@ from titiler.core.resources.responses import JSONResponse from titiler.xarray.reader import ZarrReader +import numpy as np @dataclass class ZarrTilerFactory(BaseTilerFactory): @@ -318,6 +319,31 @@ def tilejson_endpoint( # type: ignore "tiles": [tiles_url], } + + @self.router.get( + "/histogram", + response_class=JSONResponse, + responses={200: {"description": "Return histogram for this data variable"}}, + response_model_exclude_none=True, + ) + def histogram( + url: str = Query(..., description="Dataset URL"), + variable: str = Query(..., description="Variable"), + ): + with self.reader( + url, + variable=variable, + ) as src_dst: + boolean_mask = ~np.isnan(src_dst.input) + data_values = src_dst.input.values[boolean_mask] + counts, values = np.histogram(data_values, bins=10) + counts, values = counts.tolist(), values.tolist() + buckets = list(zip(values, [values[i+1] for i in range(len(values)-1)])) + hist_dict = [] + for idx, bucket in enumerate(buckets): + hist_dict.append({"bucket": bucket, "value": counts[idx]}) + return hist_dict + @self.router.get("/map", response_class=HTMLResponse) @self.router.get("/{TileMatrixSetId}/map", response_class=HTMLResponse) def map_viewer( @@ -326,7 +352,7 @@ def map_viewer( self.default_tms, description=f"TileMatrixSet Name (default: '{self.default_tms}')", ), # noqa - url: str = Query(..., description="Dataset URL"), # noqa + url: Optional[str] = Query(None, description="Dataset URL"), # noqa group: Optional[int] = Query( # noqa None, description="Select a specific Zarr Group (Zoom Level)." ), @@ -343,7 +369,7 @@ def map_viewer( decode_times: Optional[bool] = Query( # noqa True, title="decode_times", description="Whether to decode times" ), - variable: str = Query(..., description="Xarray Variable"), # noqa + variable: Optional[str] = Query(None, description="Xarray Variable"), # noqa drop_dim: Optional[str] = Query( None, description="Dimension to drop" ), # noqa @@ -379,24 +405,33 @@ def map_viewer( env=Depends(self.environment_dependency), # noqa ): """Return map Viewer.""" - tilejson_url = self.url_for( - request, "tilejson_endpoint", TileMatrixSetId=TileMatrixSetId - ) - if request.query_params._list: - tilejson_url += f"?{urlencode(request.query_params._list)}" - - tms = self.supported_tms.get(TileMatrixSetId) templates = Jinja2Templates( directory="", loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, ".")]), - ) - return templates.TemplateResponse( - name="map.html", - context={ - "request": request, - "tilejson_endpoint": tilejson_url, - "tms": tms, - "resolutions": [tms._resolution(matrix) for matrix in tms], - }, - media_type="text/html", - ) + ) + if url: + tilejson_url = self.url_for( + request, "tilejson_endpoint", TileMatrixSetId=TileMatrixSetId + ) + if request.query_params._list: + tilejson_url += f"?{urlencode(request.query_params._list)}" + + tms = self.supported_tms.get(TileMatrixSetId) + return templates.TemplateResponse( + name="map.html", + context={ + "request": request, + "tilejson_endpoint": tilejson_url, + "tms": tms, + "resolutions": [tms._resolution(matrix) for matrix in tms], + }, + media_type="text/html", + ) + else: + return templates.TemplateResponse( + name="map-form.html", + context={ + "request": request, + }, + media_type="text/html", + ) \ No newline at end of file diff --git a/titiler/xarray/map-form.html b/titiler/xarray/map-form.html new file mode 100644 index 0000000..df3c4e6 --- /dev/null +++ b/titiler/xarray/map-form.html @@ -0,0 +1,196 @@ + + + + + TiTiler Map Viewer + + + + + + + + + + + + + + + diff --git a/titiler/xarray/map.html b/titiler/xarray/map.html index b7eca39..4c3ca46 100644 --- a/titiler/xarray/map.html +++ b/titiler/xarray/map.html @@ -1,10 +1,12 @@ - + TiTiler Map Viewer - - + + @@ -14,107 +16,103 @@ + -
+ + L.tileLayer( + data.tiles[0], { + minZoom: data.minzoom, + maxNativeZoom: data.maxzoom, + bounds: L.latLngBounds([bottom, left], [top, right]), + } + ).addTo(map); + }) + .catch(err => { + console.warn(err) + }) + - + From 55967b73b880b3066ea7dae55a6fa30a50c57a34 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 13 Sep 2023 18:13:08 -0400 Subject: [PATCH 04/17] Working on variables --- titiler/xarray/map-form.html | 91 ++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/titiler/xarray/map-form.html b/titiler/xarray/map-form.html index df3c4e6..20621e2 100644 --- a/titiler/xarray/map-form.html +++ b/titiler/xarray/map-form.html @@ -61,7 +61,7 @@ } /* Button styling */ - #show-popup-btn, #go-btn, #close-popup-btn, #get-stats-btn { + #show-popup-btn, #go-btn, #close-popup-btn, #get-histogram-btn, #get-vars-btn { background-color: #007bff; color: white; border: none; @@ -71,12 +71,12 @@ cursor: pointer; } - #show-popup-btn:hover, #go-btn:hover, #close-popup-btn:hover, #get-stats-btn:hover { + #show-popup-btn:hover, #go-btn:hover, #close-popup-btn:hover, #get-histogram-btn:hover { background-color: #0056b3; } - /* Stats container */ - #stats-container { + /* histogram container */ + #histogram-container { margin-top: 20px; text-align: left; /* border: 1px solid #ccc; */ @@ -93,21 +93,33 @@ @@ -121,29 +133,28 @@

Enter URL

option.value = item; option.text = item; colormapDropdown.add(option); - }); - // Go button action - document.getElementById("go-btn").addEventListener("click", function() { + }); + + // Get vars button action + document.getElementById("get-vars-btn").addEventListener("click", function() { + document.getElementById('vars-container').innerHTML = ''; var urlValue = document.getElementById("url-input").value; - var variableValue = document.getElementById("variable-input").value; - var rescaleValue = document.getElementById("rescale-input").value; - var colormapValue = document.getElementById("colormap-dropdown").value; - if (urlValue) { - // Redirect to the map URL with the input value as a parameter - var href = `map?url=${encodeURIComponent(urlValue)}&variable=${encodeURIComponent(variableValue)}`; - if (rescaleValue) { - href += `&rescale=${encodeURIComponent(rescaleValue)}`; - } - if (colormapValue) { - href += `&colormap_name=${encodeURIComponent(colormapValue)}`; - } - window.location.href = href; - } + // Fetch data from a URL + fetch(`/variables?url=${encodeURIComponent(urlValue)}`) // Replace with the actual URL + .then(response => response.json()) // Assuming server responds with JSON + .then(data => { + // Create a new div and add the fetched data + var newDiv = document.createElement("div"); + newDiv.textContent = JSON.stringify(data, null, 2); // Formatting the JSON data for demonstration + document.getElementById("vars-container").appendChild(newDiv); + }) + .catch((error) => { + console.error('Error fetching data:', error); + }); }); - // Get stats button action - document.getElementById("get-stats-btn").addEventListener("click", function() { - //document.getElementById('stats-container').innerHTML = ''; + // Get histogram button action + document.getElementById("get-histogram-btn").addEventListener("click", function() { var urlValue = document.getElementById("url-input").value; var variableValue = document.getElementById("variable-input").value; @@ -184,12 +195,32 @@

Enter URL

x += 50; // Increment x position for the next label }); - //document.getElementById("stats-container").appendChild(newDiv); + //document.getElementById("histogram-container").appendChild(newDiv); }) .catch((error) => { console.error('Error fetching data:', error); }); }); + + + // Go button action + document.getElementById("go-btn").addEventListener("click", function() { + var urlValue = document.getElementById("url-input").value; + var variableValue = document.getElementById("variable-input").value; + var rescaleValue = document.getElementById("rescale-input").value; + var colormapValue = document.getElementById("colormap-dropdown").value; + if (urlValue) { + // Redirect to the map URL with the input value as a parameter + var href = `map?url=${encodeURIComponent(urlValue)}&variable=${encodeURIComponent(variableValue)}`; + if (rescaleValue) { + href += `&rescale=${encodeURIComponent(rescaleValue)}`; + } + if (colormapValue) { + href += `&colormap_name=${encodeURIComponent(colormapValue)}`; + } + window.location.href = href; + } + }); From 44081a6e4e2b16b4cf676024d7938841dcfb47a0 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 14 Sep 2023 15:21:22 -0400 Subject: [PATCH 05/17] Add map form --- titiler/xarray/factory.py | 8 +- titiler/xarray/map-form.html | 157 +++++++++++++++++++++++++++-------- titiler/xarray/map.html | 71 ++++++++-------- 3 files changed, 165 insertions(+), 71 deletions(-) diff --git a/titiler/xarray/factory.py b/titiler/xarray/factory.py index 562b182..3693023 100644 --- a/titiler/xarray/factory.py +++ b/titiler/xarray/factory.py @@ -187,7 +187,6 @@ def tiles_endpoint( # type: ignore if colormap: image = image.apply_colormap(colormap) - if not format: format = ImageType.jpeg if image.mask.all() else ImageType.png @@ -312,6 +311,7 @@ def tilejson_endpoint( # type: ignore [-180, -90, 180, 90], list(src_dst.geographic_bounds) ) bounds = [max(minx), max(miny), min(maxx), min(maxy)] + return { "bounds": bounds, "minzoom": minzoom if minzoom is not None else src_dst.minzoom, @@ -329,10 +329,16 @@ def tilejson_endpoint( # type: ignore def histogram( url: str = Query(..., description="Dataset URL"), variable: str = Query(..., description="Variable"), + reference: Optional[bool] = Query( + False, + title="reference", + description="Whether the src_path is a kerchunk reference", + ), ): with self.reader( url, variable=variable, + reference=reference ) as src_dst: boolean_mask = ~np.isnan(src_dst.input) data_values = src_dst.input.values[boolean_mask] diff --git a/titiler/xarray/map-form.html b/titiler/xarray/map-form.html index 20621e2..a7bf358 100644 --- a/titiler/xarray/map-form.html +++ b/titiler/xarray/map-form.html @@ -33,22 +33,22 @@ background-color: rgba(0,0,0,0.7); display: flex; justify-content: center; - align-items: center; } /* Updated content box */ #popup-content { background-color: #fff; width: 50%; - padding: 20px; - text-align: center; + padding: 30px; + text-align: left; border-radius: 12px; box-shadow: 0 4px 8px rgba(0,0,0,0.1); z-index: 1000; + overflow-y: scroll; } /* Input field styling */ - #url-input, #variable-input, #rescale-input, #colormap-dropdown { + #url-input, #variable-input, #rescale-input, #colormap-dropdown, #variable-select { width: 80%; padding: 10px; margin: 10px 0; @@ -56,7 +56,7 @@ border-radius: 4px; } - #colormap-dropdown { + #colormap-dropdown, #variable-select { width: 84.5%; } @@ -85,7 +85,30 @@ overflow-y: auto; max-height: 200px; } - + .spinner { + display: inline-block; + width: 20px; + height: 20px; + } + .spinner:after { + content: " "; + display: block; + width: 20px; + height: 20px; + margin: 3px; + border-radius: 50%; + border: 3px solid #666; + border-color: #666 transparent #666 transparent; + animation: spinner 1.2s linear infinite; + } + @keyframes spinner { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } + } @@ -95,35 +118,53 @@
-

Enter URL

+

Visualize Zarr

+

Step 1: Enter the URL of your Zarr store


+ + +
+ +

Step 2: Enter the variable you wish to visualize.

+

If you already know the variable name, type it in the input box. Otherwise, click on the "Get Variables" button to populate a drop down.

-
-
+ +

+

Step 3: Enter the rescale values.

+

By default, data values will be represented as they fall in the 0,255 RGB range. This won't work for most data. If data values are very small, ranging from 0 to 1 like many climate indices, you will see a big black box. Conversely, if most of your data is greater than 255, your data will appear white. It makes sense to rescale your data via the rescale parameter. If your data ranges from 0,1 that should be your rescale value.

+

You can enter the rescale value directly or click "Get histogram" to see a histogram of the data values.

+
-
- + - +
+ +
+
+

Step 4: Select a colormap.


- +
+