"Cacheability" option for critical REST resources (#3335)

* Closes #3329.

This implements a new optional `cacheable` parameter for these REST endpoints:
- `/rest/items`
- `/rest/things`
- `/rest/rules`

When this parameter is set, a flat list of all elements excluding
non-cacheable fields (e.g. "state", "transformedState", "stateDescription",
"commandDescription" for items, "statusInfo", "firmwareStatus",
"properties" for things, "status" for rules) will be retrieved along with
a `Last-Modified` HTTP response header. When unknown, the Last-Modified
header will be set to the date of the request.

Also only when this parameter is set, and a `If-Modified-Since` header is
found in the request, that header will be compared to the last known
modified date for the corresponding cacheable list. The last modified date
will be reset when any change is made on the elements of the underlying
registry. If the `If-Modified-Since` date is equal or more recent than the
last modified date, then a 304 Not Modified response with no content will
be served instead of the usual 200 OK, informing the client that its
cache is still valid at the provided date.

All other request parameters will be ignored except for "metadata" in the
`/rest/items` endpoint. When a metadata selector is set, the resulting
item list will be considered like a completely different resource, i.e.
it will have its own last modified date. Regarding metadata, the approach
to invalidating last modified dates is very conservative: when any metadata
is changed, all cacheable lists of items will have their last modified date
reset even if the change was in a metadata namespace that wasn't requested.

This also implements the abovedescribed behavior for the
`/rest/ui/components/{namespace}` endpoint, but no `cacheable` parameter
is necessary. The last modified date is tracked by namespace.

Signed-off-by: Yannick Schaus <[email protected]>

Author: ghys

bundles/org.openhab.core.automation.rest/src/main/java/org/openhab/core/automation/rest/internal/RuleResource.java

@@ -17,9 +17,11 @@
 import java.io.IOException;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Predicate;
@@ -29,15 +31,18 @@
 import javax.annotation.security.RolesAllowed;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Request;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 import javax.ws.rs.core.SecurityContext;
@@ -69,6 +74,7 @@
 import org.openhab.core.automation.rest.internal.dto.EnrichedRuleDTOMapper;
 import org.openhab.core.automation.util.ModuleBuilder;
 import org.openhab.core.automation.util.RuleBuilder;
+import org.openhab.core.common.registry.RegistryChangeListener;
 import org.openhab.core.config.core.ConfigUtil;
 import org.openhab.core.config.core.Configuration;
 import org.openhab.core.events.Event;
@@ -80,6 +86,7 @@
 import org.openhab.core.library.types.DateTimeType;
 import org.osgi.service.component.annotations.Activate;
 import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
 import org.osgi.service.component.annotations.Reference;
 import org.osgi.service.jaxrs.whiteboard.JaxrsWhiteboardConstants;
 import org.osgi.service.jaxrs.whiteboard.propertytypes.JSONRequired;
@@ -128,8 +135,10 @@ public class RuleResource implements RESTResource {
     private final RuleManager ruleManager;
     private final RuleRegistry ruleRegistry;
     private final ManagedRuleProvider managedRuleProvider;
+    private final ResetLastModifiedChangeListener resetLastModifiedChangeListener = new ResetLastModifiedChangeListener();
 
     private @Context @NonNullByDefault({}) UriInfo uriInfo;
+    private @Nullable Date cacheableListLastModified = null;
 
     @Activate
     public RuleResource( //
@@ -141,20 +150,52 @@ public RuleResource( //
         this.ruleManager = ruleManager;
         this.ruleRegistry = ruleRegistry;
         this.managedRuleProvider = managedRuleProvider;
+
+        this.ruleRegistry.addRegistryChangeListener(resetLastModifiedChangeListener);
+    }
+
+    @Deactivate
+    void deactivate() {
+        this.ruleRegistry.removeRegistryChangeListener(resetLastModifiedChangeListener);
     }
 
     @GET
     @RolesAllowed({ Role.USER, Role.ADMIN })
     @Produces(MediaType.APPLICATION_JSON)
     @Operation(operationId = "getRules", summary = "Get available rules, optionally filtered by tags and/or prefix.", responses = {
             @ApiResponse(responseCode = "200", description = "OK", content = @Content(array = @ArraySchema(schema = @Schema(implementation = EnrichedRuleDTO.class)))) })
-    public Response get(@Context SecurityContext securityContext, @QueryParam("prefix") final @Nullable String prefix,
-            @QueryParam("tags") final @Nullable List<String> tags,
-            @QueryParam("summary") @Parameter(description = "summary fields only") @Nullable Boolean summary) {
+    public Response get(@Context SecurityContext securityContext, @Context Request request,
+            @QueryParam("prefix") final @Nullable String prefix, @QueryParam("tags") final @Nullable List<String> tags,
+            @QueryParam("summary") @Parameter(description = "summary fields only") @Nullable Boolean summary,
+            @DefaultValue("false") @QueryParam("staticDataOnly") @Parameter(description = "provides a cacheable list of values not expected to change regularly and honors the If-Modified-Since header, all other parameters are ignored") boolean staticDataOnly) {
+
         if ((summary == null || !summary) && !securityContext.isUserInRole(Role.ADMIN)) {
             // users may only access the summary
             return JSONResponse.createErrorResponse(Status.UNAUTHORIZED, "Authentication required");
         }
+
+        if (staticDataOnly) {
+            if (cacheableListLastModified != null) {
+                Response.ResponseBuilder responseBuilder = request.evaluatePreconditions(cacheableListLastModified);
+                if (responseBuilder != null) {
+                    // send 304 Not Modified
+                    return responseBuilder.build();
+                }
+            } else {
+                cacheableListLastModified = Date.from(Instant.now().truncatedTo(ChronoUnit.SECONDS));
+            }
+
+            Stream<EnrichedRuleDTO> rules = ruleRegistry.stream()
+                    .map(rule -> EnrichedRuleDTOMapper.map(rule, ruleManager, managedRuleProvider));
+
+            CacheControl cc = new CacheControl();
+            cc.setMustRevalidate(true);
+            cc.setPrivate(true);
+            rules = dtoMapper.limitToFields(rules, "uid,templateUID,name,visibility,description,tags,editable");
+            return Response.ok(new Stream2JSONInputStream(rules)).lastModified(cacheableListLastModified)
+                    .cacheControl(cc).build();
+        }
+
         // match all
         Predicate<Rule> p = r -> true;
 
@@ -567,4 +608,26 @@ public Response setModuleConfigParam(@PathParam("ruleUID") @Parameter(descriptio
             return null;
         }
     }
+
+    private void resetStaticListLastModified() {
+        cacheableListLastModified = null;
+    }
+
+    private class ResetLastModifiedChangeListener implements RegistryChangeListener<Rule> {
+
+        @Override
+        public void added(Rule element) {
+            resetStaticListLastModified();
+        }
+
+        @Override
+        public void removed(Rule element) {
+            resetStaticListLastModified();
+        }
+
+        @Override
+        public void updated(Rule oldElement, Rule element) {
+            resetStaticListLastModified();
+        }
+    }
 }

bundles/org.openhab.core.io.rest.core/src/main/java/org/openhab/core/io/rest/core/internal/item/ItemResource.java

@@ -12,9 +12,12 @@
  */
 package org.openhab.core.io.rest.core.internal.item;
 
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -40,9 +43,11 @@
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.CacheControl;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Request;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.ResponseBuilder;
 import javax.ws.rs.core.Response.Status;
@@ -52,6 +57,7 @@
 import org.eclipse.jdt.annotation.NonNullByDefault;
 import org.eclipse.jdt.annotation.Nullable;
 import org.openhab.core.auth.Role;
+import org.openhab.core.common.registry.RegistryChangeListener;
 import org.openhab.core.events.EventPublisher;
 import org.openhab.core.io.rest.DTOMapper;
 import org.openhab.core.io.rest.JSONResponse;
@@ -68,6 +74,7 @@
 import org.openhab.core.items.ItemBuilderFactory;
 import org.openhab.core.items.ItemNotFoundException;
 import org.openhab.core.items.ItemRegistry;
+import org.openhab.core.items.ItemRegistryChangeListener;
 import org.openhab.core.items.ManagedItemProvider;
 import org.openhab.core.items.Metadata;
 import org.openhab.core.items.MetadataKey;
@@ -88,6 +95,7 @@
 import org.openhab.core.types.TypeParser;
 import org.osgi.service.component.annotations.Activate;
 import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
 import org.osgi.service.component.annotations.Reference;
 import org.osgi.service.jaxrs.whiteboard.JaxrsWhiteboardConstants;
 import org.osgi.service.jaxrs.whiteboard.propertytypes.JSONRequired;
@@ -174,6 +182,10 @@ private static void respectForwarded(final UriBuilder uriBuilder, final @Context
     private final ManagedItemProvider managedItemProvider;
     private final MetadataRegistry metadataRegistry;
     private final MetadataSelectorMatcher metadataSelectorMatcher;
+    private final ItemRegistryChangeListener resetLastModifiedItemChangeListener = new ResetLastModifiedItemChangeListener();
+    private final RegistryChangeListener<Metadata> resetLastModifiedMetadataChangeListener = new ResetLastModifiedMetadataChangeListener();
+
+    private Map<@Nullable String, Date> cacheableListsLastModified = new HashMap<>();
 
     @Activate
     public ItemResource(//
@@ -193,6 +205,15 @@ public ItemResource(//
         this.managedItemProvider = managedItemProvider;
         this.metadataRegistry = metadataRegistry;
         this.metadataSelectorMatcher = metadataSelectorMatcher;
+
+        this.itemRegistry.addRegistryChangeListener(resetLastModifiedItemChangeListener);
+        this.metadataRegistry.addRegistryChangeListener(resetLastModifiedMetadataChangeListener);
+    }
+
+    @Deactivate
+    void deactivate() {
+        this.itemRegistry.removeRegistryChangeListener(resetLastModifiedItemChangeListener);
+        this.metadataRegistry.removeRegistryChangeListener(resetLastModifiedMetadataChangeListener);
     }
 
     private UriBuilder uriBuilder(final UriInfo uriInfo, final HttpHeaders httpHeaders) {
@@ -207,17 +228,47 @@ private UriBuilder uriBuilder(final UriInfo uriInfo, final HttpHeaders httpHeade
     @Operation(operationId = "getItems", summary = "Get all available items.", responses = {
             @ApiResponse(responseCode = "200", description = "OK", content = @Content(array = @ArraySchema(schema = @Schema(implementation = EnrichedItemDTO.class)))) })
     public Response getItems(final @Context UriInfo uriInfo, final @Context HttpHeaders httpHeaders,
+            @Context Request request,
             @HeaderParam(HttpHeaders.ACCEPT_LANGUAGE) @Parameter(description = "language") @Nullable String language,
             @QueryParam("type") @Parameter(description = "item type filter") @Nullable String type,
             @QueryParam("tags") @Parameter(description = "item tag filter") @Nullable String tags,
             @DefaultValue(".*") @QueryParam("metadata") @Parameter(description = "metadata selector - a comma separated list or a regular expression (returns all if no value given)") @Nullable String namespaceSelector,
             @DefaultValue("false") @QueryParam("recursive") @Parameter(description = "get member items recursively") boolean recursive,
-            @QueryParam("fields") @Parameter(description = "limit output to the given fields (comma separated)") @Nullable String fields) {
+            @QueryParam("fields") @Parameter(description = "limit output to the given fields (comma separated)") @Nullable String fields,
+            @DefaultValue("false") @QueryParam("staticDataOnly") @Parameter(description = "provides a cacheable list of values not expected to change regularly and checks the If-Modified-Since header, all other parameters are ignored except \"metadata\"") boolean staticDataOnly) {
         final Locale locale = localeService.getLocale(language);
         final Set<String> namespaces = splitAndFilterNamespaces(namespaceSelector, locale);
 
         final UriBuilder uriBuilder = uriBuilder(uriInfo, httpHeaders);
 
+        if (staticDataOnly) {
+            Date lastModifiedDate = Date.from(Instant.now());
+            if (cacheableListsLastModified.containsKey(namespaceSelector)) {
+                lastModifiedDate = cacheableListsLastModified.get(namespaceSelector);
+                Response.ResponseBuilder responseBuilder = request.evaluatePreconditions(lastModifiedDate);
+                if (responseBuilder != null) {
+                    // send 304 Not Modified
+                    return responseBuilder.build();
+                }
+            } else {
+                lastModifiedDate = Date.from(Instant.now().truncatedTo(ChronoUnit.SECONDS));
+                cacheableListsLastModified.put(namespaceSelector, lastModifiedDate);
+            }
+
+            Stream<EnrichedItemDTO> itemStream = getItems(null, null).stream() //
+                    .map(item -> EnrichedItemDTOMapper.map(item, false, null, uriBuilder, locale)) //
+                    .peek(dto -> addMetadata(dto, namespaces, null)) //
+                    .peek(dto -> dto.editable = isEditable(dto.name));
+            itemStream = dtoMapper.limitToFields(itemStream,
+                    "name,label,type,groupType,function,category,editable,groupNames,link,tags,metadata");
+
+            CacheControl cc = new CacheControl();
+            cc.setMustRevalidate(true);
+            cc.setPrivate(true);
+            return Response.ok(new Stream2JSONInputStream(itemStream)).lastModified(lastModifiedDate).cacheControl(cc)
+                    .build();
+        }
+
         Stream<EnrichedItemDTO> itemStream = getItems(type, tags).stream() //
                 .map(item -> EnrichedItemDTOMapper.map(item, recursive, null, uriBuilder, locale)) //
                 .peek(dto -> addMetadata(dto, namespaces, null)) //
@@ -935,4 +986,48 @@ private void addMetadata(EnrichedItemDTO dto, Set<String> namespaces, @Nullable
     private boolean isEditable(String itemName) {
         return managedItemProvider.get(itemName) != null;
     }
+
+    private void resetCacheableListsLastModified() {
+        this.cacheableListsLastModified.clear();
+    }
+
+    private class ResetLastModifiedItemChangeListener implements ItemRegistryChangeListener {
+        @Override
+        public void added(Item element) {
+            resetCacheableListsLastModified();
+        }
+
+        @Override
+        public void allItemsChanged(Collection<String> oldItemNames) {
+            resetCacheableListsLastModified();
+        }
+
+        @Override
+        public void removed(Item element) {
+            resetCacheableListsLastModified();
+        }
+
+        @Override
+        public void updated(Item oldElement, Item element) {
+            resetCacheableListsLastModified();
+        }
+    }
+
+    private class ResetLastModifiedMetadataChangeListener implements RegistryChangeListener<Metadata> {
+
+        @Override
+        public void added(Metadata element) {
+            resetCacheableListsLastModified();
+        }
+
+        @Override
+        public void removed(Metadata element) {
+            resetCacheableListsLastModified();
+        }
+
+        @Override
+        public void updated(Metadata oldElement, Metadata element) {
+            resetCacheableListsLastModified();
+        }
+    }
 }