Switching to include/exclude filters; refactored all APIs.

Author: chsou

buster37.dockerfile

@@ -1,6 +1,10 @@
 FROM python:3.7-slim-buster
 
-RUN apt-get update && apt-get -y install git
+RUN sed -i 's|deb.debian.org/debian|archive.debian.org/debian|g' /etc/apt/sources.list \
+ && sed -i 's|security.debian.org/debian-security|archive.debian.org/debian-security|g' /etc/apt/sources.list \
+ && apt-get update \
+ && apt-get -y install git \
+ && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt /
 RUN pip install --upgrade pip && pip install -r requirements.txt

c8y_api/model/_base.py

@@ -789,7 +789,8 @@ def _iterate(
             base_query: str,
             page_number: int | None,
             limit: int | None,
-            filter: str | JsonMatcher | None,
+            include: str | JsonMatcher | None,
+            exclude: str | JsonMatcher | None,
             parse_fun
     ):
         # if no specific page is defined we just start at 1
@@ -799,15 +800,17 @@ def _iterate(
         #  - there is no result (i.e. we were at the last page)
         num_results = 0
         # compile/prepare filter if defined
-        if isinstance(filter, str):
-            filter = self.default_matcher(filter)
+        if isinstance(include, str):
+            include = self.default_matcher(include)
+        if isinstance(exclude, str):
+            exclude = self.default_matcher(exclude)
 
         while True:
-            if filter:
-                results = [parse_fun(x) for x in self._get_page(base_query, current_page) if filter.safe_matches(x)]
-            else:
-                results = [parse_fun(x) for x in self._get_page(base_query, current_page)]
-            # no results, so we are done
+            results = [
+                parse_fun(x) for x in self._get_page(base_query, current_page)
+                if (not include or include.safe_matches(x))
+                   and (not exclude or not exclude.safe_matches(x))
+            ]
             if not results:
                 break
             for result in results:

c8y_api/model/_util.py

@@ -5,7 +5,6 @@
 from typing import Union
 
 from dateutil import parser
-from re import sub, PatternError
 
 
 class _StringUtil(object):
@@ -48,7 +47,7 @@ def matches(expression: str, string: str):
         """Check if regex expression matches a string."""
         try:
             return re.search(expression, string) is not None
-        except PatternError:
+        except re.error:
             return False
 
 class _QueryUtil(object):
@@ -59,15 +58,15 @@ def encode_odata_query_value(value):
         http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part2-url-conventions.html#sec_URLParsing
         http://docs.oasis-open.org/odata/odata/v4.01/cs01/abnf/odata-abnf-construction-rules.txt """
         # single quotes escaped through single quote
-        return sub('\'', '\'\'', value)
+        return re.sub('\'', '\'\'', value)
 
     @staticmethod
     def encode_odata_text_value(value):
         """Encode value strings according to OData query rules.
         http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part2-url-conventions.html#sec_URLParsing
         http://docs.oasis-open.org/odata/odata/v4.01/cs01/abnf/odata-abnf-construction-rules.txt """
         # single quotes escaped through single quote
-        encoded_quotes = sub('\'', '\'\'', value)
+        encoded_quotes = re.sub('\'', '\'\'', value)
         return encoded_quotes if " " not in encoded_quotes else f"'{encoded_quotes}'"
 
 

c8y_api/model/administration.py

@@ -262,9 +262,9 @@ def from_json(cls, json: dict) -> GlobalRole:
         role: GlobalRole = cls._from_json(json, GlobalRole())
         # role ID are int for some reason - convert for consistency
         role.id = str(role.id)
-        if json['roles'] and json['roles']['references']:
+        if 'roles' in json and json['roles'] and json['roles']['references']:
             role.permission_ids = {ref['role']['id'] for ref in json['roles']['references']}
-        if json['applications']:
+        if 'applications' in json and json['applications']:
             role.application_ids = {ref['id'] for ref in json['applications']}
         return role
 
@@ -943,7 +943,8 @@ def get(self, role_id: str | int) -> InventoryRole:
     def select(
             self,
             limit: int = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None,
+            exclude: str | JsonMatcher = None,
             page_size: int = 1000,
             page_number: int = None,
     ) -> Generator[InventoryRole]:
@@ -956,8 +957,12 @@ def select(
 
         Args:
             limit (int): Limit the number of results to this number.
-            filter (str | JsonMatcher): Matcher/expression to filter the query
-                results (on client side). Uses JMESPath by default.
+            include (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The inclusion is applied first.
+                Creates a JMESPath matcher by default for strings.
+            exclude (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The exclusion is applied second.
+                Creates a JMESPath matcher by default for strings.
             page_size (int): Define the number of objects read (and parsed
                 in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
@@ -967,12 +972,12 @@ def select(
             Generator for InventoryRole objects
         """
         base_query = self._prepare_query(page_size=page_size)
-        return super()._iterate(base_query, page_number, limit, filter, InventoryRole.from_json)
+        return super()._iterate(base_query, page_number, limit, include, exclude, InventoryRole.from_json)
 
     def get_all(
             self,
             limit: int = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None, exclude: str | JsonMatcher = None,
             page_size: int = 1000,
             page_number: int = None,
     ) -> List[InventoryRole]:
@@ -986,7 +991,12 @@ def get_all(
         Returns:
             List of InventoryRole objects
         """
-        return list(self.select(limit=limit, filter=filter, page_size=page_size, page_number=page_number))
+        return list(self.select(
+            limit=limit,
+            include=include,
+            exclude=exclude,
+            page_size=page_size,
+            page_number=page_number))
 
     def select_assignments(self, username: str) -> Generator[InventoryRoleAssignment]:
         """Get all inventory role assignments of a user.
@@ -1080,7 +1090,7 @@ def select(
             only_devices: bool = None,
             with_subusers_count: bool = None,
             limit: int = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None, exclude: str | JsonMatcher = None,
             page_size: int = 5,
             page_number: int = None,
             as_values: str | tuple | list[str | tuple] = None,
@@ -1104,8 +1114,12 @@ def select(
             with_subusers_count (bool):  Whether to include an additional field
              `subusersCount` which holds the number of direct sub users.
             limit (int): Limit the number of results to this number.
-            filter (str | JsonMatcher): Matcher/expression to filter the query
-                results (on client side). Uses JMESPath by default.
+            include (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The inclusion is applied first.
+                Creates a JMESPath matcher by default for strings.
+            exclude (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The exclusion is applied second.
+                Creates a JMESPath matcher by default for strings.
             page_size (int): Define the number of events which are read (and
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
@@ -1149,7 +1163,8 @@ def select(
             base_query,
             page_number,
             limit,
-            filter,
+            include,
+            exclude,
             User.from_json if not as_values else
             lambda x: parse_as_values(x, as_values))
 
@@ -1160,7 +1175,7 @@ def get_all(
             owner: str = None,
             only_devices: bool = None,
             with_subusers_count: bool = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None, exclude: str | JsonMatcher = None,
             page_size: int = 1000,
             as_values: str | tuple | list[str|tuple] = None,
             **kwargs
@@ -1180,7 +1195,8 @@ def get_all(
             owner=owner,
             only_devices=only_devices,
             with_subusers_count=with_subusers_count,
-            filter=filter,
+            include=include,
+            exclude=exclude,
             page_size=page_size,
             as_values=as_values,
             **kwargs))
@@ -1310,16 +1326,21 @@ def get(self, role_id: int | str) -> GlobalRole:
     def select(
             self,
             username: str = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None,
+            exclude: str | JsonMatcher = None,
             page_size: int = 5,
     ) -> Generator[GlobalRole]:
         """Iterate over global roles.
 
         Args:
             username (str): Retrieve global roles assigned to a specified user
                 If omitted, all available global roles are returned
-            filter (str | JsonMatcher): Matcher/expression to filter the query
-                results (on client side). Uses JMESPath by default.
+            include (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The inclusion is applied first.
+                Creates a JMESPath matcher by default for strings.
+            exclude (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The exclusion is applied second.
+                Creates a JMESPath matcher by default for strings.
             page_size (int): Maximum number of entries fetched per requests;
                 this is a performance setting
 
@@ -1330,16 +1351,22 @@ def select(
         # generic _iterate method, we have to do everything manually.
         if username:
             # compile/prepare filter if defined
-            if isinstance(filter, str):
-                filter = self.default_matcher(filter)
+            if isinstance(include, str):
+                include = self.default_matcher(include)
+            if isinstance(exclude, str):
+                exclude = self.default_matcher(exclude)
             # select by username
             query = f'/user/{self.c8y.tenant_id}/users/{username}/groups?pageSize={page_size}&currentPage='
             page_number = 1
             while True:
                 response_json = self.c8y.get(query + str(page_number))
                 references = (
-                    response_json['references'] if not filter
-                    else [x for x in response_json['references'] if filter.safe_matches(x['group'])]
+                    response_json['references'] if not include and not exclude
+                    else [
+                        x for x in response_json['references']
+                        if (not include or include.safe_matches(x['group']))
+                           and (not exclude or not exclude.safe_matches(x['group']))
+                    ]
                 )
                 if not references:
                     break
@@ -1352,28 +1379,38 @@ def select(
             # select all
             base_query = self._prepare_query(page_size=page_size)
             yield from super()._iterate(
-                base_query, page_number=None, limit=None, filter=filter, parse_fun=GlobalRole.from_json)
+                base_query,
+                page_number=None,
+                limit=None,
+                include=include,
+                exclude=exclude,
+                parse_fun=GlobalRole.from_json)
 
     def get_all(
             self,
             username: str = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None,
+            exclude: str | JsonMatcher = None,
             page_size: int = 1000
     ) -> List[GlobalRole]:
         """Retrieve global roles.
 
         Args:
             username (str): Retrieve global roles assigned to a specified user
                 If omitted, all available global roles are returned
-            filter (str | JsonMatcher): Matcher/expression to filter the query
-                results (on client side). Uses JMESPath by default.
+            include (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The inclusion is applied first.
+                Creates a JMESPath matcher by default for strings.
+            exclude (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The exclusion is applied second.
+                Creates a JMESPath matcher by default for strings.
             page_size (int): Maximum number of entries fetched per requests;
                 this is a performance setting
 
         Return:
             List of GlobalRole instances
         """
-        return list(self.select(username=username, filter=filter, page_size=page_size))
+        return list(self.select(username=username, include=include, exclude=exclude, page_size=page_size))
 
     def assign_users(self, role_id: int | str, *usernames: str):
         """Add users to a global role.

c8y_api/model/alarms.py

@@ -6,7 +6,7 @@
 from typing import List, Generator
 
 from c8y_api._base_api import CumulocityRestApi
-from c8y_api.model import JsonMatcher
+from c8y_api.model.matcher import JsonMatcher
 from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject
 from c8y_api.model._parser import as_values as parse_as_values, ComplexObjectParser
 from c8y_api.model._util import _DateUtil
@@ -255,7 +255,7 @@ def select(self,
                min_age: timedelta = None, max_age: timedelta = None,
                with_source_assets: bool = None, with_source_devices: bool = None,
                reverse: bool = False, limit: int = None,
-               filter: str | JsonMatcher = None,
+               include: str | JsonMatcher = None, exclude: str | JsonMatcher = None,
                page_size: int = 1000, page_number: int = None,
                as_values: str | tuple | list[str | tuple] = None,
                **kwargs) -> Generator[Alarm]:
@@ -305,8 +305,12 @@ def select(self,
             reverse (bool):  Invert the order of results, starting with the
                 most recent one
             limit (int): Limit the number of results to this number.
-            filter (str | JsonMatcher): Matcher/expression to filter the query
-                results (on client side). Uses JMESPath by default.
+            include (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The inclusion is applied first.
+                Creates a JMESPath matcher by default for strings.
+            exclude (str | JsonMatcher): Matcher/expression to filter the query
+                results (on client side). The exclusion is applied second.
+                Creates a JMESPath matcher by default for strings.
             page_size (int): Define the number of alarms which are read (and
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
@@ -339,7 +343,8 @@ def select(self,
             base_query,
             page_number,
             limit,
-            filter,
+            include,
+            exclude,
             Alarm.from_json if not as_values else
             lambda x: parse_as_values(x, as_values))
 
@@ -357,7 +362,7 @@ def get_all(
             min_age: timedelta = None, max_age: timedelta = None,
             with_source_assets: bool = None, with_source_devices: bool = None,
             reverse: bool = False, limit: int = None,
-            filter: str | JsonMatcher = None,
+            include: str | JsonMatcher = None, exclude: str | JsonMatcher = None,
             page_size: int = 1000, page_number: int = None,
             as_values: str | tuple | list[str | tuple] = None,
             **kwargs) -> List[Alarm]:
@@ -383,7 +388,8 @@ def get_all(
             last_updated_from=last_updated_from, last_updated_to=last_updated_to,
             min_age=min_age, max_age=max_age, reverse=reverse,
             with_source_devices=with_source_devices, with_source_assets=with_source_assets,
-            limit=limit, filter=filter, page_size=page_size, page_number=page_number,
+            limit=limit, include=include, exclude=exclude,
+            page_size=page_size, page_number=page_number,
             as_values=as_values,
             **kwargs))
 

c8y_api/model/applications.py

@@ -319,7 +319,7 @@ def select(
             page_size=page_size,
             **kwargs
         )
-        return super()._iterate(base_query, page_number, limit, None, Application.from_json)
+        return super()._iterate(base_query, page_number, limit, None, None, Application.from_json)
 
     def get_all(
             self,