Allow direct communication with Openshift Quota API

The Openshift allocator will now only make the minimal
`resourcequota` object for each namespace, with no
support for scopes.

Most of the integration code and test cases have been
adapted from `openshift-acct-mgt`. Notable exclusions
were any code related to the `quota.json`[1],
`limits.json`[2], and quota scopes[3].

[1] https://github.com/CCI-MOC/openshift-acct-mgt/blob/master/k8s/base/quotas.json
[2] https://github.com/CCI-MOC/openshift-acct-mgt/blob/master/k8s/base/limits.json
[3] https://github.com/CCI-MOC/openshift-acct-mgt/blob/42db8f80962fd355eac1bc80a1894dc6bb824f12/acct_mgt/moc_openshift.py#L418-L431

Author: QuanMPhm

src/coldfront_plugin_cloud/management/commands/validate_allocations.py

@@ -183,7 +183,7 @@ def handle(self, *args, **options):
                 )
                 continue
 
-            quota = allocator.get_quota(project_id)["Quota"]
+            quota = allocator.get_quota(project_id)
 
             failed_validation = Command.sync_users(project_id, allocation, allocator, options["apply"])
 

src/coldfront_plugin_cloud/openshift.py

@@ -36,12 +36,12 @@ def clean_openshift_metadata(obj):
     return obj
 
 QUOTA_KEY_MAPPING = {
-    attributes.QUOTA_LIMITS_CPU: lambda x: {":limits.cpu": f"{x * 1000}m"},
-    attributes.QUOTA_LIMITS_MEMORY: lambda x: {":limits.memory": f"{x}Mi"},
-    attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: lambda x: {":limits.ephemeral-storage": f"{x}Gi"},
-    attributes.QUOTA_REQUESTS_STORAGE: lambda x: {":requests.storage": f"{x}Gi"},
-    attributes.QUOTA_REQUESTS_GPU: lambda x: {":requests.nvidia.com/gpu": f"{x}"},
-    attributes.QUOTA_PVC: lambda x: {":persistentvolumeclaims": f"{x}"},
+    attributes.QUOTA_LIMITS_CPU: lambda x: {"limits.cpu": f"{x * 1000}m"},
+    attributes.QUOTA_LIMITS_MEMORY: lambda x: {"limits.memory": f"{x}Mi"},
+    attributes.QUOTA_LIMITS_EPHEMERAL_STORAGE_GB: lambda x: {"limits.ephemeral-storage": f"{x}Gi"},
+    attributes.QUOTA_REQUESTS_STORAGE: lambda x: {"requests.storage": f"{x}Gi"},
+    attributes.QUOTA_REQUESTS_GPU: lambda x: {"requests.nvidia.com/gpu": f"{x}"},
+    attributes.QUOTA_PVC: lambda x: {"persistentvolumeclaims": f"{x}"},
 }
 
 
@@ -146,20 +146,43 @@ def create_project(self, suggested_project_name):
         project_name = project_id
         self._create_project(project_name, project_id)
         return self.Project(project_name, project_id)
+    
+    def delete_moc_quotas(self, project_id):
+        """deletes all resourcequotas from an openshift project"""
+        resourcequotas = self._openshift_get_resourcequotas(project_id)
+        for resourcequota in resourcequotas:
+            self._openshift_delete_resourcequota(project_id, resourcequota["metadata"]["name"])
+
+        logger.info(f"All quotas for {project_id} successfully deleted")
 
     def set_quota(self, project_id):
-        url = f"{self.auth_url}/projects/{project_id}/quota"
-        payload = dict()
+        """Sets the quota for a project, creating a minimal resourcequota
+        object in the project namespace with no extra scopes"""
+
+        quota_spec = {}
         for key, func in QUOTA_KEY_MAPPING.items():
             if (x := self.allocation.get_attribute(key)) is not None:
-                payload.update(func(x))
-        r = self.session.put(url, data=json.dumps({'Quota': payload}))
-        self.check_response(r)
+                quota_spec.update(func(x))
+
+        quota_def = {
+            "metadata": {"name": f"{project_id}-project"},
+            "spec": {"hard": quota_spec},
+        }
+
+        self.delete_moc_quotas(project_id)
+        self._openshift_create_resourcequota(project_id, quota_def)
+
+        logger.info(f"Quota for {project_id} successfully created")
 
     def get_quota(self, project_id):
-        url = f"{self.auth_url}/projects/{project_id}/quota"
-        r = self.session.get(url)
-        return self.check_response(r)
+        cloud_quotas = self._openshift_get_resourcequotas(project_id)
+        combined_quota = {}
+        # TODO: {Quan} Do our project namespace have more than one quota object?
+        # What should I do??
+        for cloud_quota in cloud_quotas:
+            combined_quota.update(cloud_quota["spec"]["hard"])
+
+        return combined_quota
 
     def create_project_defaults(self, project_id):
         pass
@@ -300,3 +323,48 @@ def _openshift_useridentitymapping_exists(self, user_name, id_user):
             for identity in user.get("identities", [])
         )
 
+    def _openshift_get_project(self, project_name):
+        api = self.get_resource_api(API_PROJECT, "Project")
+        return clean_openshift_metadata(api.get(name=project_name).to_dict())
+
+    def _openshift_get_resourcequotas(self, project_id):
+        """Returns a list of resourcequota objects in namespace with name `project_id`"""
+        # Raise a NotFound error if the project doesn't exist
+        self._openshift_get_project(project_id)
+        api = self.get_resource_api(API_CORE, "ResourceQuota")
+        res = clean_openshift_metadata(api.get(namespace=project_id).to_dict())
+
+        return res["items"]
+    
+    def _wait_for_quota_to_settle(self, project_id, resource_quota):
+        """Wait for quota on resourcequotas to settle.
+
+        When creating a new resourcequota that sets a quota on resourcequota objects, we need to
+        wait for OpenShift to calculate the quota usage before we attempt to create any new
+        resourcequota objects.
+        """
+
+        if "resourcequotas" in resource_quota["spec"]["hard"]:
+            logger.info("waiting for resourcequota quota")
+
+            api = self.get_resource_api(API_CORE, "ResourceQuota")
+            while True:
+                resp = clean_openshift_metadata(
+                    api.get(
+                        namespace=project_id, name=resource_quota["metadata"]["name"]
+                    ).to_dict()
+                )
+                if "resourcequotas" in resp["status"].get("used", {}):
+                    break
+                time.sleep(0.1)
+    
+    def _openshift_create_resourcequota(self, project_id, quota_def):
+        api = self.get_resource_api(API_CORE, "ResourceQuota")
+        res = api.create(namespace=project_id, body=quota_def).to_dict()
+        self._wait_for_quota_to_settle(project_id, res)
+    
+    def _openshift_delete_resourcequota(self, project_id, resourcequota_name):
+        """In an openshift namespace {project_id) delete a specified resourcequota"""
+        api = self.get_resource_api(API_CORE, "ResourceQuota")
+        return api.delete(namespace=project_id, name=resourcequota_name).to_dict()
+    

src/coldfront_plugin_cloud/tests/functional/openshift/test_allocation.py

@@ -126,17 +126,16 @@ def test_new_allocation_quota(self):
         self.assertEqual(allocation.get_attribute(attributes.QUOTA_REQUESTS_GPU), 2 * 0)
         self.assertEqual(allocation.get_attribute(attributes.QUOTA_PVC), 2 * 2)
 
-        quota = allocator.get_quota(project_id)['Quota']
-        quota = {k: v for k, v in quota.items() if v is not None}
+        quota = allocator.get_quota(project_id)
         # The return value will update to the most relevant unit, so
         # 2000m cores becomes 2 and 8192Mi becomes 8Gi
         self.assertEqual(quota, {
-            ":limits.cpu": "2",
-            ":limits.memory": "8Gi",
-            ":limits.ephemeral-storage": "10Gi",
-            ":requests.storage": "40Gi",
-            ":requests.nvidia.com/gpu": "0",
-            ":persistentvolumeclaims": "4",
+            "limits.cpu": "2",
+            "limits.memory": "8Gi",
+            "limits.ephemeral-storage": "10Gi",
+            "requests.storage": "40Gi",
+            "requests.nvidia.com/gpu": "0",
+            "persistentvolumeclaims": "4",
         })
 
         # change a bunch of attributes
@@ -157,16 +156,16 @@ def test_new_allocation_quota(self):
         # This call should update the openshift quota to match the current attributes
         call_command('validate_allocations', apply=True)
 
-        quota = allocator.get_quota(project_id)['Quota']
+        quota = allocator.get_quota(project_id)
         quota = {k: v for k, v in quota.items() if v is not None}
 
         self.assertEqual(quota, {
-            ":limits.cpu": "6",
-            ":limits.memory": "8Gi",
-            ":limits.ephemeral-storage": "50Gi",
-            ":requests.storage": "100Gi",
-            ":requests.nvidia.com/gpu": "1",
-            ":persistentvolumeclaims": "10",
+            "limits.cpu": "6",
+            "limits.memory": "8Gi",
+            "limits.ephemeral-storage": "50Gi",
+            "requests.storage": "100Gi",
+            "requests.nvidia.com/gpu": "1",
+            "persistentvolumeclaims": "10",
         })
 
     def test_reactivate_allocation(self):
@@ -183,19 +182,17 @@ def test_reactivate_allocation(self):
 
         self.assertEqual(allocation.get_attribute(attributes.QUOTA_LIMITS_CPU), 2)
 
-        quota = allocator.get_quota(project_id)['Quota']
+        quota = allocator.get_quota(project_id)
 
-        # https://github.com/CCI-MOC/openshift-acct-mgt
-        quota = {k: v for k, v in quota.items() if v is not None}
         # The return value will update to the most relevant unit, so
         # 2000m cores becomes 2 and 8192Mi becomes 8Gi
         self.assertEqual(quota, {
-            ":limits.cpu": "2",
-            ":limits.memory": "8Gi",
-            ":limits.ephemeral-storage": "10Gi",
-            ":requests.storage": "40Gi",
-            ":requests.nvidia.com/gpu": "0",
-            ":persistentvolumeclaims": "4",
+            "limits.cpu": "2",
+            "limits.memory": "8Gi",
+            "limits.ephemeral-storage": "10Gi",
+            "requests.storage": "40Gi",
+            "requests.nvidia.com/gpu": "0",
+            "persistentvolumeclaims": "4",
         })
 
         # Simulate an attribute change request and subsequent approval which
@@ -204,17 +201,16 @@ def test_reactivate_allocation(self):
         tasks.activate_allocation(allocation.pk)
         allocation.refresh_from_db()
 
-        quota = allocator.get_quota(project_id)['Quota']
-        quota = {k: v for k, v in quota.items() if v is not None}
+        quota = allocator.get_quota(project_id)
         # The return value will update to the most relevant unit, so
         # 3000m cores becomes 3 and 8192Mi becomes 8Gi
         self.assertEqual(quota, {
-            ":limits.cpu": "3",
-            ":limits.memory": "8Gi",
-            ":limits.ephemeral-storage": "10Gi",
-            ":requests.storage": "40Gi",
-            ":requests.nvidia.com/gpu": "0",
-            ":persistentvolumeclaims": "4",
+            "limits.cpu": "3",
+            "limits.memory": "8Gi",
+            "limits.ephemeral-storage": "10Gi",
+            "requests.storage": "40Gi",
+            "requests.nvidia.com/gpu": "0",
+            "persistentvolumeclaims": "4",
         })
 
         allocator._get_role(user.username, project_id)

src/coldfront_plugin_cloud/tests/unit/openshift/test_quota.py

@@ -0,0 +1,99 @@
+from unittest import mock
+
+from coldfront_plugin_cloud.tests import base
+from coldfront_plugin_cloud.openshift import OpenShiftResourceAllocator
+
+
+class TestOpenshiftQuota(base.TestBase):
+    def setUp(self) -> None:
+        mock_resource = mock.Mock()
+        mock_allocation = mock.Mock()
+        self.allocator = OpenShiftResourceAllocator(mock_resource, mock_allocation)
+        self.allocator.id_provider = "fake_idp"
+        self.allocator.k8_client = mock.Mock()
+
+    @mock.patch("coldfront_plugin_cloud.openshift.OpenShiftResourceAllocator._openshift_get_project", mock.Mock())
+    def test_get_resourcequotas(self):
+        fake_quota = mock.Mock(spec=["to_dict"])
+        fake_quota.to_dict.return_value = {"items": []}
+        self.allocator.k8_client.resources.get.return_value.get.return_value = fake_quota
+        res = self.allocator._openshift_get_resourcequotas("fake-project")
+        self.allocator.k8_client.resources.get.return_value.get.assert_called()
+        assert res == []
+
+
+    def test_delete_quota(self):
+        fake_quota = mock.Mock(spec=["to_dict"])
+        fake_quota.to_dict.return_value = {}
+        self.allocator.k8_client.resources.get.return_value.delete.return_value = fake_quota
+        res = self.allocator._openshift_delete_resourcequota("test-project", "test-quota")
+        self.allocator.k8_client.resources.get.return_value.delete.assert_called()
+        assert res == {}
+
+
+    @mock.patch("coldfront_plugin_cloud.openshift.OpenShiftResourceAllocator._openshift_get_resourcequotas")
+    def test_delete_moc_quota(self, fake_get_resourcequotas):
+        fake_get_resourcequotas.return_value = [{"metadata": {"name": "fake-quota"}}]
+        self.allocator.delete_moc_quotas("test-project")
+        self.allocator.k8_client.resources.get.return_value.delete.assert_any_call(
+            namespace="test-project", name="fake-quota"
+        )
+
+
+    @mock.patch("coldfront_plugin_cloud.openshift.OpenShiftResourceAllocator._wait_for_quota_to_settle")
+    def test_create_shift_quotas(self, fake_wait_quota):
+        quotadefs = {
+            "metadata": {"name": "fake-project-project"},
+            "spec": {"hard": {"configmaps": "1", "cpu": "1", "resourcequotas": "1"}},
+        }
+
+        self.allocator.k8_client.resources.get.return_value.create.return_value = mock.Mock()
+
+        self.allocator._openshift_create_resourcequota("fake-project", quotadefs)
+
+        self.allocator.k8_client.resources.get.return_value.create.assert_called_with(
+            namespace="fake-project",
+            body={
+                "metadata": {"name": "fake-project-project"},
+                "spec": {"hard": {"configmaps": "1", "cpu": "1", "resourcequotas": "1"}},
+            },
+        )
+
+        fake_wait_quota.assert_called()
+
+
+    def test_wait_for_quota_to_settle(self):
+        fake_quota = mock.Mock(spec=["to_dict"])
+        fake_quota.to_dict.return_value = {
+            "metadata": {"name": "fake-quota"},
+            "spec": {"hard": {"resourcequotas": "1"}},
+            "status": {"used": {"resourcequotas": "1"}},
+        }
+        self.allocator.k8_client.resources.get.return_value.get.return_value = fake_quota
+
+        self.allocator._wait_for_quota_to_settle("fake-project", fake_quota.to_dict())
+
+        self.allocator.k8_client.resources.get.return_value.get.assert_called_with(
+            namespace="fake-project",
+            name="fake-quota",
+        )
+
+    @mock.patch("coldfront_plugin_cloud.openshift.OpenShiftResourceAllocator._openshift_get_resourcequotas")
+    def test_get_moc_quota(self, fake_get_quota):
+        expected_quota = {
+            "services": "2",
+            "configmaps": None,
+            "cpu": "1000",
+        }
+        fake_get_quota.return_value = [
+            {
+                "spec": {
+                    "hard": expected_quota
+                },
+            }
+        ]
+        res = self.allocator.get_quota("fake-project")
+        self.assertEqual(res, expected_quota)
+
+
+