WIP: adapt implementation to new API

The structured parameter allocation logic was written from scratch in
staging/src/k8s.io/dynamic-resource-allocation/structured. Besides the new
features (amount, admin access) and API it now supports backtracking when the
initial device selection doesn't lead to a complete allocation of all claims.

"make" works. A local cluster comes up and can run the DRA E2E tests.

pkg/scheduler/framework/plugins/dynamicresources unit tests pass.

test/integration/scheduler_perf unit tests pass for DRA.

Many other unit tests haven't been adapted yet and coverage of new code is
poor.

DRA: use VAP to control "admin access" permissions

The advantages of using a validation admission policy (VAP) are that no changes
are needed in Kubernetes and that admins have full flexibility if and how they
want to control which users are allowed to use "admin access" in their
requests.

The downside is that without admins taking actions, the feature is enabled
out-of-the-box in a cluster. Documentation for DRA will have to make it very
clear that something needs to be done in multi-tenant clusters.

The test/e2e/testing-manifests/dra/admin-access-policy.yaml shows how to do
this. The corresponding E2E tests ensures that it actually works as intended.

For some reason, adding the namespace to the message expression leads to a
type check errors, so it's currently commented out.

DRA kubelet: always call NodePrepareResources, even if not used

This could be useful for drivers where that call has some effect other than
injecting CDI device IDs into containers.

DRA: remove obsolete support for deterministic claim name

Now only the claim name as recorded in the pod status matters, which can be
looked up without listing claims.

DRA kubelet: extend and clean up logging

Because of a faulty E2E test, kubelet was told to contact the wrong driver for
a claim. This was not visible in the kubelet log output. Now changes to the
claim info cache are getting logged. While at it, naming of variables and some
existing log output gets harmonized.

Co-authored-by: Ed Bartosh <[email protected]>
Co-authored-by: Oksana Baranova <[email protected]>

Author: 3 people

pkg/controller/resourceclaim/controller_test.go

@@ -528,7 +528,12 @@ func makeClaim(name, namespace, classname string, owner *metav1.OwnerReference)
 	claim := &resourceapi.ResourceClaim{
 		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
 		Spec: resourceapi.ResourceClaimSpec{
-			ResourceClassName: classname,
+			Devices: resourceapi.DeviceClaim{
+				Requests: []resourceapi.DeviceRequest{{
+					Name:            "req1",
+					DeviceClassName: classname,
+				}},
+			},
 		},
 	}
 	if owner != nil {
@@ -547,7 +552,12 @@ func makeGeneratedClaim(podClaimName, generateName, namespace, classname string,
 			Annotations:  map[string]string{"resource.kubernetes.io/pod-claim-name": podClaimName},
 		},
 		Spec: resourceapi.ResourceClaimSpec{
-			ResourceClassName: classname,
+			Devices: resourceapi.DeviceClaim{
+				Requests: []resourceapi.DeviceRequest{{
+					Name:            "req1",
+					DeviceClassName: classname,
+				}},
+			},
 		},
 	}
 	if owner != nil {
@@ -606,7 +616,12 @@ func makeTemplate(name, namespace, classname string) *resourceapi.ResourceClaimT
 		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
 		Spec: resourceapi.ResourceClaimTemplateSpec{
 			Spec: resourceapi.ResourceClaimSpec{
-				ResourceClassName: classname,
+				Devices: resourceapi.DeviceClaim{
+					Requests: []resourceapi.DeviceRequest{{
+						Name:            "req1",
+						DeviceClassName: classname,
+					}},
+				},
 			},
 		},
 	}

pkg/kubelet/cm/container_manager_linux.go

@@ -660,10 +660,6 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe
 		if err != nil {
 			return nil, err
 		}
-		// NOTE: Passing CDI device names as annotations is a temporary solution
-		// It will be removed after all runtimes are updated
-		// to get CDI device names from the ContainerConfig.CDIDevices field
-		opts.Annotations = append(opts.Annotations, resOpts.Annotations...)
 		opts.CDIDevices = append(opts.CDIDevices, resOpts.CDIDevices...)
 	}
 	// Allocate should already be called during predicateAdmitHandler.Admit(),
@@ -968,15 +964,18 @@ func (cm *containerManagerImpl) GetDynamicResources(pod *v1.Pod, container *v1.C
 		// a set of CDIDevices from a different kubelet plugin. In the future we may want to
 		// include the name of the kubelet plugin and/or other types of resources that are
 		// not CDIDevices (assuming the DRAmanager supports this).
-		for _, klPluginCdiDevices := range containerClaimInfo.CDIDevices {
+		for /* driverName */ _, driverState := range containerClaimInfo.Drivers {
 			var cdiDevices []*podresourcesapi.CDIDevice
-			for _, cdiDevice := range klPluginCdiDevices {
-				cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice})
+			for /*requestName */ _, requestDevices := range driverState.CDIDevices {
+				for _, cdiDevice := range requestDevices {
+					cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice})
+				}
 			}
 			claimResources = append(claimResources, &podresourcesapi.ClaimResource{CDIDevices: cdiDevices})
 		}
 		containerDynamicResource := podresourcesapi.DynamicResource{
-			ClassName:      containerClaimInfo.ClassName,
+			// As of 1.31, ResourceClaims are not associated with a single class anymore.
+			// ClassName: ???
 			ClaimName:      containerClaimInfo.ClaimName,
 			ClaimNamespace: containerClaimInfo.Namespace,
 			ClaimResources: claimResources,

pkg/kubelet/cm/devicemanager/topology_hints_test.go

@@ -440,7 +440,6 @@ func TestTopologyAlignedAllocation(t *testing.T) {
 				opts: &pluginapi.DevicePluginOptions{GetPreferredAllocationAvailable: true},
 			}
 		}
-
 		allocated, err := m.devicesToAllocate("podUID", "containerName", tc.resource, tc.request, sets.New[string]())
 		if err != nil {
 			t.Errorf("Unexpected error: %v", err)

pkg/kubelet/cm/dra/claiminfo.go

@@ -17,26 +17,22 @@ limitations under the License.
 package dra
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 
 	resourceapi "k8s.io/api/resource/v1alpha3"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/kubelet/cm/dra/state"
-	"k8s.io/kubernetes/pkg/kubelet/cm/util/cdi"
-	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
 // ClaimInfo holds information required
 // to prepare and unprepare a resource claim.
 // +k8s:deepcopy-gen=true
 type ClaimInfo struct {
 	state.ClaimInfoState
-	// annotations is a mapping of container annotations per DRA plugin associated with
-	// a prepared resource
-	annotations map[string][]kubecontainer.Annotation
-	prepared    bool
+	prepared bool
 }
 
 // claimInfoCache is a cache of processed resource claims keyed by namespace/claimname.
@@ -47,89 +43,50 @@ type claimInfoCache struct {
 }
 
 // newClaimInfoFromClaim creates a new claim info from a resource claim.
-func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) *ClaimInfo {
-	// Grab the allocation.resourceHandles. If there are no
-	// allocation.resourceHandles, create a single resourceHandle with no
-	// content. This will trigger processing of this claim by a single
-	// kubelet plugin whose name matches resourceClaim.Status.DriverName.
-	resourceHandles := claim.Status.Allocation.ResourceHandles
-	if len(resourceHandles) == 0 {
-		resourceHandles = make([]resourceapi.ResourceHandle, 1)
-	}
+// It verifies that the kubelet can handle the claim.
+func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) (*ClaimInfo, error) {
 	claimInfoState := state.ClaimInfoState{
-		DriverName:      claim.Status.DriverName,
-		ClassName:       claim.Spec.ResourceClassName,
-		ClaimUID:        claim.UID,
-		ClaimName:       claim.Name,
-		Namespace:       claim.Namespace,
-		PodUIDs:         sets.New[string](),
-		ResourceHandles: resourceHandles,
-		CDIDevices:      make(map[string][]string),
+		ClaimUID:  claim.UID,
+		ClaimName: claim.Name,
+		Namespace: claim.Namespace,
+		PodUIDs:   sets.New[string](),
+		Drivers:   make(map[string]state.DriverState),
+	}
+	if claim.Status.Allocation == nil {
+		return nil, errors.New("not allocated")
+	}
+	for _, result := range claim.Status.Allocation.Devices.Results {
+		claimInfoState.Drivers[result.Driver] = state.DriverState{
+			CDIDevices: make(map[string][]string),
+		}
 	}
 	info := &ClaimInfo{
 		ClaimInfoState: claimInfoState,
-		annotations:    make(map[string][]kubecontainer.Annotation),
 		prepared:       false,
 	}
-	return info
+	return info, nil
 }
 
 // newClaimInfoFromClaim creates a new claim info from a checkpointed claim info state object.
 func newClaimInfoFromState(state *state.ClaimInfoState) *ClaimInfo {
 	info := &ClaimInfo{
 		ClaimInfoState: *state.DeepCopy(),
-		annotations:    make(map[string][]kubecontainer.Annotation),
 		prepared:       false,
 	}
-	for pluginName, devices := range info.CDIDevices {
-		annotations, _ := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, devices)
-		info.annotations[pluginName] = append(info.annotations[pluginName], annotations...)
-	}
 	return info
 }
 
 // setCDIDevices adds a set of CDI devices to the claim info.
-func (info *ClaimInfo) setCDIDevices(pluginName string, cdiDevices []string) error {
-	// NOTE: Passing CDI device names as annotations is a temporary solution
-	// It will be removed after all runtimes are updated
-	// to get CDI device names from the ContainerConfig.CDIDevices field
-	annotations, err := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, cdiDevices)
-	if err != nil {
-		return fmt.Errorf("failed to generate container annotations, err: %+v", err)
-	}
-
-	if info.CDIDevices == nil {
-		info.CDIDevices = make(map[string][]string)
-	}
-
-	if info.annotations == nil {
-		info.annotations = make(map[string][]kubecontainer.Annotation)
+func (info *ClaimInfo) setCDIDevices(driverName string, requestName string, cdiDevices []string) {
+	if info.Drivers == nil {
+		info.Drivers = make(map[string]state.DriverState)
 	}
-
-	info.CDIDevices[pluginName] = cdiDevices
-	info.annotations[pluginName] = annotations
-
-	return nil
-}
-
-// annotationsAsList returns container annotations as a single list.
-func (info *ClaimInfo) annotationsAsList() []kubecontainer.Annotation {
-	var lst []kubecontainer.Annotation
-	for _, v := range info.annotations {
-		lst = append(lst, v...)
-	}
-	return lst
-}
-
-// cdiDevicesAsList returns a list of CDIDevices from the provided claim info.
-func (info *ClaimInfo) cdiDevicesAsList() []kubecontainer.CDIDevice {
-	var cdiDevices []kubecontainer.CDIDevice
-	for _, devices := range info.CDIDevices {
-		for _, device := range devices {
-			cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: device})
-		}
+	driverState := info.Drivers[driverName]
+	if driverState.CDIDevices == nil {
+		driverState.CDIDevices = make(map[string][]string)
 	}
-	return cdiDevices
+	driverState.CDIDevices[requestName] = cdiDevices
+	info.Drivers[driverName] = driverState
 }
 
 // addPodReference adds a pod reference to the claim info.