Add support for setting gdrcopyEnabled

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

api/config/v1/flags.go

@@ -60,6 +60,7 @@ type CommandLineFlags struct {
 	MpsRoot                 *string                 `json:"mpsRoot,omitempty"          yaml:"mpsRoot,omitempty"`
 	NvidiaDriverRoot        *string                 `json:"nvidiaDriverRoot,omitempty" yaml:"nvidiaDriverRoot,omitempty"`
 	NvidiaDevRoot           *string                 `json:"nvidiaDevRoot,omitempty"    yaml:"nvidiaDevRoot,omitempty"`
+	GDRCopyEnabled          *bool                   `json:"gdrcopyEnabled"             yaml:"gdrcopyEnabled"`
 	GDSEnabled              *bool                   `json:"gdsEnabled"                 yaml:"gdsEnabled"`
 	MOFEDEnabled            *bool                   `json:"mofedEnabled"               yaml:"mofedEnabled"`
 	UseNodeFeatureAPI       *bool                   `json:"useNodeFeatureAPI"          yaml:"useNodeFeatureAPI"`
@@ -126,6 +127,8 @@ func (f *Flags) UpdateFromCLIFlags(c *cli.Context, flags []cli.Flag) {
 				updateFromCLIFlag(&f.NvidiaDriverRoot, c, n)
 			case "dev-root", "nvidia-dev-root":
 				updateFromCLIFlag(&f.NvidiaDevRoot, c, n)
+			case "gdrcopy-enabled":
+				updateFromCLIFlag(&f.GDRCopyEnabled, c, n)
 			case "gds-enabled":
 				updateFromCLIFlag(&f.GDSEnabled, c, n)
 			case "mofed-enabled":

api/config/v1/flags_test.go

@@ -160,6 +160,7 @@ func TestMarshalFlags(t *testing.T) {
 			output: `{
 				"migStrategy": null,
 				"failOnInitError": null,
+				"gdrcopyEnabled": null,
 				"gdsEnabled": null,
 				"mofedEnabled": null,
 				"useNodeFeatureAPI": null,
@@ -177,6 +178,7 @@ func TestMarshalFlags(t *testing.T) {
 			output: `{
 				"migStrategy": null,
 				"failOnInitError": null,
+				"gdrcopyEnabled": null,
 				"gdsEnabled": null,
 				"mofedEnabled": null,
 				"useNodeFeatureAPI": null,
@@ -201,6 +203,7 @@ func TestMarshalFlags(t *testing.T) {
 			output: `{
 				"migStrategy": null,
 				"failOnInitError": null,
+				"gdrcopyEnabled": null,
 				"gdsEnabled": null,
 				"mofedEnabled": null,
 				"useNodeFeatureAPI": null,

cmd/nvidia-device-plugin/main.go

@@ -100,14 +100,19 @@ func main() {
 			Usage:   "the desired strategy for passing device IDs to the underlying runtime:\n\t\t[uuid | index]",
 			EnvVars: []string{"DEVICE_ID_STRATEGY"},
 		},
+		&cli.BoolFlag{
+			Name:    "gdrcopy-enabled",
+			Usage:   "ensure that containers that request NVIDIA GPU resources are started with GDRCopy support",
+			EnvVars: []string{"GDRCOPY_ENABLED"},
+		},
 		&cli.BoolFlag{
 			Name:    "gds-enabled",
-			Usage:   "ensure that containers are started with NVIDIA_GDS=enabled",
+			Usage:   "ensure that containers that request NVIDIA GPU resources are started with GPUDirect Storage support",
 			EnvVars: []string{"GDS_ENABLED"},
 		},
 		&cli.BoolFlag{
 			Name:    "mofed-enabled",
-			Usage:   "ensure that containers are started with NVIDIA_MOFED=enabled",
+			Usage:   "ensure that containers that request NVIDIA GPU resources are started with MOFED support",
 			EnvVars: []string{"MOFED_ENABLED"},
 		},
 		&cli.StringFlag{

cmd/nvidia-device-plugin/plugin-manager.go

@@ -54,6 +54,7 @@ func GetPlugins(ctx context.Context, infolib info.Interface, nvmllib nvml.Interf
 		cdi.WithNvidiaCTKPath(*config.Flags.Plugin.NvidiaCTKPath),
 		cdi.WithDeviceIDStrategy(*config.Flags.Plugin.DeviceIDStrategy),
 		cdi.WithVendor("k8s.device-plugin.nvidia.com"),
+		cdi.WithGdrcopyEnabled(*config.Flags.GDRCopyEnabled),
 		cdi.WithGdsEnabled(*config.Flags.GDSEnabled),
 		cdi.WithMofedEnabled(*config.Flags.MOFEDEnabled),
 		cdi.WithImexChannels(imexChannels),

deployments/helm/nvidia-device-plugin/templates/daemonset-device-plugin.yml

@@ -169,6 +169,10 @@ spec:
           - name: NVIDIA_CDI_HOOK_PATH
             value: {{ .Values.cdi.nvidiaHookPath }}
         {{- end }}
+        {{- if typeIs "bool" .Values.gdrcopyEnabled }}
+          - name: GDRCOPY_ENABLED
+            value: {{ .Values.gdrcopyEnabled | quote }}
+        {{- end }}
         {{- if typeIs "bool" .Values.gdsEnabled }}
           - name: GDS_ENABLED
             value: {{ .Values.gdsEnabled | quote }}

deployments/helm/nvidia-device-plugin/values.yaml

@@ -33,6 +33,7 @@ failOnInitError: null
 deviceListStrategy: null
 deviceIDStrategy: null
 nvidiaDriverRoot: null
+gdrcopyEnabled: null
 gdsEnabled: null
 mofedEnabled: null
 deviceDiscoveryStrategy: null

internal/cdi/cdi.go

@@ -57,8 +57,9 @@ type cdiHandler struct {
 
 	deviceListStrategies spec.DeviceListStrategies
 
-	gdsEnabled   bool
-	mofedEnabled bool
+	gdsEnabled     bool
+	mofedEnabled   bool
+	gdrcopyEnabled bool
 
 	imexChannels imex.Channels
 
@@ -134,6 +135,9 @@ func New(infolib info.Interface, nvmllib nvml.Interface, devicelib device.Interf
 	}
 
 	var additionalModes []string
+	if c.gdrcopyEnabled {
+		additionalModes = append(additionalModes, "gdrcopy")
+	}
 	if c.gdsEnabled {
 		additionalModes = append(additionalModes, "gds")
 	}

internal/cdi/options.go

@@ -80,14 +80,21 @@ func WithVendor(vendor string) Option {
 	}
 }
 
-// WithGdsEnabled provides and option to set whether a GDS CDI spec should be generated
+// WithGdrcopyEnabled provides an option to set whether a GDS CDI spec should be generated
+func WithGdrcopyEnabled(enabled bool) Option {
+	return func(c *cdiHandler) {
+		c.gdrcopyEnabled = enabled
+	}
+}
+
+// WithGdsEnabled provides an option to set whether a GDS CDI spec should be generated
 func WithGdsEnabled(enabled bool) Option {
 	return func(c *cdiHandler) {
 		c.gdsEnabled = enabled
 	}
 }
 
-// WithMofedEnabled provides and option to set whether a MOFED CDI spec should be generated
+// WithMofedEnabled provides an option to set whether a MOFED CDI spec should be generated
 func WithMofedEnabled(enabled bool) Option {
 	return func(c *cdiHandler) {
 		c.mofedEnabled = enabled

internal/plugin/server.go

@@ -351,6 +351,9 @@ func (plugin *nvidiaDevicePlugin) getAllocateResponse(requestIds []string) (*plu
 	if *plugin.config.Flags.Plugin.PassDeviceSpecs {
 		response.Devices = append(response.Devices, plugin.apiDeviceSpecs(*plugin.config.Flags.NvidiaDevRoot, requestIds)...)
 	}
+	if *plugin.config.Flags.GDRCopyEnabled {
+		response.Envs["NVIDIA_GDRCOPY"] = "enabled"
+	}
 	if *plugin.config.Flags.GDSEnabled {
 		response.Envs["NVIDIA_GDS"] = "enabled"
 	}
@@ -383,6 +386,9 @@ func (plugin *nvidiaDevicePlugin) updateResponseForCDI(response *pluginapi.Conta
 	if *plugin.config.Flags.MOFEDEnabled {
 		devices = append(devices, plugin.cdiHandler.QualifiedName("mofed", "all"))
 	}
+	if *plugin.config.Flags.GDRCopyEnabled {
+		devices = append(devices, plugin.cdiHandler.QualifiedName("gdrcopy", "all"))
+	}
 
 	if len(devices) == 0 {
 		return nil