device-injector: add support for CDI injection.

Add support for injecting annotated CDI devices using the new native NRI CDI injection API. Signed-off-by: Krisztian Litkey <[email protected]>
containerd · Aug 12, 2024 · ec34222 · ec34222
1 parent 646502f
commit ec34222
Show file tree

Hide file tree

Showing 5 changed files with 439 additions and 77 deletions.
diff --git a/plugins/device-injector/README.md b/plugins/device-injector/README.md
@@ -1,16 +1,19 @@
 ## Device Injector Plugin
 
-This sample plugin can inject devices and mounts into containers using
-pod annotations.
+This sample plugin can inject Linux device nodes, CDI devices, and mounts into
+containers using pod annotations.
 
 ### Device Annotations
 
 Devices are annotated using the `devices.nri.io` annotation key prefix.
 The key `devices.nri.io/container.$CONTAINER_NAME` annotates devices to
 be injected into `$CONTAINER_NAME`. The keys `devices.nri.io` and
-`devices.nri.io/pod` annotate devices to be injected into all containers.
+`devices.nri.io/pod` annotate devices to be injected into containers
+without any other, container-specific device annotations. Only one of
+these latter two annotations will be ever taken into account. If both are
+present, it is unspecified which one is used.
 
-The annotation syntax for device injection is
+The annotation value syntax for device injection is
 
 ```
 - path: /dev/dev0
@@ -26,10 +29,40 @@ The annotation syntax for device injection is
 
 `file_mode`, `uid` and `gid` can be omitted, the rest are mandatory.
 
+### CDI Device Annotations
+
+CDI devices are annotated in a similar manner to devices, but using the
+`cdi-devices.nri.io` annotation key prefix. The annotation value for CDI
+devices is the list of CDI device names to inject.
+
+For instance, the following annotation
+
+```
+metadata:
+  name: bash
+  annotations:
+    cdi-devices.nri.io/container.c0: |
+      - vendor0.com/device=null
+    cdi-devices.nri.io/container.c1: |
+      - vendor0.com/device=zero
+    cdi-devices.nri.io/container.c2: |
+      - vendor0.com/device=dev0
+      - vendor1.com/device=dev0
+      - vendor1.com/device=dev1
+    cdi-devices.nri.io/container.mgmt: |
+      - vendor0.com/device=all
+```
+
+requests the injection of the CDI device vendor0.com/device=null to container
+c0, the injection of the CDI device vendor0.com/device=zero to container c1,
+the injection of the CDI devices vendor0.com/device=dev0, vendor1.com/device=dev0
+and vendor1.com/device=dev1 to container c2, and the injection of the CDI device
+vendor0.com/device=all to container mgmt.
+
 ### Mount Annotations
 
 Mounts are annotated in a similar manner to devices, but using the
-`mounts.nri.io` annotation key prefix. The annotation syntax for mount
+`mounts.nri.io` annotation key prefix. The annotation value syntax for mount
 injection is
 
 ```

diff --git a/plugins/device-injector/device-injector.go b/plugins/device-injector/device-injector.go
@@ -35,6 +35,8 @@ const (
 	deviceKey = "devices.nri.io"
 	// Prefix of the key used for mount annotations.
 	mountKey = "mounts.nri.io"
+	// Prefix of the key used for CDI device annotations.
+	cdiDeviceKey = "cdi-devices.nri.io"
 )
 
 var (
@@ -67,131 +69,183 @@ type plugin struct {
 }
 
 // CreateContainer handles container creation requests.
-func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, container *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
-	var (
-		ctrName string
-		devices []device
-		mounts  []mount
-		err     error
-	)
-
-	ctrName = containerName(pod, container)
-
+func (p *plugin) CreateContainer(_ context.Context, pod *api.PodSandbox, ctr *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) {
 	if verbose {
-		dump("CreateContainer", "pod", pod, "container", container)
+		dump("CreateContainer", "pod", pod, "container", ctr)
 	}
 
 	adjust := &api.ContainerAdjustment{}
 
-	// inject devices to container
-	devices, err = parseDevices(container.Name, pod.Annotations)
-	if err != nil {
+	if err := injectDevices(pod, ctr, adjust); err != nil {
 		return nil, nil, err
 	}
 
-	if len(devices) == 0 {
-		log.Infof("%s: no devices annotated...", ctrName)
-	} else {
-		if verbose {
-			dump(ctrName, "annotated devices", devices)
-		}
-
-		for _, d := range devices {
-			adjust.AddDevice(d.toNRI())
-			if !verbose {
-				log.Infof("%s: injected device %q...", ctrName, d.Path)
-			}
-		}
+	if err := injectCDIDevices(pod, ctr, adjust); err != nil {
+		return nil, nil, err
 	}
 
-	// inject mounts to container
-	mounts, err = parseMounts(container.Name, pod.Annotations)
-	if err != nil {
+	if err := injectMounts(pod, ctr, adjust); err != nil {
 		return nil, nil, err
 	}
 
-	if len(mounts) == 0 {
-		log.Infof("%s: no mounts annotated...", ctrName)
-	} else {
-		if verbose {
-			dump(ctrName, "annotated mounts", mounts)
-		}
+	return adjust, nil, nil
 
-		for _, m := range mounts {
-			adjust.AddMount(m.toNRI())
-			if !verbose {
-				log.Infof("%s: injected mount %q -> %q...", ctrName, m.Source, m.Destination)
-			}
-		}
+	if verbose {
+		dump(containerName(pod, ctr), "ContainerAdjustment", adjust)
+	}
+
+	return adjust, nil, nil
+}
+
+func injectDevices(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error {
+	devices, err := parseDevices(ctr.Name, pod.Annotations)
+	if err != nil {
+		return err
+	}
+
+	if len(devices) == 0 {
+		log.Debugf("%s: no devices annotated...", containerName(pod, ctr))
+		return nil
 	}
 
 	if verbose {
-		dump(ctrName, "ContainerAdjustment", adjust)
+		dump(containerName(pod, ctr), "annotated devices", devices)
 	}
 
-	return adjust, nil, nil
+	for _, d := range devices {
+		a.AddDevice(d.toNRI())
+		if !verbose {
+			log.Infof("%s: injected device %q...", containerName(pod, ctr), d.Path)
+		}
+	}
+
+	return nil
 }
 
 func parseDevices(ctr string, annotations map[string]string) ([]device, error) {
 	var (
-		key        string
-		annotation []byte
-		devices    []device
+		devices []device
 	)
 
-	// look up effective device annotation and unmarshal devices
-	for _, key = range []string{
-		deviceKey + "/container." + ctr,
-		deviceKey + "/pod",
-		deviceKey,
-	} {
-		if value, ok := annotations[key]; ok {
-			annotation = []byte(value)
-			break
-		}
+	annotation := getAnnotation(annotations, deviceKey, ctr)
+	if annotation == nil {
+		return nil, nil
 	}
 
 	if annotation == nil {
 		return nil, nil
 	}
 
 	if err := yaml.Unmarshal(annotation, &devices); err != nil {
-		return nil, fmt.Errorf("invalid device annotation %q: %w", key, err)
+		return nil, fmt.Errorf("invalid device annotation %q: %w", string(annotation), err)
 	}
 
 	return devices, nil
 }
 
-func parseMounts(ctr string, annotations map[string]string) ([]mount, error) {
+func injectCDIDevices(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error {
+	devices, err := parseCDIDevices(ctr.Name, pod.Annotations)
+	if err != nil {
+		return err
+	}
+
+	if len(devices) == 0 {
+		log.Debugf("%s: no CDI devices annotated...", containerName(pod, ctr))
+		return nil
+	}
+
+	if verbose {
+		dump(containerName(pod, ctr), "annotated CDI devices", devices)
+	}
+
+	for _, name := range devices {
+		a.AddCDIDevice(
+			&api.CDIDevice{
+				Name: name,
+			},
+		)
+		if !verbose {
+			log.Infof("%s: injected CDI device %q...", containerName(pod, ctr), name)
+		}
+	}
+
+	return nil
+}
+
+func parseCDIDevices(ctr string, annotations map[string]string) ([]string, error) {
 	var (
-		key        string
-		annotation []byte
-		mounts     []mount
+		cdiDevices []string
 	)
 
-	// look up effective device annotation and unmarshal devices
-	for _, key = range []string{
-		mountKey + "/container." + ctr,
-		mountKey + "/pod",
-		mountKey,
-	} {
-		if value, ok := annotations[key]; ok {
-			annotation = []byte(value)
-			break
+	annotation := getAnnotation(annotations, cdiDeviceKey, ctr)
+	if annotation == nil {
+		return nil, nil
+	}
+
+	if err := yaml.Unmarshal(annotation, &cdiDevices); err != nil {
+		return nil, fmt.Errorf("invalid CDI device annotation %q: %w", string(annotation), err)
+	}
+
+	return cdiDevices, nil
+}
+
+func injectMounts(pod *api.PodSandbox, ctr *api.Container, a *api.ContainerAdjustment) error {
+	mounts, err := parseMounts(ctr.Name, pod.Annotations)
+	if err != nil {
+		return err
+	}
+
+	if len(mounts) == 0 {
+		log.Debugf("%s: no mounts annotated...", containerName(pod, ctr))
+		return nil
+	}
+
+	if verbose {
+		dump(containerName(pod, ctr), "annotated mounts", mounts)
+	}
+
+	for _, m := range mounts {
+		a.AddMount(m.toNRI())
+		if !verbose {
+			log.Infof("%s: injected mount %q -> %q...", containerName(pod, ctr),
+				m.Source, m.Destination)
 		}
 	}
 
+	return nil
+}
+
+func parseMounts(ctr string, annotations map[string]string) ([]mount, error) {
+	var (
+		mounts []mount
+	)
+
+	annotation := getAnnotation(annotations, mountKey, ctr)
 	if annotation == nil {
 		return nil, nil
 	}
 
 	if err := yaml.Unmarshal(annotation, &mounts); err != nil {
-		return nil, fmt.Errorf("invalid mount annotation %q: %w", key, err)
+		return nil, fmt.Errorf("invalid mount annotation %q: %w", string(annotation), err)
 	}
 
 	return mounts, nil
 }
 
+func getAnnotation(annotations map[string]string, mainKey, ctr string) []byte {
+	for _, key := range []string{
+		mainKey + "/container." + ctr,
+		mainKey + "/pod",
+		mainKey,
+	} {
+		if value, ok := annotations[key]; ok {
+			return []byte(value)
+		}
+	}
+
+	return nil
+}
+
 // Convert a device to the NRI API representation.
 func (d *device) toNRI() *api.LinuxDevice {
 	apiDev := &api.LinuxDevice{