diff --git a/pkg/driver/node_linux.go b/pkg/driver/node_linux.go index a61486b297..4f00006459 100644 --- a/pkg/driver/node_linux.go +++ b/pkg/driver/node_linux.go @@ -22,7 +22,9 @@ package driver import ( "fmt" "os" + "os/exec" "path/filepath" + "regexp" "strconv" "strings" @@ -87,7 +89,8 @@ func (d *nodeService) findDevicePath(devicePath, volumeID, partition string) (st // which AWS presents NVME devices under /dev/disk/by-id/. For example, // vol-0fab1d5e3f72a5e23 creates a symlink at // /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23 - nvmeName := "nvme-Amazon_Elastic_Block_Store_" + strings.Replace(volumeID, "-", "", -1) + strippedVolumeName := strings.Replace(volumeID, "-", "", -1) + nvmeName := "nvme-Amazon_Elastic_Block_Store_" + strippedVolumeName nvmeDevicePath, err := findNvmeVolume(d.deviceIdentifier, nvmeName) @@ -112,6 +115,26 @@ func (d *nodeService) findDevicePath(devicePath, volumeID, partition string) (st return "", errNoDevicePathFound(devicePath, volumeID) } + // In some rare cases, a race condition can lead to the /dev/disk/by-id/ symlink becoming out of date + // See https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1224 for more info + // Attempt to use lsblk to double check that the nvme device selected was the correct volume + output, err := exec.Command("lsblk", "--noheadings", "--ascii", "--nodeps", "--output", "SERIAL", canonicalDevicePath).CombinedOutput() + + if err != nil { + // Look for an EBS volume ID in the output, compare all matches against what we expect + // (in some rare cases there may be multiple matches due to lsblk printing partitions) + // If no volume ID is in the output (non-Nitro instances, SBE devices, etc) silently proceed + volumeRegex := regexp.MustCompile(`vol[a-z0-9]+`) + for _, volume := range volumeRegex.FindAllString(string(output), -1) { + if volume != strippedVolumeName { + return "", fmt.Errorf("Refusing to mount %s because it claims to be %s but should be %s", canonicalDevicePath, volume, strippedVolumeName) + } + } + } else { + // If the command fails (for example, because lsblk is not available), silently ignore the error and proceed + klog.V(5).ErrorS(err, "Ignoring lsblk failure", "devicePath", devicePath, "volumeID", volumeID) + } + canonicalDevicePath = d.appendPartition(canonicalDevicePath, partition) return canonicalDevicePath, nil }