Fix detection of Mellanox VFs in switchdev mode

Signed-off-by: Alexander Maslennikov <[email protected]>

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/Mellanox/rdmamap v1.0.0
+	github.com/Mellanox/sriovnet v1.0.2
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/jaypipes/ghw v0.6.0
 	github.com/jaypipes/pcidb v0.5.0

go.sum

@@ -11,6 +11,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Mellanox/rdmamap v1.0.0 h1:5OQX6n85GDKetHP9wBqxy/0Ce2+taRg+4nSTsNOCyKQ=
 github.com/Mellanox/rdmamap v1.0.0/go.mod h1:fN+/V9lf10ABnDCwTaXRjeeWijLt2iVLETnK+sx/LY8=
+github.com/Mellanox/sriovnet v1.0.2 h1:VTQHD7OHU6QejTtclt5a2obDfsW1ATRxTCgZmsiKmXI=
+github.com/Mellanox/sriovnet v1.0.2/go.mod h1:pXdSZwahlvP0Xn8nuXcVthBE38Nqf2czo449p5ALLXY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -80,6 +82,7 @@ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
@@ -107,6 +110,7 @@ github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20
 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d/go.mod h1:+1DpV8uIwteAhxNO0lgRox8gHkTG6w3OeDfAlg+qqjA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -139,12 +143,16 @@ github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
 github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
+github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -167,6 +175,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=

pkg/utils/netlink_provider.go

@@ -62,3 +62,12 @@ func UseFakeLinks() func() {
 		getLinkByName = nl.LinkByName
 	}
 }
+
+// GetDevLinkDeviceEswitchAttrs returns a devlink device's attributes
+func GetDevLinkDeviceEswitchAttrs(ifName string) (*nl.DevlinkDevEswitchAttr, error) {
+	dev, err := nl.DevLinkGetDeviceByName("pci", ifName)
+	if err != nil {
+		return nil, fmt.Errorf("error getting devlink device attributes for net device %s %v", ifName, err)
+	}
+	return &(dev.Attrs.Eswitch), nil
+}

pkg/utils/sriovnet_ops.go

@@ -0,0 +1,29 @@
+package utils
+
+import (
+	"github.com/Mellanox/sriovnet"
+)
+
+// SriovnetOps is a wrapper type over sriovnet library
+type SriovnetOps interface {
+	GetUplinkRepresentor(vfPciAddress string) (string, error)
+}
+
+type defaultSriovnetOps struct {
+}
+
+var sriovnetOps SriovnetOps = &defaultSriovnetOps{}
+
+// SetSriovnetOpsInst method would be used by unit tests in other packages
+func SetSriovnetOpsInst(inst SriovnetOps) {
+	sriovnetOps = inst
+}
+
+// GetSriovnetOps will be invoked by functions in other packages that would need access to the sriovnet library methods.
+func GetSriovnetOps() SriovnetOps {
+	return sriovnetOps
+}
+
+func (defaultSriovnetOps) GetUplinkRepresentor(vfPciAddress string) (string, error) {
+	return sriovnet.GetUplinkRepresentor(vfPciAddress)
+}

pkg/utils/utils.go

@@ -28,6 +28,19 @@ import (
 var (
 	sysBusPci = "/sys/bus/pci/devices"
 	devDir    = "/dev"
+	// getPfEswitchMode returns PF's eswitch mode for the given VF
+	// If device is not VF then it will return its own eswitch mode
+	getPfEswitchMode = func(pciAddr string) (string, error) {
+		pfAddr, err := GetPfAddr(pciAddr)
+		if err != nil {
+			return "", fmt.Errorf("error getting PF eswitch mode for PCI device %s %v", pciAddr, err)
+		}
+		devLinkDeviceAttrs, err := GetDevLinkDeviceEswitchAttrs(pfAddr)
+		if err != nil {
+			return "", err
+		}
+		return devLinkDeviceAttrs.Mode, nil
+	}
 )
 
 const (
@@ -76,7 +89,21 @@ func GetPfName(pciAddr string) (string, error) {
 		}
 		return "", err
 	} else if len(files) > 0 {
+		pfEswitchMode, err := getPfEswitchMode(pciAddr)
+		if err != nil {
+			return "", err
+		}
+		if pfEswitchMode == "switchdev" {
+			name, err := GetSriovnetOps().GetUplinkRepresentor(pciAddr)
+
+			if err != nil {
+				return "", err
+			}
+
+			return name, nil
+		}
 		return files[0].Name(), nil
+
 	}
 	return "", fmt.Errorf("the PF name is not found for device %s", pciAddr)
 }

pkg/utils/utils_test.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	mocks "github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils/mocks"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
@@ -396,35 +397,52 @@ var _ = Describe("In the utils package", func() {
 	)
 
 	DescribeTable("getting PF names",
-		func(fs *FakeFilesystem, device string, expected string, shouldFail bool) {
+		func(fs *FakeFilesystem, eswitchMode string, device string, expected string, shouldFail bool) {
+			getPfEswitchMode = func(pciAddr string) (string, error) {
+				return eswitchMode, nil
+			}
+			fakeSriovnetOps := mocks.SriovnetOps{}
+			fakeSriovnetOps.
+				On("GetUplinkRepresentor", "0000:01:10.0").
+				Return("fakeSwitchdevPF", nil)
+			SetSriovnetOpsInst(&fakeSriovnetOps)
+
 			defer fs.Use()()
 			actual, err := GetPfName(device)
 			Expect(actual).To(Equal(expected))
 			assertShouldFail(err, shouldFail)
 		},
-		Entry("device doesn't exist", &FakeFilesystem{}, "0000:01:10.0", nil, true),
+		Entry("device doesn't exist", &FakeFilesystem{}, "fake", "0000:01:10.0", nil, true),
 		Entry("device is a VF and interface name exists",
 			&FakeFilesystem{Dirs: []string{"sys/bus/pci/devices/0000:01:10.0/physfn/net/fakePF"}},
-			"0000:01:10.0", "fakePF", false,
+			"fake", "0000:01:10.0", "fakePF", false,
 		),
 		Entry("device is a VF and interface name does not exist",
 			&FakeFilesystem{Dirs: []string{"sys/bus/pci/devices/0000:01:10.0/physfn/net/"}},
-			"0000:01:10.0", "", true,
+			"fake", "0000:01:10.0", "", true,
 		),
 		Entry("device is a PF and interface name exists",
 			&FakeFilesystem{Dirs: []string{"sys/bus/pci/devices/0000:01:10.0/net/fakeIF"}},
-			"0000:01:10.0", "fakeIF", false,
+			"fake", "0000:01:10.0", "fakeIF", false,
 		),
 		Entry("device is a PF interface name does not exist",
 			&FakeFilesystem{Dirs: []string{"sys/bus/pci/devices/0000:01:10.0/net/fakeIF"}},
-			"0000:01:10.0", "fakeIF", false,
+			"fake", "0000:01:10.0", "fakeIF", false,
 		),
 		Entry("net is not a directory at all",
 			&FakeFilesystem{
 				Dirs:  []string{"sys/bus/pci/devices/0000:01:10.0"},
 				Files: map[string][]byte{"sys/bus/pci/devices/0000:01:10.0/net": []byte("junk")},
 			},
-			"0000:01:10.0", "", true,
+			"fake", "0000:01:10.0", "", true,
+		),
+		Entry("device is a VF and PF is in switchdev mode",
+			&FakeFilesystem{
+				Dirs: []string{"sys/bus/pci/devices/0000:01:10.0/physfn/net/",
+					"sys/bus/pci/devices/0000:01:10.0/physfn/net/fakeVF",
+					"sys/bus/pci/devices/0000:01:10.0/physfn/net/fakePF"},
+			},
+			"switchdev", "0000:01:10.0", "fakeSwitchdevPF", false,
 		),
 	)