From 0b0644c21b8334820af1d31eccdf4ef97397524d Mon Sep 17 00:00:00 2001 From: Benjamin Drung Date: Thu, 23 May 2019 19:23:18 +0200 Subject: [PATCH] Add support for InfiniBand The Prometheus node exporter collects metrics for the InfiniBand network protocol including the amount of packets sent and received, the number of times the link has been downed and how many times the link has recovered from an error state. Reading all those information from sysfs is better placed in the procfs library. Also collect the state, physical state, and the rate. Signed-off-by: Benjamin Drung --- fixtures.ttar | 211 +++++++++++++++++++++++++ sysfs/class_infiniband.go | 272 +++++++++++++++++++++++++++++++++ sysfs/class_infiniband_test.go | 170 +++++++++++++++++++++ 3 files changed, 653 insertions(+) create mode 100644 sysfs/class_infiniband.go create mode 100644 sysfs/class_infiniband_test.go diff --git a/fixtures.ttar b/fixtures.ttar index 951d909af..ff94014bf 100644 --- a/fixtures.ttar +++ b/fixtures.ttar @@ -609,6 +609,217 @@ Mode: 664 Directory: fixtures/sys/class Mode: 775 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband/mlx4_0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband/mlx4_0/ports +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband/mlx4_0/ports/1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/excessive_buffer_overrun_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/link_downed +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/link_error_recovery +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/local_link_integrity_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_rcv_constraint_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_rcv_data +Lines: 1 +2221223609 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_rcv_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_rcv_packets +Lines: 1 +87169372 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_rcv_remote_physical_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_rcv_switch_relay_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_xmit_constraint_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_xmit_data +Lines: 1 +26509113295 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_xmit_discards +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_xmit_packets +Lines: 1 +85734114 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/port_xmit_wait +Lines: 1 +3599 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/counters/symbol_error +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/phys_state +Lines: 1 +5: LinkUp +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/rate +Lines: 1 +40 Gb/sec (4X QDR) +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/1/state +Lines: 1 +4: ACTIVE +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband/mlx4_0/ports/2 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/excessive_buffer_overrun_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/link_downed +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/link_error_recovery +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/local_link_integrity_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_rcv_constraint_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_rcv_data +Lines: 1 +2460436784 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_rcv_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_rcv_packets +Lines: 1 +89332064 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_rcv_remote_physical_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_rcv_switch_relay_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_xmit_constraint_errors +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_xmit_data +Lines: 1 +26540356890 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_xmit_discards +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_xmit_packets +Lines: 1 +88622850 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/port_xmit_wait +Lines: 1 +3846 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/counters/symbol_error +Lines: 1 +0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/phys_state +Lines: 1 +5: LinkUp +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/rate +Lines: 1 +40 Gb/sec (4X QDR) +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/sys/class/infiniband/mlx4_0/ports/2/state +Lines: 1 +4: ACTIVE +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: fixtures/sys/class/net Mode: 775 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sysfs/class_infiniband.go b/sysfs/class_infiniband.go new file mode 100644 index 000000000..56a16a0c4 --- /dev/null +++ b/sysfs/class_infiniband.go @@ -0,0 +1,272 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package sysfs + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" +) + +const infinibandclassPath = "class/infiniband" + +// InfiniBandCounters contains info from files in +// /sys/class/infiniband//ports// for single port of one device. +type InfiniBandCounters struct { + LegacyPortMulticastRcvPackets *uint64 `fileName:"counters_ext/port_multicast_rcv_packets"` + LegacyPortMulticastXmitPackets *uint64 `fileName:"counters_ext/port_multicast_xmit_packets"` + LegacyPortRcvData64 *uint64 `fileName:"counters_ext/port_rcv_data_64"` + LegacyPortRcvPackets64 *uint64 `fileName:"counters_ext/port_rcv_packets_64"` + LegacyPortUnicastRcvPackets *uint64 `fileName:"counters_ext/port_unicast_rcv_packets"` + LegacyPortUnicastXmitPackets *uint64 `fileName:"counters_ext/port_unicast_xmit_packets"` + LegacyPortXmitData64 *uint64 `fileName:"counters_ext/port_xmit_data_64"` + LegacyPortXmitPackets64 *uint64 `fileName:"counters_ext/port_xmit_packets_64"` + + LinkDowned *uint64 `fileName:"counters/link_downed"` + LinkErrorRecovery *uint64 `fileName:"counters/link_error_recovery"` + MulticastRcvPackets *uint64 `fileName:"counters/multicast_rcv_packets"` + MulticastXmitPackets *uint64 `fileName:"counters/multicast_xmit_packets"` + PortRcvConstraintErrors *uint64 `fileName:"counters/port_rcv_constraint_errors"` + PortRcvData *uint64 `fileName:"counters/port_rcv_data"` + PortRcvDiscards *uint64 `fileName:"counters/port_rcv_discards"` + PortRcvErrors *uint64 `fileName:"counters/port_rcv_errors"` + PortRcvPackets *uint64 `fileName:"counters/port_rcv_packets"` + PortXmitConstraintErrors *uint64 `fileName:"counters/port_xmit_constraint_errors"` + PortXmitData *uint64 `fileName:"counters/port_xmit_data"` + PortXmitDiscards *uint64 `fileName:"counters/port_xmit_discards"` + PortXmitPackets *uint64 `fileName:"counters/port_xmit_packets"` + PortXmitWait *uint64 `fileName:"counters/port_xmit_wait"` + UnicastRcvPackets *uint64 `fileName:"counters/unicast_rcv_packets"` + UnicastXmitPackets *uint64 `fileName:"counters/unicast_xmit_packets"` +} + +type InfiniBandPort struct { + Name string + Port uint + State string // String representation from /sys/class/infiniband//ports//state + StateID uint // ID from /sys/class/infiniband//ports//state + PhysState string // String representation from /sys/class/infiniband//ports//phys_state + PhysStateID uint // String representation from /sys/class/infiniband//ports//phys_state + Rate uint64 // in bytes/second from /sys/class/infiniband//ports//rate + Counters InfiniBandCounters +} + +type InfiniBandClass map[string]map[uint]InfiniBandPort + +// InfiniBandClassDevices scans /sys/class/infiniband for devices and returns them as a list of names. +func (fs FS) InfiniBandClassDevices() ([]string, error) { + var res []string + path := fs.sys.Path(infinibandclassPath) + + devices, err := ioutil.ReadDir(path) + if err != nil { + return res, fmt.Errorf("cannot access %s dir %s", path, err) + } + + for _, deviceDir := range devices { + if deviceDir.Mode().IsRegular() { + continue + } + res = append(res, deviceDir.Name()) + } + + return res, nil +} + +// InfiniBandDevicePorts scans /sys/class/infiniband/ for ports and returns them as a list of integers. +func (fs FS) InfiniBandDevicePorts(device string) ([]uint, error) { + var res []uint + path := fs.sys.Path(infinibandclassPath, device, "ports") + + ports, err := ioutil.ReadDir(path) + if err != nil { + return res, fmt.Errorf("cannot access %s dir %s", path, err) + } + + for _, portDir := range ports { + if portDir.Mode().IsRegular() { + continue + } + port, err := strconv.ParseUint(portDir.Name(), 10, 32) + if err != nil { + return res, fmt.Errorf("failed to convert %s into uint", portDir.Name()) + } + res = append(res, uint(port)) + } + + return res, nil +} + +// InfiniBandClass returns info for all ports (port) of all InfiniBand devices (device) read from /sys/class/infiniband//ports/. +func (fs FS) InfiniBandClass() (InfiniBandClass, error) { + devices, err := fs.InfiniBandClassDevices() + if err != nil { + return nil, err + } + + infiniBandClass := InfiniBandClass{} + for _, device := range devices { + ports, err := fs.InfiniBandDevicePorts(device) + if err != nil { + return nil, err + } + + infiniBandClass[device] = map[uint]InfiniBandPort{} + for _, port := range ports { + path := fs.sys.Path(infinibandclassPath, device, "ports", fmt.Sprint(port)) + portClass, err := infiniBandClass.parseInfiniBandPort(path) + if err != nil { + return nil, err + } + portClass.Name = device + portClass.Port = port + infiniBandClass[device][port] = *portClass + } + } + return infiniBandClass, nil +} + +// Parse InfiniBand state. Expected format: ": " +func parseState(s string) (uint, string, error) { + var id uint + var name string + parts := strings.Split(s, ":") + if len(parts) != 2 { + return id, name, fmt.Errorf("failed to split %s into 'ID: NAME'", s) + } + name = strings.TrimSpace(parts[1]) + value, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 32) + if err != nil { + return id, name, fmt.Errorf("failed to convert %s into uint", strings.TrimSpace(parts[0])) + } + id = uint(value) + return id, name, nil +} + +// Parse rate (example: "100 Gb/sec (4X EDR)") and return it as bytes/second +func parseRate(s string) (uint64, error) { + var rate uint64 + parts := strings.Split(s, "Gb/sec") + if len(parts) != 2 { + return rate, fmt.Errorf("failed to split '%s' by 'Gb/sec'", s) + } + value, err := strconv.ParseFloat(strings.TrimSpace(parts[0]), 32) + if err != nil { + return rate, fmt.Errorf("failed to convert %s into uint", strings.TrimSpace(parts[0])) + } + rate = uint64(value * 125000000) + return rate, nil +} + +// parseInfiniBandPort scans predefined files in /sys/class/infiniband//ports/ +// directory and gets their contents. +func (InfiniBandClass) parseInfiniBandPort(portPath string) (*InfiniBandPort, error) { + infiniBandPort := InfiniBandPort{} + + content, err := ioutil.ReadFile(filepath.Join(portPath, "state")) + if err != nil { + return nil, err + } + id, name, err := parseState(string(content)) + if err != nil { + return nil, fmt.Errorf("could not parse state file in %s: %s", portPath, err) + } + infiniBandPort.State = name + infiniBandPort.StateID = id + + content, err = ioutil.ReadFile(filepath.Join(portPath, "phys_state")) + if err != nil { + return nil, err + } + id, name, err = parseState(string(content)) + if err != nil { + return nil, fmt.Errorf("could not parse phys_state file in %s: %s", portPath, err) + } + infiniBandPort.PhysState = name + infiniBandPort.PhysStateID = id + + content, err = ioutil.ReadFile(filepath.Join(portPath, "rate")) + if err != nil { + return nil, err + } + infiniBandPort.Rate, err = parseRate(string(content)) + if err != nil { + return nil, fmt.Errorf("could not parse rate file in %s: %s", portPath, err) + } + + interfaceElem := reflect.ValueOf(&infiniBandPort.Counters).Elem() + interfaceType := reflect.TypeOf(infiniBandPort.Counters) + for i := 0; i < interfaceElem.NumField(); i++ { + fieldType := interfaceType.Field(i) + fieldValue := interfaceElem.Field(i) + filename := fieldType.Tag.Get("fileName") + + if filename == "" { + panic(fmt.Errorf("field %s does not have a filename tag", fieldType.Name)) + } + + value, err := ioutil.ReadFile(filepath.Join(portPath, filename)) + + if err != nil { + if os.IsNotExist(err) || err.Error() == "operation not supported" || err.Error() == "invalid argument" { + continue + } + return nil, fmt.Errorf("could not access file %s: %s", filename, err) + } + + switch fieldValue.Kind() { + case reflect.Ptr: + var uint64ptr *uint64 + switch fieldValue.Type() { + case reflect.TypeOf(uint64ptr): + var uintValue uint64 + uintValue, err = strconv.ParseUint(strings.TrimSpace(string(value)), 10, 64) + if err != nil { + // Ugly workaround for handling https://github.com/prometheus/node_exporter/issues/966 + // when counters are `N/A (not available)`. + // This was already patched and submitted, see + // https://www.spinics.net/lists/linux-rdma/msg68596.html + // Remove this as soon as the fix lands in the enterprise distros. + if strings.Contains(err.Error(), "N/A (no PMA)") { + continue + } + return nil, fmt.Errorf("expected Uint64 value for %s, got: %s", fieldType.Name, value) + } + + // According to Mellanox, the following metrics "are divided by 4 unconditionally" + // as they represent the amount of data being transmitted and received per lane. + // Mellanox cards have 4 lanes per port, so all values must be multiplied by 4 + // to get the expected value. + switch filename { + case "counters/port_rcv_data", "counters/port_xmit_data", "counters/port_rcv_data_64", "counters/port_xmit_data_64": + uintValue *= 4 + } + + fieldValue.Set(reflect.ValueOf(&uintValue)) + default: + return nil, fmt.Errorf("unhandled pointer type %q", fieldValue.Type()) + } + default: + return nil, fmt.Errorf("unhandled type %q", fieldValue.Kind()) + } + } + + return &infiniBandPort, nil +} diff --git a/sysfs/class_infiniband_test.go b/sysfs/class_infiniband_test.go new file mode 100644 index 000000000..26dffb2d7 --- /dev/null +++ b/sysfs/class_infiniband_test.go @@ -0,0 +1,170 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package sysfs + +import ( + "reflect" + "testing" +) + +func TestParseSlowRate(t *testing.T) { + rate, err := parseRate("2.5 Gb/sec (1X SDR)") + if err != nil { + t.Fatal(err) + } + if rate != 312500000 { + t.Errorf("Result for InfiniBand rate not correct: want %v, have %v", 312500000, rate) + } +} + +func TestParseRate(t *testing.T) { + rate, err := parseRate("500 Gb/sec (4X HDR)") + if err != nil { + t.Fatal(err) + } + if rate != 62500000000 { + t.Errorf("Result for InfiniBand rate not correct: want %v, have %v", 62500000000, rate) + } +} + +func TestInfiniBandClassDevices(t *testing.T) { + fs, err := NewFS(sysTestFixtures) + if err != nil { + t.Fatal(err) + } + + devices, err := fs.InfiniBandClassDevices() + if err != nil { + t.Fatal(err) + } + + if len(devices) != 1 { + t.Errorf("Unexpected number of devices, want %d, have %d", 1, len(devices)) + } + if devices[0] != "mlx4_0" { + t.Errorf("Found unexpected device, want %s, have %s", "mlx4_0", devices[0]) + } +} + +func TestInfiniBandDevicePorts(t *testing.T) { + fs, err := NewFS(sysTestFixtures) + if err != nil { + t.Fatal(err) + } + + ports, err := fs.InfiniBandDevicePorts("mlx4_0") + if err != nil { + t.Fatal(err) + } + + if len(ports) != 2 { + t.Errorf("Unexpected number of ports, want %d, have %d", 2, len(ports)) + } + if !reflect.DeepEqual([]uint{1, 2}, ports) { + t.Errorf("Result not correct: want %v, have %v", []uint{1, 2}, ports) + } +} + +func TestInfiniBandClass(t *testing.T) { + fs, err := NewFS(sysTestFixtures) + if err != nil { + t.Fatal(err) + } + + nc, err := fs.InfiniBandClass() + if err != nil { + t.Fatal(err) + } + + var ( + port1LinkDowned uint64 = 0 + port1LinkErrorRecovery uint64 = 0 + port1PortRcvConstraintErrors uint64 = 0 + port1PortRcvData uint64 = 8884894436 + port1PortRcvErrors uint64 = 0 + port1PortRcvPackets uint64 = 87169372 + port1PortXmitConstraintErrors uint64 = 0 + port1PortXmitData uint64 = 106036453180 + port1PortXmitDiscards uint64 = 0 + port1PortXmitPackets uint64 = 85734114 + port1PortXmitWait uint64 = 3599 + + port2LinkDowned uint64 = 0 + port2LinkErrorRecovery uint64 = 0 + port2PortRcvConstraintErrors uint64 = 0 + port2PortRcvData uint64 = 9841747136 + port2PortRcvErrors uint64 = 0 + port2PortRcvPackets uint64 = 89332064 + port2PortXmitConstraintErrors uint64 = 0 + port2PortXmitData uint64 = 106161427560 + port2PortXmitDiscards uint64 = 0 + port2PortXmitPackets uint64 = 88622850 + port2PortXmitWait uint64 = 3846 + ) + + infiniBandClass := InfiniBandClass{ + "mlx4_0": { + 1: { + Name: "mlx4_0", + Port: 1, + State: "ACTIVE", + StateID: 4, + PhysState: "LinkUp", + PhysStateID: 5, + Rate: 5000000000, + Counters: InfiniBandCounters{ + LinkDowned: &port1LinkDowned, + LinkErrorRecovery: &port1LinkErrorRecovery, + PortRcvConstraintErrors: &port1PortRcvConstraintErrors, + PortRcvData: &port1PortRcvData, + PortRcvErrors: &port1PortRcvErrors, + PortRcvPackets: &port1PortRcvPackets, + PortXmitConstraintErrors: &port1PortXmitConstraintErrors, + PortXmitData: &port1PortXmitData, + PortXmitDiscards: &port1PortXmitDiscards, + PortXmitPackets: &port1PortXmitPackets, + PortXmitWait: &port1PortXmitWait, + }, + }, + 2: { + Name: "mlx4_0", + Port: 2, + State: "ACTIVE", + StateID: 4, + PhysState: "LinkUp", + PhysStateID: 5, + Rate: 5000000000, + Counters: InfiniBandCounters{ + LinkDowned: &port2LinkDowned, + LinkErrorRecovery: &port2LinkErrorRecovery, + PortRcvConstraintErrors: &port2PortRcvConstraintErrors, + PortRcvData: &port2PortRcvData, + PortRcvErrors: &port2PortRcvErrors, + PortRcvPackets: &port2PortRcvPackets, + PortXmitConstraintErrors: &port2PortXmitConstraintErrors, + PortXmitData: &port2PortXmitData, + PortXmitDiscards: &port2PortXmitDiscards, + PortXmitPackets: &port2PortXmitPackets, + PortXmitWait: &port2PortXmitWait, + }, + }, + }, + } + + if !reflect.DeepEqual(infiniBandClass, nc) { + t.Errorf("Result not correct: want %v, have %v", infiniBandClass, nc) + } +}