feat: enable auto-setup of BMC

This PR will enable the ability to automatically configure a sidero user
in the BMC if we can complete the process. It removes the need for users
to configure this info per-server if enabled.

Signed-off-by: Spencer Smith <[email protected]>

Dockerfile

@@ -136,13 +136,21 @@ RUN chmod +x /agent
 
 FROM base AS initramfs-archive-amd64
 WORKDIR /initramfs
+COPY --from=pkg-ca-certificates / .
+COPY --from=pkg-musl / .
+COPY --from=pkg-libressl / .
+COPY --from=pkg-ipmitool / .
 COPY --from=agent-build-amd64 /agent ./init
 COPY --from=pkg-linux-firmware-amd64 /lib/firmware/bnx2 ./lib/firmware/bnx2
 COPY --from=pkg-linux-firmware-amd64 /lib/firmware/bnx2x ./lib/firmware/bnx2x
 RUN set -o pipefail && find . 2>/dev/null | cpio -H newc -o | xz -v -C crc32 -0 -e -T 0 -z >/initramfs.xz
 
 FROM base AS initramfs-archive-arm64
 WORKDIR /initramfs
+COPY --from=pkg-ca-certificates / .
+COPY --from=pkg-musl / .
+COPY --from=pkg-libressl / .
+COPY --from=pkg-ipmitool / .
 COPY --from=agent-build-arm64 /agent ./init
 COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2 ./lib/firmware/bnx2
 COPY --from=pkg-linux-firmware-arm64 /lib/firmware/bnx2x ./lib/firmware/bnx2x

app/cluster-api-provider-sidero/config/rbac/role.yaml

@@ -18,8 +18,12 @@ rules:
   resources:
   - secrets
   verbs:
+  - create
+  - delete
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - cluster.x-k8s.io

app/cluster-api-provider-sidero/controllers/metalmachine_controller.go

@@ -51,7 +51,7 @@ type MetalMachineReconciler struct {
 // +kubebuilder:rbac:groups=metal.sidero.dev,resources=serverclasses/status,verbs=get;list;watch;
 // +kubebuilder:rbac:groups=metal.sidero.dev,resources=servers,verbs=get;list;watch;
 // +kubebuilder:rbac:groups=metal.sidero.dev,resources=servers/status,verbs=get;update;patch
-// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 
 func (r *MetalMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, err error) {

app/metal-controller-manager/api/v1alpha1/server_types.go

@@ -11,6 +11,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -34,13 +35,24 @@ type BMC struct {
 	// Source for the password value. Cannot be used if Pass is not empty.
 	// +optional
 	PassFrom *CredentialSource `json:"passFrom,omitempty"`
+	// BMC Interface Type. Defaults to lanplus.
+	// +optional
+	Interface string `json:"interface,omitempty"`
 }
 
 // CredentialSource defines a reference to the credential value.
 type CredentialSource struct {
-	// Selects a key of a secret in the cluster namespace
-	// +optional
-	SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef,omitempty"`
+	SecretKeyRef *SecretKeyRef `json:"secretKeyRef,omitempty"`
+}
+
+// SecretKeyRef defines a ref to a given key within a secret.
+type SecretKeyRef struct {
+	// Namespace and name of credential secret
+	// nb: can't use namespacedname here b/c it doesn't have json tags in the struct :(
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+	// Key to select
+	Key string `json:"key"`
 }
 
 // Resolve the value using the references.
@@ -55,7 +67,14 @@ func (source *CredentialSource) Resolve(ctx context.Context, reader client.Clien
 
 	var secrets corev1.Secret
 
-	if err := reader.Get(ctx, client.ObjectKey{Namespace: corev1.NamespaceDefault, Name: source.SecretKeyRef.Name}, &secrets); err != nil {
+	if err := reader.Get(
+		ctx,
+		types.NamespacedName{
+			Namespace: source.SecretKeyRef.Namespace,
+			Name:      source.SecretKeyRef.Name,
+		},
+		&secrets,
+	); err != nil {
 		return "", fmt.Errorf("error getting secret %q: %w", source.SecretKeyRef.Name, err)
 	}
 

app/metal-controller-manager/cmd/agent/main.go

@@ -6,15 +6,19 @@ package main
 
 import (
 	"context"
+	"crypto/rand"
+	"errors"
 	"fmt"
 	"log"
+	"math/big"
 	"net"
 	"os"
 	"sync"
 	"time"
 
 	"github.com/talos-systems/go-blockdevice/blockdevice"
 	"github.com/talos-systems/go-blockdevice/blockdevice/util"
+	kmsg "github.com/talos-systems/go-kmsg"
 	"github.com/talos-systems/go-procfs/procfs"
 	"github.com/talos-systems/go-retry/retry"
 	"github.com/talos-systems/go-smbios/smbios"
@@ -23,7 +27,9 @@ import (
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
 
+	"github.com/talos-systems/sidero/app/metal-controller-manager/api/v1alpha1"
 	"github.com/talos-systems/sidero/app/metal-controller-manager/internal/api"
+	"github.com/talos-systems/sidero/app/metal-controller-manager/internal/power/ipmi"
 	"github.com/talos-systems/sidero/app/metal-controller-manager/pkg/constants"
 )
 
@@ -68,14 +74,14 @@ func setup() error {
 		return err
 	}
 
-	kmsg, err := os.OpenFile("/dev/kmsg", os.O_RDWR|unix.O_CLOEXEC|unix.O_NONBLOCK|unix.O_NOCTTY, 0o666)
-	if err != nil {
-		return fmt.Errorf("failed to open /dev/kmsg: %w", err)
+	if err := kmsg.SetupLogger(nil, "[sidero]", nil); err != nil {
+		return err
 	}
 
-	log.SetOutput(kmsg)
-	log.SetPrefix("[sidero]" + " ")
-	log.SetFlags(0)
+	// Set the PATH env var.
+	if err := os.Setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"); err != nil {
+		return errors.New("error setting PATH")
+	}
 
 	return nil
 }
@@ -239,6 +245,17 @@ func mainFunc() error {
 
 	log.Println("Registration complete")
 
+	if createResp.GetSetupBmc() {
+		log.Println("Attempting to automatically discover and configure BMC")
+
+		// nb: we don't consider failure to get BMC info a hard failure.
+		//     users can always patch the bmc info to the server themselves.
+		err := attemptBMCSetup(ctx, client, s)
+		if err != nil {
+			log.Printf("encountered error setting up BMC. skipping setup: %q", err.Error())
+		}
+	}
+
 	ips, err := talosnet.IPAddrs()
 	if err != nil {
 		log.Println("failed to discover IPs")
@@ -349,3 +366,163 @@ func mainFunc() error {
 func main() {
 	shutdown(mainFunc())
 }
+
+func attemptBMCSetup(ctx context.Context, client api.AgentClient, s *smbios.SMBIOS) error {
+	uuid, err := s.SystemInformation().UUID()
+	if err != nil {
+		return err
+	}
+
+	bmcInfo := &api.BMCInfo{}
+
+	// Create "open" client
+	bmcSpec := v1alpha1.BMC{
+		Interface: "open",
+	}
+
+	ipmiClient, err := ipmi.NewClient(bmcSpec)
+	if err != nil {
+		return err
+	}
+
+	// Fetch BMC IP
+	ipResp, err := ipmiClient.GetBMCIP()
+	if err != nil {
+		return err
+	}
+
+	bmcIP := net.IP(ipResp.Data)
+	bmcInfo.Ip = bmcIP.String()
+
+	// Get user summary to see how many user slots
+	summResp, err := ipmiClient.GetUserSummary()
+	if err != nil {
+		return err
+	}
+
+	maxUsers := summResp.MaxUsers & 0x1F // Only bits [0:5] provide this number
+
+	// Check if sidero user already exists by combing through all userIDs
+	// nb: we start looking at user id 2, because 1 should always be an unamed admin user and
+	//     we don't want to confuse that unnamed admin with an open slot we can take over.
+	exists := false
+	sideroUserID := uint8(0)
+
+	for i := uint8(2); i <= maxUsers; i++ {
+		userRes, err := ipmiClient.GetUserName(i)
+		if err != nil {
+			// nb: A failure here actually seems to mean that the user slot is unused,
+			// even though you can also have a slot with empty user as well. *scratches head*
+			// We'll take note of this spot if we haven't already found another empty one.
+			if sideroUserID == 0 {
+				sideroUserID = i
+			}
+
+			continue
+		}
+
+		// Found pre-existing sidero user
+		if userRes.Username == "sidero" {
+			exists = true
+			sideroUserID = i
+			log.Printf("Sidero user already present in slot %d. We'll claim it as our very own.\n", i)
+
+			break
+		} else if userRes.Username == "" && sideroUserID == 0 {
+			// If this is the first empty user that's not the UID 1 (which we skip),
+			// we'll take this spot for sidero user
+			log.Printf("Found empty user slot %d. Noting as a possible place for sidero user.\n", i)
+			sideroUserID = i
+		}
+	}
+
+	// User didn't pre-exist and there's no room
+	// Return without sidero user :(
+	if sideroUserID == 0 {
+		return errors.New("no slot available for sidero user")
+	}
+
+	// Not already present and there's an empty slot so we add sidero user
+	if !exists {
+		log.Printf("Adding sidero user to slot %d\n", sideroUserID)
+
+		_, err := ipmiClient.SetUserName(sideroUserID, "sidero")
+		if err != nil {
+			return err
+		}
+	}
+
+	// Reset pass for sidero user
+	// nb: we _always_ reset the user pass because we can't ever get
+	//     it back out when we find an existing sidero user.
+	pass, err := genPass16()
+	if err != nil {
+		return err
+	}
+
+	_, err = ipmiClient.SetUserPass(sideroUserID, pass)
+	if err != nil {
+		return err
+	}
+
+	// Make sidero an admin
+	// Options: 0xD1 == Callin false, Link false, IPMI Msg true, Channel 1
+	// Limits: 0x03 == Administrator
+	// Session: 0x00 No session limit
+	_, err = ipmiClient.SetUserAccess(0xD1, sideroUserID, 0x04, 0x00)
+	if err != nil {
+		return err
+	}
+
+	// Enable the sidero user
+	_, err = ipmiClient.EnableUser(sideroUserID)
+	if err != nil {
+		return err
+	}
+
+	// Finally fill in info for update request
+	bmcInfo.User = "sidero"
+	bmcInfo.Pass = pass
+
+	// Attempt to update server object
+	err = retry.Constant(5*time.Minute, retry.WithUnits(30*time.Second), retry.WithErrorLogging(true)).Retry(func() error {
+		ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+		defer cancel()
+
+		_, err = client.UpdateBMCInfo(
+			ctx,
+			&api.UpdateBMCInfoRequest{
+				Uuid:    uuid.String(),
+				BmcInfo: bmcInfo,
+			},
+		)
+
+		if err != nil {
+			return retry.ExpectedError(err)
+		}
+
+		return nil
+	})
+
+	return nil
+}
+
+// Returns a random pass string of len 16.
+func genPass16() (string, error) {
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+
+	b := make([]rune, 16)
+	for i := range b {
+		rando, err := rand.Int(
+			rand.Reader,
+			big.NewInt(int64(len(letterRunes))),
+		)
+		if err != nil {
+			return "", err
+		}
+
+		b[i] = letterRunes[rando.Int64()]
+	}
+
+	return string(b), nil
+}