Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MCO-1017: MCO-1018 MCO-1019: MCO-1020: MCO-1021: OCPBUGS-32583: OCPBUGS-32679: Implement PinnedImageSets #4303

Merged
merged 18 commits into from
Apr 26, 2024
Merged

MCO-1017: MCO-1018 MCO-1019: MCO-1020: MCO-1021: OCPBUGS-32583: OCPBUGS-32679: Implement PinnedImageSets #4303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
996688b
bump deps
hexfusion Apr 13, 2024
6be73bf
daemon: add CRI client
hexfusion Apr 13, 2024
380c200
daemon: add pinned image set manager
hexfusion Apr 13, 2024
ce9933f
controller: add pinned image set controller
hexfusion Apr 13, 2024
08f6549
daemon: ensure informer factory starts after observing feature gates
hexfusion Apr 15, 2024
413b943
daemon: improve caching and log reporting of status
hexfusion Apr 16, 2024
a2f6ba1
daemon: better handle clearing workload when a set is mutated
hexfusion Apr 16, 2024
9b00591
deps: bump openshift/api and openshift/client-go
hexfusion Apr 17, 2024
7fb269f
install: sync crds
hexfusion Apr 17, 2024
eb6d2bf
mcc: update pinned image set controller to use correct degraded type
hexfusion Apr 17, 2024
8509ca9
*: plumb in pinned image set degraded condition
hexfusion Apr 17, 2024
eaf6206
bootstrap-e2e: add remote-bucket flag
hexfusion Apr 18, 2024
97d6a4c
pkg: cleanup controller and improve tests
hexfusion Apr 18, 2024
baa37ae
*: add MCP PoolSynchronizersStatus support for pinned image sets
hexfusion Apr 23, 2024
6ce48d6
daemon/pinnedimageset: ensure pool is degraded if out of space
hexfusion Apr 25, 2024
8a5f52c
daemon/pinnedimagesets: retry if local node is not found by informer
hexfusion Apr 25, 2024
836dd8d
daemon/pinnedimagesets: ensure sync on first boot
hexfusion Apr 25, 2024
fd2c36e
daemon/pinned-image-set: ensure cleanup of crio config file on delete
hexfusion Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion cmd/machine-config-controller/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"time"

configv1 "github.com/openshift/api/config/v1"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/openshift/machine-config-operator/cmd/common"
"github.com/openshift/machine-config-operator/internal/clients"
Expand All @@ -16,6 +17,7 @@ import (
kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config"
machinesetbootimage "github.com/openshift/machine-config-operator/pkg/controller/machine-set-boot-image"
"github.com/openshift/machine-config-operator/pkg/controller/node"
"github.com/openshift/machine-config-operator/pkg/controller/pinnedimageset"
"github.com/openshift/machine-config-operator/pkg/controller/render"
"github.com/openshift/machine-config-operator/pkg/controller/template"
"github.com/openshift/machine-config-operator/pkg/version"
Expand Down Expand Up @@ -103,6 +105,19 @@ func runStartCmd(_ *cobra.Command, _ []string) {

enabled, disabled := getEnabledDisabledFeatures(features)
klog.Infof("FeatureGates initialized: enabled=%v disabled=%v", enabled, disabled)
if features.Enabled(configv1.FeatureGatePinnedImages) && features.Enabled(configv1.FeatureGateMachineConfigNodes) {
pinnedImageSet := pinnedimageset.New(
ctrlctx.InformerFactory.Machineconfiguration().V1alpha1().PinnedImageSets(),
ctrlctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
ctrlctx.ClientBuilder.KubeClientOrDie("pinned-image-set-controller"),
ctrlctx.ClientBuilder.MachineConfigClientOrDie("pinned-image-set-controller"),
)

go pinnedImageSet.Run(2, ctrlctx.Stop)
// start the informers again to enable feature gated types.
// see comments in SharedInformerFactory interface.
ctrlctx.InformerFactory.Start(ctrlctx.Stop)
}
case <-time.After(1 * time.Minute):
klog.Errorf("timed out waiting for FeatureGate detection")
os.Exit(1)
Expand All @@ -113,7 +128,7 @@ func runStartCmd(_ *cobra.Command, _ []string) {
}
go draincontroller.Run(5, ctrlctx.Stop)

// wait here in this function until the context gets cancelled (which tells us whe were being shut down)
// wait here in this function until the context gets cancelled (which tells us when we are being shut down)
<-ctx.Done()
}

Expand Down
36 changes: 35 additions & 1 deletion cmd/machine-config-daemon/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,15 @@ import (
"os"
"time"

"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/client-go/tools/clientcmd"

configv1 "github.com/openshift/api/config/v1"
"github.com/openshift/machine-config-operator/internal/clients"
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
"github.com/openshift/machine-config-operator/pkg/daemon"
"github.com/openshift/machine-config-operator/pkg/daemon/constants"
"github.com/openshift/machine-config-operator/pkg/daemon/cri"
"github.com/openshift/machine-config-operator/pkg/version"
"github.com/spf13/cobra"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -191,6 +195,7 @@ func runStartCmd(_ *cobra.Command, _ []string) {
klog.Fatalf("Failed to initialize: %v", err)
}

// start config informer early because feature gate depends on it
ctrlctx.ConfigInformerFactory.Start(ctrlctx.Stop)
ctrlctx.KubeInformerFactory.Start(stopCh)
ctrlctx.KubeNamespacedInformerFactory.Start(stopCh)
Expand All @@ -200,11 +205,41 @@ func runStartCmd(_ *cobra.Command, _ []string) {

select {
case <-ctrlctx.FeatureGateAccess.InitialFeatureGatesObserved():
// ok to start the rest of the informers now that we have observed the initial feature gates

featureGates, err := ctrlctx.FeatureGateAccess.CurrentFeatureGates()
if err != nil {
klog.Fatalf("Could not get FG: %v", err)
} else {
klog.Infof("FeatureGates initialized: knownFeatureGates=%v", featureGates.KnownFeatures())
if featureGates.Enabled(configv1.FeatureGatePinnedImages) && featureGates.Enabled(configv1.FeatureGateMachineConfigNodes) {
klog.Infof("Feature enabled: %s", configv1.FeatureGatePinnedImages)
criClient, err := cri.NewClient(ctx, constants.DefaultCRIOSocketPath)
if err != nil {
klog.Fatalf("Failed to initialize CRI client: %v", err)
}

prefetchTimeout := 2 * time.Minute
pinnedImageSetManager := daemon.NewPinnedImageSetManager(
startOpts.nodeName,
criClient,
ctrlctx.ClientBuilder.MachineConfigClientOrDie(componentName),
ctrlctx.InformerFactory.Machineconfiguration().V1alpha1().PinnedImageSets(),
ctrlctx.KubeInformerFactory.Core().V1().Nodes(),
ctrlctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
resource.MustParse(constants.MinFreeStorageAfterPrefetch),
constants.DefaultCRIOSocketPath,
constants.KubeletAuthFile,
constants.ContainerRegistryConfPath,
prefetchTimeout,
ctrlctx.FeatureGateAccess,
)

go pinnedImageSetManager.Run(2, stopCh)
hexfusion marked this conversation as resolved.
Show resolved Hide resolved
// start the informers for the pinned image set again after the feature gate is enabled this is allowed.
// see comments in SharedInformerFactory interface.
ctrlctx.InformerFactory.Start(stopCh)
}
}
case <-time.After(1 * time.Minute):
klog.Fatalf("Could not get FG, timed out: %v", err)
Expand All @@ -221,5 +256,4 @@ func runStartCmd(_ *cobra.Command, _ []string) {
os.Exit(255)
}
}

}
11 changes: 7 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ require (
github.com/google/renameio v0.1.0
github.com/imdario/mergo v0.3.13
github.com/opencontainers/go-digest v1.0.0
github.com/openshift/api v0.0.0-20240410141538-3c0461467316
github.com/openshift/client-go v0.0.0-20240408153607-64bd6feb83ae
github.com/openshift/api v0.0.0-20240422085825-2624175e9673
github.com/openshift/client-go v0.0.0-20240422164335-6c851f4919dd
github.com/openshift/cluster-config-operator v0.0.0-alpha.0.0.20231213185242-e4dc676febfe
github.com/openshift/library-go v0.0.0-20240412173449-eb2f24c36528
github.com/openshift/runtime-utils v0.0.0-20230921210328-7bdb5b9c177b
Expand All @@ -39,12 +39,14 @@ require (
github.com/vincent-petithory/dataurl v1.0.0
golang.org/x/net v0.19.0
golang.org/x/time v0.3.0
google.golang.org/grpc v1.58.3
k8s.io/api v0.29.2
k8s.io/apiextensions-apiserver v0.29.2
k8s.io/apimachinery v0.29.2
k8s.io/client-go v0.29.2
k8s.io/code-generator v0.29.2
k8s.io/component-base v0.29.2
k8s.io/cri-api v0.29.2
k8s.io/kubectl v0.29.2
k8s.io/kubelet v0.29.2
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
Expand Down Expand Up @@ -106,6 +108,7 @@ require (
go.mongodb.org/mongo-driver v1.11.3 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
go.tmz.dev/musttag v0.7.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect
k8s.io/cli-runtime v0.29.2 // indirect
Expand Down Expand Up @@ -176,7 +179,7 @@ require (
github.com/gobwas/glob v0.2.3 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
github.com/golang/protobuf v1.5.3 // indirect
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
Expand Down Expand Up @@ -242,7 +245,7 @@ require (
github.com/nishanths/exhaustive v0.11.0 // indirect
github.com/nishanths/predeclared v0.2.2 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
github.com/opencontainers/image-spec v1.1.0-rc5
github.com/opencontainers/runc v1.1.10 // indirect
github.com/opencontainers/runtime-spec v1.1.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
Expand Down
14 changes: 10 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,10 +689,10 @@ github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQB
github.com/opencontainers/runc v1.1.10/go.mod h1:+/R6+KmDlh+hOO8NkjmgkG9Qzvypzk0yXxAPYYR65+M=
github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/openshift/api v0.0.0-20240410141538-3c0461467316 h1:YfatsH1e28WTbuNUl3LZl6PKCr6zBaYLR7JSxg+LJSU=
github.com/openshift/api v0.0.0-20240410141538-3c0461467316/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
github.com/openshift/client-go v0.0.0-20240408153607-64bd6feb83ae h1:WzEMf853apLG0ZgkfmKvYXBKBqhzx7nalP306yQP1ck=
github.com/openshift/client-go v0.0.0-20240408153607-64bd6feb83ae/go.mod h1:YOfx7b9ieudQJImuo95BcVzifosCrCGBErbO/w/njBU=
github.com/openshift/api v0.0.0-20240422085825-2624175e9673 h1:D4qblu6z2A92fh7u9Nt1YskDtu+GySKiYP/D3tMWQ6A=
github.com/openshift/api v0.0.0-20240422085825-2624175e9673/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
github.com/openshift/client-go v0.0.0-20240422164335-6c851f4919dd h1:z5TPsTaB8Zzvv9fK/kVB6X+FG1GtwM56WfoanhlbyyQ=
github.com/openshift/client-go v0.0.0-20240422164335-6c851f4919dd/go.mod h1:OC07uJXbaW/s21N6XDucROlmfUOhMXD5OrY3ZN3DmiM=
github.com/openshift/cluster-config-operator v0.0.0-alpha.0.0.20231213185242-e4dc676febfe h1:wDQtyIbJJIoif2Ux0S+9MJWIWEGV0oG+iLm8WtqwdSw=
github.com/openshift/cluster-config-operator v0.0.0-alpha.0.0.20231213185242-e4dc676febfe/go.mod h1:SGUtv1pKZSzSVr2YCxXFvhE+LbGfI+vcetEhNicKayw=
github.com/openshift/kube-openapi v0.0.0-20230816122517-ffc8f001abb0 h1:GPlAy197Jkr+D0T2FNWanamraTdzS/r9ZkT29lxvHaA=
Expand Down Expand Up @@ -1338,6 +1338,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand All @@ -1354,6 +1356,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down Expand Up @@ -1430,6 +1434,8 @@ k8s.io/code-generator v0.29.2 h1:c9/iw2KnNpw2IRV+wwuG/Wns2TjPSgjWzbbjTevyiHI=
k8s.io/code-generator v0.29.2/go.mod h1:FwFi3C9jCrmbPjekhaCYcYG1n07CYiW1+PAPCockaos=
k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8=
k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM=
k8s.io/cri-api v0.29.2 h1:LLSeWVC3h1nVMpV9vHiE+mO3spDYmz/C0GvxH6p6tkg=
k8s.io/cri-api v0.29.2/go.mod h1:9fQTFm+wi4FLyqrkVUoMJiUB3mE74XrVvHz8uFY/sSw=
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 h1:pWEwq4Asjm4vjW7vcsmijwBhOr1/shsbSYiWXmNGlks=
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
Expand Down
Loading