Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
@prodanlabs
prodanlabs committed Jan 22, 2022
1 parent a40ed12 commit 6af72db
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 78 deletions.
3 changes: 3 additions & 0 deletions pkg/karmadactl/cmdinit/cmdinit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ func newCmdDockerInit(cmdOut io.Writer, parentCommand string) *cobra.Command {
}
flags := cmd.Flags()
flags.StringVarP(&opts.KarmadaAPIServerHostPort, "port", "p", "32443", "Karmada apiserver service host port")
flags.StringVarP(&opts.DockerNetwork, "docker-network", "", "karmada", "docker network name")
flags.StringVarP(&opts.DockerNetworkSubnet, "docker-network-subnet", "", "182.16.0.0/16", "docker network subnet")
flags.StringVarP(&opts.DockerNetworkGateway, "docker-network-gateway", "", "182.16.0.1", "docker network gateway")
options.AddFlags(flags)
return cmd
}
Expand Down
5 changes: 2 additions & 3 deletions pkg/karmadactl/cmdinit/docker/check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,15 @@ func (d *CommandInitDockerOption) WaitContainerReady(containerID string, sleepTi
if err != nil {
return err
}
containerIP := container.NetworkSettings.Networks["karmada"].IPAddress
if err := wait.Poll(time.Second, timeout, func() (bool, error) {
if !container.State.Running {
klog.Warningf("Container: %s not ready. Status: %s. ContainerIP: %s", strings.Trim(container.Name, "/"), container.State.Status, containerIP)
klog.Warningf("Container: %s not ready. Status: %s.", strings.Trim(container.Name, " / "), container.State.Status)
return false, nil
}
return true, nil
}); err != nil {
return err
}
klog.Infof("Container: %s is ready. Status: %s. ContainerIP: %s", strings.Trim(container.Name, "/"), container.State.Status, containerIP)
klog.Infof("Container: %s is ready. Status: %s.", strings.Trim(container.Name, "/"), container.State.Status)
return nil
}
37 changes: 23 additions & 14 deletions pkg/karmadactl/cmdinit/docker/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ package docker
import (
"fmt"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network"
"github.com/docker/go-connections/nat"
"k8s.io/klog/v2"

"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options"
)
Expand All @@ -21,7 +23,7 @@ const (
karmadaControllerManagerContainerAndHostName = "karmada-controller-manager"
karmadaWebhookContainerAndHostName = "karmada-webhook"
karmadaAggregatedAPIServerContainerAndHostName = "karmada-aggregated-apiserver"
externalName = "karmada-aggregated-apiserver.karmada-system.svc.cluster.local"
aaExternalName = "karmada-aggregated-apiserver.karmada-system.svc.cluster.local"
webhookExternalName = "karmada-webhook.karmada-system.svc"
)

Expand Down Expand Up @@ -78,7 +80,7 @@ func (d *CommandInitDockerOption) createEtcdContainer() (string, error) {
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {},
d.DockerNetwork: {},
},
}, nil, etcdContainerAndHostName)
if err != nil {
Expand Down Expand Up @@ -146,12 +148,12 @@ func (d *CommandInitDockerOption) createKarmadaAPIServerContainer() (string, err
etcdContainerAndHostName,
},
ExtraHosts: []string{
externalName + ":166.233.0.167",
webhookExternalName + ":166.233.0.166",
aaExternalName + ":" + d.aggregatedAPIServerContainerIP,
webhookExternalName + ":" + d.webhookContainerIP,
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {},
d.DockerNetwork: {},
},
}, nil, karmadaAPIServerContainerAndHostName)
if err != nil {
Expand Down Expand Up @@ -201,7 +203,7 @@ func (d *CommandInitDockerOption) createKubeControllerManagerContainer() (string
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {},
d.DockerNetwork: {},
},
}, nil, kubeControllerManagerContainerAndHostName)
if err != nil {
Expand All @@ -225,7 +227,7 @@ func (d *CommandInitDockerOption) createKarmadaSchedulerContainer() (string, err
"--secure-port=10351",
"--feature-gates=Failover=true",
"--enable-scheduler-estimator=true",
"--leader-elect-resource-namespace=karmada-system",
//"--leader-elect-resource-namespace=karmada-system",
"--leader-elect=true",
"--v=4",
},
Expand All @@ -240,7 +242,7 @@ func (d *CommandInitDockerOption) createKarmadaSchedulerContainer() (string, err
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {},
d.DockerNetwork: {},
},
}, nil, karmadaSchedulerContainerAndHostName)
if err != nil {
Expand Down Expand Up @@ -276,7 +278,7 @@ func (d *CommandInitDockerOption) createKarmadaControllerManagerContainer() (str
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {},
d.DockerNetwork: {},
},
}, nil, karmadaControllerManagerContainerAndHostName)
if err != nil {
Expand Down Expand Up @@ -324,9 +326,9 @@ func (d *CommandInitDockerOption) createKarmadaWebhookContainer() (string, error
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {
d.DockerNetwork: {
IPAMConfig: &network.EndpointIPAMConfig{
IPv4Address: "166.233.0.166",
IPv4Address: d.webhookContainerIP,
},
},
},
Expand Down Expand Up @@ -372,13 +374,13 @@ func (d *CommandInitDockerOption) createKarmadaAggregatedAPIServerContainer() (s
etcdContainerAndHostName,
},
ExtraHosts: []string{
webhookExternalName + ":166.233.0.166",
webhookExternalName + ":" + d.webhookContainerIP,
},
}, &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"karmada": {
d.DockerNetwork: {
IPAMConfig: &network.EndpointIPAMConfig{
IPv4Address: "166.233.0.167",
IPv4Address: d.aggregatedAPIServerContainerIP,
},
},
},
Expand All @@ -389,3 +391,10 @@ func (d *CommandInitDockerOption) createKarmadaAggregatedAPIServerContainer() (s

return body.ID, nil
}

// startContainer start the container.
func (d *CommandInitDockerOption) startContainer(containerID string) {
if err := d.cli.ContainerStart(d.ctx, containerID, types.ContainerStartOptions{}); err != nil {
klog.Exit(err)
}
}
57 changes: 26 additions & 31 deletions pkg/karmadactl/cmdinit/docker/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strings"
"time"

"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
certutil "k8s.io/client-go/util/cert"
"k8s.io/klog/v2"
Expand All @@ -22,10 +21,15 @@ import (

// CommandInitDockerOption holds all flags options for init.
type CommandInitDockerOption struct {
hostIP net.IP
KarmadaAPIServerHostPort string
cli *client.Client
ctx context.Context
hostIP net.IP
DockerNetwork string
DockerNetworkSubnet string
DockerNetworkGateway string
KarmadaAPIServerHostPort string
cli *client.Client
ctx context.Context
aggregatedAPIServerContainerIP string
webhookContainerIP string
}

// Validate Check that there are enough flags to run the command.
Expand Down Expand Up @@ -61,6 +65,9 @@ func (d *CommandInitDockerOption) Complete() error {
return err
}

if err := d.assignContainerIP(); err != nil {
return err
}
return nil
}

Expand All @@ -85,7 +92,7 @@ func (d *CommandInitDockerOption) genCerts() error {
karmadaAPIServerContainerAndHostName,
karmadaWebhookContainerAndHostName,
karmadaAggregatedAPIServerContainerAndHostName,
externalName,
aaExternalName,
webhookExternalName,
}
karmadaDNS = append(karmadaDNS, utils.FlagsDNS(options.ExternalDNS)...)
Expand Down Expand Up @@ -119,7 +126,7 @@ func (d *CommandInitDockerOption) genCerts() error {
}

func (d *CommandInitDockerOption) createKubeConfig() error {
// Container
// kubeconfig mounted to the container
karmadaContainerServerURL := fmt.Sprintf("https://%s:5443", karmadaAPIServerContainerAndHostName)
// local
karmadaServerURL := fmt.Sprintf("https://%s:%s", d.hostIP, d.KarmadaAPIServerHostPort)
Expand Down Expand Up @@ -193,21 +200,19 @@ func (d *CommandInitDockerOption) initKarmadaAPIServer() error {
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, etcdContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
if err := d.WaitContainerReady(etcdContainerID, 10*time.Second, 30*time.Second); err != nil {
d.startContainer(etcdContainerID)

if err := d.WaitContainerReady(etcdContainerID, 5*time.Second, 30*time.Second); err != nil {
return err
}

apiContainerID, err := d.createKarmadaAPIServerContainer()
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, apiContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
if err := d.WaitContainerReady(apiContainerID, 15*time.Second, 60*time.Second); err != nil {
d.startContainer(apiContainerID)

if err := d.WaitContainerReady(apiContainerID, 10*time.Second, 60*time.Second); err != nil {
return err
}
return nil
Expand All @@ -218,43 +223,33 @@ func (d *CommandInitDockerOption) initKarmadaComponent() error {
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, aaContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
if err := d.WaitContainerReady(aaContainerID, 10*time.Second, 15*time.Second); err != nil {
d.startContainer(aaContainerID)
if err := d.WaitContainerReady(aaContainerID, 5*time.Second, 15*time.Second); err != nil {
return err
}

kubeControllerManagerContainerID, err := d.createKubeControllerManagerContainer()
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, kubeControllerManagerContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
d.startContainer(kubeControllerManagerContainerID)

karmadaSchedulerContainerID, err := d.createKarmadaSchedulerContainer()
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, karmadaSchedulerContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
d.startContainer(karmadaSchedulerContainerID)

karmadaControllerManagerContainerID, err := d.createKarmadaControllerManagerContainer()
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, karmadaControllerManagerContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
d.startContainer(karmadaControllerManagerContainerID)

karmadaWebhookContainerID, err := d.createKarmadaWebhookContainer()
if err != nil {
return err
}
if err := d.cli.ContainerStart(d.ctx, karmadaWebhookContainerID, types.ContainerStartOptions{}); err != nil {
return err
}
d.startContainer(karmadaWebhookContainerID)
return nil
}
28 changes: 23 additions & 5 deletions pkg/karmadactl/cmdinit/docker/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package docker

import (
"fmt"
"strings"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
Expand All @@ -14,17 +15,18 @@ func (d *CommandInitDockerOption) networkCreate() error {
}

for _, v := range allNetworks {
if v.Name == "karmada" {
return fmt.Errorf("network karmada already exists. run 'docker network rm karmada'")
if v.Name == d.DockerNetwork {
return fmt.Errorf("network %q already exists. run 'docker network rm %s' or specify a new name via --docker-network", d.DockerNetwork, d.DockerNetwork)
}

}

_, err = d.cli.NetworkCreate(d.ctx, "karmada", types.NetworkCreate{
_, err = d.cli.NetworkCreate(d.ctx, d.DockerNetwork, types.NetworkCreate{
IPAM: &network.IPAM{
Config: []network.IPAMConfig{
{
Subnet: "166.233.0.0/16",
Gateway: "166.233.0.1",
Subnet: d.DockerNetworkSubnet,
Gateway: d.DockerNetworkGateway,
},
},
},
Expand All @@ -35,3 +37,19 @@ func (d *CommandInitDockerOption) networkCreate() error {

return nil
}

// assignContainerIP Get the IPs of aggregated-apiserver and webhook containers
func (d *CommandInitDockerOption) assignContainerIP() error {
net, err := d.cli.NetworkInspect(d.ctx, d.DockerNetwork, types.NetworkInspectOptions{})
if err != nil {
return err
}

ipRangeList := strings.Split(net.IPAM.Config[0].Subnet, "/")
d.webhookContainerIP = strings.Replace(ipRangeList[0], "0.0", "0.10", 1)
d.aggregatedAPIServerContainerIP = strings.Replace(ipRangeList[0], "0.0", "0.11", 1)
if d.webhookContainerIP == "" && d.aggregatedAPIServerContainerIP == "" {
return fmt.Errorf("failed to assign container ip")
}
return nil
}
Loading

0 comments on commit 6af72db

Please sign in to comment.