Skip to content

Commit

Permalink
Refresh client versions and installation.
Browse files Browse the repository at this point in the history
  • Loading branch information
Robert Glonek committed Dec 3, 2024
1 parent 478dcff commit 1af7a4c
Show file tree
Hide file tree
Showing 15 changed files with 262 additions and 285 deletions.
12 changes: 10 additions & 2 deletions CHANGELOG/7.7.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,13 @@
_Release Date: LONGMONTH DAYth, 2024_

**Release Notes:**
* TODO Replace problematic `tcconfig` with `easytc` for `net loss-delay` command.
* TODO Client version refresh
* EksCtl: Ensure that distro/version checks happen prior to deployment.
* Command `xdr connect` with connectors on docker did not populate connector IPs.
* Update client used versions:
* AMS: `ubuntu 22.04 -> 24.04`, `loki 2.5.0 -> 3.3.0`, `grafana-fallback 10.4.1 -> 11.3.1`.
* Rest Gateway: `ubuntu 22.04 -> 24.04`, `openjdk-19-jre -> openjdk-21-jre`.
* ElasticSearch: `ubuntu 22.04 -> 24.04`, `es-outbound-connector 1.0.0 -> configurable, default 2.2.5`, `openjdk-8-jre -> openjdk-21-jre`.
* Trino: `ubuntu 22.04 -> 24.04`, `connector default 4.5.0-431 -> 4.6.0-436`, `trino default 435 -> 446`.
* VSCode: `ubuntu 20.04 -> 24.04`, `go 1.22.5 -> 1.23.3`, `maven 3.8.8 -> 3.9.9`, `dotnet 6.0.300 -> 8.0.404`, `Microsoft.dotnet-interactive 1.0.355307 -> 1.0.556801`.
* Vector: Move to `v1` on `ubuntu 24.04`, use only a single namespace, update documentation and clone usage examples into the instance.
* Remove vector secondary namespace option from `conf generate` command.
17 changes: 17 additions & 0 deletions src/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ help:
\n\
INSTALL COMMANDS:\n\
\tinstall - Install a previously built aerolab on the current system (linux and mac only)\n\
\tnosudo-install - Install a previously built aerolab on the current system (linux and mac only), no sudo\n\
\n\
CLEAN COMMANDS:\n\
\tclean - Remove remainders of a build and reset source modified during build\n\
Expand Down Expand Up @@ -227,6 +228,22 @@ else
endif
endif

.PHONY: nosudo-install
nosudo-install:
ifeq ($(OS), Darwin)
ifeq ($(CPU), x86_64)
cp ../bin/aerolab-macos-amd64 /usr/local/bin/aerolab
else
cp ../bin/aerolab-macos-arm64 /usr/local/bin/aerolab
endif
else
ifeq ($(CPU), x86_64)
cp ../bin/aerolab-linux-amd64 /usr/local/bin/aerolab
else
cp ../bin/aerolab-linux-arm64 /usr/local/bin/aerolab
endif
endif

.PHONY: reset1
reset1:
printf "package main\n\nvar nLinuxBinaryX64 []byte\n\nvar nLinuxBinaryArm64 []byte\n" > embed_linux.go
Expand Down
18 changes: 9 additions & 9 deletions src/cmdClientDoAMS.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ func (c *clientCreateAMSCmd) Execute(args []string) error {
}
}
if c.DistroVersion == "latest" {
c.DistroVersion = "22.04"
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("22.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("AMS is only supported on ubuntu:22.04, selected %s:%s", c.DistroName, c.DistroVersion)
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("24.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("AMS is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}
custom := []CustomAMSDashboard{}
if c.Dashboards != "" {
Expand Down Expand Up @@ -145,10 +145,10 @@ func (c *clientAddAMSCmd) Execute(args []string) error {
return nil
}
if c.DistroVersion == "latest" {
c.DistroVersion = "22.04"
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("22.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("AMS is only supported on ubuntu:22.04, selected %s:%s", c.DistroName, c.DistroVersion)
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("24.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("AMS is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}
if c.ConnectClusters != "" {
nodes, err := c.checkClustersExist(c.ConnectClusters.String())
Expand Down Expand Up @@ -626,10 +626,10 @@ func installLokiScript(isArm bool) (script string, size int) {
}
script = `apt-get update && apt-get -y install wget curl unzip || exit 1
cd /root
wget https://github.com/grafana/loki/releases/download/v2.5.0/loki-linux-` + arch + `.zip || exit 1
wget https://github.com/grafana/loki/releases/download/v3.3.0/loki-linux-` + arch + `.zip || exit 1
unzip loki-linux-` + arch + `.zip || exit 1
mv loki-linux-` + arch + ` /usr/bin/loki || exit 1
wget https://github.com/grafana/loki/releases/download/v2.5.0/logcli-linux-` + arch + `.zip || exit 1
wget https://github.com/grafana/loki/releases/download/v3.3.0/logcli-linux-` + arch + `.zip || exit 1
unzip logcli-linux-` + arch + `.zip || exit 1
mv logcli-linux-` + arch + ` /usr/bin/logcli || exit 1
chmod 755 /usr/bin/logcli /usr/bin/loki || exit 1
Expand Down Expand Up @@ -685,7 +685,7 @@ function grafana_fallback() {
[[ $(uname -m) =~ arm ]] && platform=arm64
[[ $(uname -p) =~ arm ]] && platform=arm64
apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/oss/release/grafana_10.4.1_amd64.deb
wget https://dl.grafana.com/oss/release/grafana_11.3.1_amd64.deb
dpkg -i grafana_10.4.1_${platform}.deb
}
Expand Down
6 changes: 6 additions & 0 deletions src/cmdClientDoEksctl.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ func (c *clientCreateEksCtlCmd) Execute(args []string) error {
if earlyProcess(args) {
return nil
}
if c.DistroVersion == "latest" {
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || c.DistroVersion != TypeDistroVersion("24.04") {
return fmt.Errorf("eksctl is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}
if strings.HasPrefix(c.EksAwsKeyId, "ENV::") {
c.EksAwsKeyId = os.Getenv(strings.Split(c.EksAwsKeyId, "::")[1])
}
Expand Down
47 changes: 25 additions & 22 deletions src/cmdClientDoElasticSearch.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,19 @@ import (

type clientCreateElasticSearchCmd struct {
clientCreateBaseCmd
RamLimit int `long:"mem-limit" description:"By Default ES will use most of any machine RAM; set this to a number of GB to limit each ES instance" default:"0"`
JustDoIt bool `long:"confirm" description:"set this parameter to confirm any warning questions without being asked to press ENTER to continue" webdisable:"true" webset:"true"`
RamLimit int `long:"mem-limit" description:"By Default ES will use most of any machine RAM; set this to a number of GB to limit each ES instance" default:"0"`
JustDoIt bool `long:"confirm" description:"set this parameter to confirm any warning questions without being asked to press ENTER to continue" webdisable:"true" webset:"true"`
ConnectorVersion string `long:"connector-version" default:"2.2.5" description:"set a specific connector version"`
}

type clientAddElasticSearchCmd struct {
ClientName TypeClientName `short:"n" long:"group-name" description:"Client group name" default:"client"`
Machines TypeMachines `short:"l" long:"machines" description:"Comma separated list of machines, empty=all" default:""`
StartScript flags.Filename `short:"X" long:"start-script" description:"optionally specify a script to be installed which will run when the client machine starts"`
RamLimit int `long:"mem-limit" description:"By Default ES will use most of any machine RAM; set this to a number of GB to limit each ES instance" default:"0"`
existingNodes []int
Help helpCmd `command:"help" subcommands-optional:"true" description:"Print help"`
ClientName TypeClientName `short:"n" long:"group-name" description:"Client group name" default:"client"`
Machines TypeMachines `short:"l" long:"machines" description:"Comma separated list of machines, empty=all" default:""`
StartScript flags.Filename `short:"X" long:"start-script" description:"optionally specify a script to be installed which will run when the client machine starts"`
RamLimit int `long:"mem-limit" description:"By Default ES will use most of any machine RAM; set this to a number of GB to limit each ES instance" default:"0"`
ConnectorVersion string `long:"connector-version" default:"2.2.5" description:"set a specific connector version"`
existingNodes []int
Help helpCmd `command:"help" subcommands-optional:"true" description:"Print help"`
}

func (c *clientCreateElasticSearchCmd) Execute(args []string) error {
Expand All @@ -46,10 +48,10 @@ func (c *clientCreateElasticSearchCmd) Execute(args []string) error {
return fmt.Errorf("more than one elasticsearch node cannot be started without specifying RAM limits in docker - elasticsearch default will cause OOM-kills")
}
if c.DistroVersion == "latest" {
c.DistroVersion = "22.04"
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("22.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("ES is only supported on ubuntu:22.04, selected %s:%s", c.DistroName, c.DistroVersion)
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("24.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("ES is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}
b.WorkOnClients()
clusters, err := b.ClusterList()
Expand Down Expand Up @@ -77,6 +79,7 @@ func (c *clientCreateElasticSearchCmd) Execute(args []string) error {
a.opts.Client.Add.ElasticSearch.StartScript = c.StartScript
a.opts.Client.Add.ElasticSearch.Machines = TypeMachines(intSliceToString(machines, ","))
a.opts.Client.Add.ElasticSearch.RamLimit = c.RamLimit
a.opts.Client.Add.ElasticSearch.ConnectorVersion = c.ConnectorVersion
return a.opts.Client.Add.ElasticSearch.addElasticSearch(args)
}

Expand All @@ -93,7 +96,7 @@ func (c *clientAddElasticSearchCmd) addElasticSearch(args []string) error {
isDocker := false
if a.opts.Config.Backend.Type == "docker" {
isDocker = true
out, err := exec.Command("docker", "run", "--rm", "-i", "--privileged", "ubuntu:22.04", "sysctl", "-w", "vm.max_map_count=262144").CombinedOutput()
out, err := exec.Command("docker", "run", "--rm", "-i", "--privileged", "ubuntu:24.04", "sysctl", "-w", "vm.max_map_count=262144").CombinedOutput()
if err != nil {
fmt.Println("Workaround `sysctl -w vm.max_map_count=262144` for docker failed, elasticsearch might fail to start...")
fmt.Println(err)
Expand All @@ -103,7 +106,7 @@ func (c *clientAddElasticSearchCmd) addElasticSearch(args []string) error {
b.WorkOnClients()
masterNode := 1
if len(c.existingNodes) == 0 {
script := c.installScriptAllNodes(c.RamLimit, isDocker) + c.installScriptMasterNode()
script := c.installScriptAllNodes(c.RamLimit, isDocker, c.ConnectorVersion) + c.installScriptMasterNode()
err := b.CopyFilesToCluster(c.ClientName.String(), []fileList{{filePath: "/root/install.sh", fileContents: script, fileSize: len(script)}}, []int{1})
if err != nil {
return err
Expand Down Expand Up @@ -149,7 +152,7 @@ func (c *clientAddElasticSearchCmd) addElasticSearch(args []string) error {
return err
}
cert := base64.StdEncoding.EncodeToString(out[0])
script := c.installScriptAllNodes(c.RamLimit, isDocker) + c.installScriptSlaveNodesOnSlaves(token, cert)
script := c.installScriptAllNodes(c.RamLimit, isDocker, c.ConnectorVersion) + c.installScriptSlaveNodesOnSlaves(token, cert)
err = b.CopyFilesToCluster(c.ClientName.String(), []fileList{{filePath: "/root/install.sh", fileContents: script, fileSize: len(script)}}, []int{node})
if err != nil {
return err
Expand All @@ -175,10 +178,10 @@ func (c *clientAddElasticSearchCmd) addElasticSearch(args []string) error {
ES username/password is: elastic/elastic
Usage examples to query in browser:
* https://ELASTICIP:9200/test/_search
* https://ELASTICIP:9200/test/_search?size=50
* https://ELASTICIP:9200/test/_search?q=mybin:binvalue
* https://ELASTICIP:9200/test/_search?q=metadata.set:myset
* https://ELASTICIP:9200/aerospike/_search
* https://ELASTICIP:9200/aerospike/_search?size=50
* https://ELASTICIP:9200/aerospike/_search?q=mybin:binvalue
* https://ELASTICIP:9200/aerospike/_search?q=metadata.set:myset
For best results, use FireFox, as it has a built-in JSON explorer features. It also accepts invalid slef-signed certificates that ES provides, while Chrome doesn't allow to continue.
Expand All @@ -191,10 +194,10 @@ For best results, use FireFox, as it has a built-in JSON explorer features. It a
return nil
}

func (c *clientAddElasticSearchCmd) installScriptAllNodes(ramlimit int, isDocker bool) string {
func (c *clientAddElasticSearchCmd) installScriptAllNodes(ramlimit int, isDocker bool, esVersion string) string {
script := `#!/bin/bash
apt-get update
apt-get -y install apt-transport-https ca-certificates curl gnupg wget openjdk-8-jre
apt-get -y install apt-transport-https ca-certificates curl gnupg wget openjdk-21-jre
echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-8.x.list
curl -L https://artifacts.elastic.co/GPG-KEY-elasticsearch |apt-key add -
apt-get update && apt-get -y install elasticsearch
Expand All @@ -210,8 +213,8 @@ EOF
`, ramlimit, ramlimit)
}
script = script + `cd /root
wget https://download.aerospike.com/artifacts/enterprise/aerospike-elasticsearch/1.0.0/aerospike-elasticsearch-outbound-1.0.0.all.deb
dpkg -i aerospike-elasticsearch-outbound-1.0.0.all.deb
wget https://download.aerospike.com/artifacts/enterprise/aerospike-elasticsearch/` + esVersion + `/aerospike-elasticsearch-outbound-` + esVersion + `.all.deb
dpkg -i aerospike-elasticsearch-outbound-` + esVersion + `.all.deb
sed -i -E 's/(.*)port: 9200/\1port: 9200\n\1scheme: https/g' /etc/aerospike-elasticsearch-outbound/aerospike-elasticsearch-outbound.yml
printf " auth-config:\n type: basic\n username: elastic\n password-file: /etc/aerospike-elasticsearch-outbound/password.conf\n" >> /etc/aerospike-elasticsearch-outbound/aerospike-elasticsearch-outbound.yml
printf "elastic" > /etc/aerospike-elasticsearch-outbound/password.conf
Expand Down
8 changes: 4 additions & 4 deletions src/cmdClientDoRestGateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ func (c *clientCreateRestGatewayCmd) Execute(args []string) error {
}
}
if c.DistroVersion == "latest" {
c.DistroVersion = "22.04"
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("22.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("RG is only supported on ubuntu:22.04, selected %s:%s", c.DistroName, c.DistroVersion)
if c.DistroName != TypeDistro("ubuntu") || (c.DistroVersion != TypeDistroVersion("24.04") && c.DistroVersion != TypeDistroVersion("latest")) {
return fmt.Errorf("RG is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}

url, err := c.Version.GetDownloadURL()
Expand Down Expand Up @@ -228,7 +228,7 @@ func (c *clientAddRestGatewayCmd) addRestGateway(args []string) error {
func (c *clientAddRestGatewayCmd) installScript(seedPort string) string {
return fmt.Sprintf(`set -e
apt-get update
apt-get -y install openjdk-19-jre openjdk-19-jre-headless curl wget
apt-get -y install openjdk-21-jre openjdk-21-jre-headless curl wget
cd /opt
wget %s
tar -zxvf %s.tgz
Expand Down
23 changes: 11 additions & 12 deletions src/cmdClientDoTrino.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (
type clientCreateTrinoCmd struct {
clientCreateBaseCmd
ConnectCluster string `short:"s" long:"seed" description:"seed IP:PORT (can be changed later using client configure command)" default:"127.0.0.1:3000"`
TrinoVersion string `long:"trino-version" description:"trino version" default:"435"`
ConnectorVersion string `long:"connector-version" description:"aerospike connector version" default:"4.5.0-431"`
TrinoVersion string `long:"trino-version" description:"trino version" default:"446"`
ConnectorVersion string `long:"connector-version" description:"aerospike connector version" default:"4.6.0-436"`
chDirCmd
}

Expand All @@ -22,8 +22,8 @@ type clientAddTrinoCmd struct {
StartScript flags.Filename `short:"X" long:"start-script" description:"optionally specify a script to be installed which will run when the client machine starts"`
osSelectorCmd
ConnectCluster string `short:"s" long:"seed" description:"seed IP:PORT (can be changed later using client configure command)" default:"127.0.0.1:3000"`
TrinoVersion string `long:"trino-version" description:"trino version" default:"435"`
ConnectorVersion string `long:"connector-version" description:"aerospike connector version" default:"4.5.0-431"`
TrinoVersion string `long:"trino-version" description:"trino version" default:"446"`
ConnectorVersion string `long:"connector-version" description:"aerospike connector version" default:"4.6.0-436"`
Help helpCmd `command:"help" subcommands-optional:"true" description:"Print help"`
}

Expand All @@ -32,10 +32,10 @@ func (c *clientCreateTrinoCmd) Execute(args []string) error {
return nil
}
if c.DistroVersion == "latest" {
c.DistroVersion = "22.04"
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || c.DistroVersion != TypeDistroVersion("22.04") {
return fmt.Errorf("trino is only supported on ubuntu:22.04, selected %s:%s", c.DistroName, c.DistroVersion)
if c.DistroName != TypeDistro("ubuntu") || c.DistroVersion != TypeDistroVersion("24.04") {
return fmt.Errorf("trino is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}
machines, err := c.createBase(args, "trino")
if err != nil {
Expand All @@ -60,10 +60,10 @@ func (c *clientAddTrinoCmd) Execute(args []string) error {
return nil
}
if c.DistroVersion == "latest" {
c.DistroVersion = "22.04"
c.DistroVersion = "24.04"
}
if c.DistroName != TypeDistro("ubuntu") || c.DistroVersion != TypeDistroVersion("22.04") {
return fmt.Errorf("trino is only supported on ubuntu:22.04, selected %s:%s", c.DistroName, c.DistroVersion)
if c.DistroName != TypeDistro("ubuntu") || c.DistroVersion != TypeDistroVersion("24.04") {
return fmt.Errorf("trino is only supported on ubuntu:24.04, selected %s:%s", c.DistroName, c.DistroVersion)
}
return c.addTrino(args)
}
Expand Down Expand Up @@ -198,7 +198,7 @@ cd /home/trino
# Update and install necessary packages
apt-get update
apt-get install unzip openjdk-17-jdk python-is-python3 uuid-runtime -y
apt-get install unzip openjdk-21-jdk python-is-python3 uuid-runtime -y
# Update limits.conf with suggested limits
sed -i.bak 's/# End of file/trino soft nofile 131072\ntrino hard nofile 131072\n\n# End of file/' /etc/security/limits.conf
Expand Down Expand Up @@ -274,7 +274,6 @@ aerospike.scanpolicy.recordsPerSecond=0
aerospike.clientpolicy.timeout=60000
aerospike.clientpolicy.tendInterval=1000
aerospike.clientpolicy.maxSocketIdle=30
aerospike.clientpolicy.sharedThreadPool=true
aerospike.clientpolicy.connPoolsPerNode=1
aerospike.scanpolicy.maxConcurrentNodes=0
aerospike.policy.socketTimeout=600000
Expand Down
Loading

0 comments on commit 1af7a4c

Please sign in to comment.