From e39a18fbc472148e012fe9b1df47325d411f863e Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 12 Oct 2023 15:45:26 -0700 Subject: [PATCH 001/142] Build basic setup for k8s kind e2e tests. --- k8se2e/docker-compose.yml | 32 +++++ k8se2e/elasticsearchDeployment.yaml | 45 ++++++ k8se2e/jest.config.js | 7 + k8se2e/kindConfig.yaml | 10 ++ k8se2e/masterConfig/teraslice.yaml | 31 ++++ k8se2e/masterDeployment.yaml | 53 +++++++ k8se2e/ns.yaml | 4 + k8se2e/package.json | 55 +++++++ k8se2e/priorityClass.yaml | 7 + k8se2e/role.yaml | 9 ++ k8se2e/roleBinding.yaml | 13 ++ k8se2e/workerConfig/teraslice.yaml | 29 ++++ package.json | 1 + packages/scripts/src/helpers/packages.ts | 12 ++ packages/scripts/src/helpers/scripts.ts | 82 +++++++++++ .../scripts/src/helpers/test-runner/index.ts | 134 +++++++++++++++++- 16 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 k8se2e/docker-compose.yml create mode 100644 k8se2e/elasticsearchDeployment.yaml create mode 100644 k8se2e/jest.config.js create mode 100644 k8se2e/kindConfig.yaml create mode 100644 k8se2e/masterConfig/teraslice.yaml create mode 100644 k8se2e/masterDeployment.yaml create mode 100644 k8se2e/ns.yaml create mode 100644 k8se2e/package.json create mode 100644 k8se2e/priorityClass.yaml create mode 100644 k8se2e/role.yaml create mode 100644 k8se2e/roleBinding.yaml create mode 100644 k8se2e/workerConfig/teraslice.yaml diff --git a/k8se2e/docker-compose.yml b/k8se2e/docker-compose.yml new file mode 100644 index 00000000000..e8cd69ed047 --- /dev/null +++ b/k8se2e/docker-compose.yml @@ -0,0 +1,32 @@ +version: '2.4' +services: + teraslice-master: + image: teraslice-workspace:e2e + ports: + - '45678:45678' + scale: 1 + restart: 'no' + stop_grace_period: 30s + environment: + - TERAFOUNDATION_CONFIG=/app/config/teraslice-master.json + - NODE_OPTIONS=--max-old-space-size=256 + volumes: + - ./logs:/app/logs:delegated + - ./autoload:/app/autoload:ro + - ./.config:/app/config:ro + - ./.assets:/app/assets:delegated + teraslice-worker: + image: teraslice-workspace:e2e + scale: 2 + restart: 'no' + stop_grace_period: 30s + links: + - teraslice-master + environment: + - TERAFOUNDATION_CONFIG=/app/config/teraslice-worker.json + - NODE_OPTIONS=--max-old-space-size=256 + volumes: + - ./logs:/app/logs:delegated + - ./autoload:/app/autoload:ro + - ./.config:/app/config:ro + - ./.assets:/app/assets:delegated diff --git a/k8se2e/elasticsearchDeployment.yaml b/k8se2e/elasticsearchDeployment.yaml new file mode 100644 index 00000000000..5ea8287b1aa --- /dev/null +++ b/k8se2e/elasticsearchDeployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + nodeType: master + template: + metadata: + labels: + app: elasticsearch + nodeType: master + spec: + containers: + - name: elasticsearch + image: elasticsearch:7.9.3 + ports: + - containerPort: 9200 + env: + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: single-node +--- +kind: Service +apiVersion: v1 +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + nodeType: master + ports: + - port: 9200 + targetPort: 9200 + nodePort: 30200 # the external port teraslice can be accessed on + type: NodePort diff --git a/k8se2e/jest.config.js b/k8se2e/jest.config.js new file mode 100644 index 00000000000..0a4be3a992a --- /dev/null +++ b/k8se2e/jest.config.js @@ -0,0 +1,7 @@ +'use strict'; + +const config = require('../jest.config.base')(__dirname); + +config.collectCoverage = false; +delete config.transform; +module.exports = config; diff --git a/k8se2e/kindConfig.yaml b/k8se2e/kindConfig.yaml new file mode 100644 index 00000000000..e9d87e52e73 --- /dev/null +++ b/k8se2e/kindConfig.yaml @@ -0,0 +1,10 @@ +kind: Cluster +name: k8se2e +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30200 + hostPort: 9200 + - containerPort: 30678 + hostPort: 5678 diff --git a/k8se2e/masterConfig/teraslice.yaml b/k8se2e/masterConfig/teraslice.yaml new file mode 100644 index 00000000000..8f1aaddea3e --- /dev/null +++ b/k8se2e/masterConfig/teraslice.yaml @@ -0,0 +1,31 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: true + master_hostname: "127.0.0.1" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image_pull_secrets: + - "docker-tera1-secret" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 diff --git a/k8se2e/masterDeployment.yaml b/k8se2e/masterDeployment.yaml new file mode 100644 index 00000000000..14433977e66 --- /dev/null +++ b/k8se2e/masterDeployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: teraslice-master + labels: + app: teraslice + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: teraslice + nodeType: master + template: + metadata: + labels: + app: teraslice + nodeType: master + clusterName: ts-dev1 + spec: + containers: + - name: teraslice-master + image: teraslice-workspace:k8se2e + ports: + - containerPort: 5678 + volumeMounts: + - mountPath: /app/config # defines the directory + name: config + volumes: + - name: config + configMap: + name: teraslice-master + items: + - key: teraslice.yaml + path: teraslice.yaml # the filename that the configMap gets written to, inside the matching mountPath + imagePullSecrets: + - name: docker-tera1-secret +--- +kind: Service +apiVersion: v1 +metadata: + name: teraslice-master + labels: + app: teraslice +spec: + selector: + app: teraslice + nodeType: master + ports: + - port: 5678 + targetPort: 5678 + nodePort: 30678 # the external port teraslice can be accessed on + type: NodePort diff --git a/k8se2e/ns.yaml b/k8se2e/ns.yaml new file mode 100644 index 00000000000..1af803c0c4f --- /dev/null +++ b/k8se2e/ns.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: ts-dev1 \ No newline at end of file diff --git a/k8se2e/package.json b/k8se2e/package.json new file mode 100644 index 00000000000..0b261e4873f --- /dev/null +++ b/k8se2e/package.json @@ -0,0 +1,55 @@ +{ + "name": "k8se2e", + "displayName": "K8S E2E Tests", + "version": "0.0.1", + "private": true, + "description": "Teraslice in Kubernetes integration test suite", + "keywords": [ // ??? + "docker-compose", + "elasticsearch", + "teraslice" + ], + "homepage": "https://github.com/terascope/teraslice/tree/master/k8se2e/#readme", + "bugs": { + "url": "https://github.com/terascope/teraslice/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/terascope/teraslice.git" + }, + "license": "MIT", + "author": "Terascope, LLC ", + "scripts": { // check these + "clean": "docker-compose down --volumes --remove-orphans --timeout=5", + "logs": "./scripts/logs.sh", + "logs-follow": "./scripts/logs.sh -f", + "setup": "yarn --silent", + "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --debug --", + "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --watch --" + }, + "resolutions": { // ??? + "ms": "^2.1.3" + }, + "devDependencies": { // ??? + "bunyan": "^1.8.15", + "elasticsearch-store": "^0.72.0", + "fs-extra": "^10.1.0", + "ms": "^2.1.3", + "nanoid": "^3.3.4", + "semver": "^7.3.8", + "signale": "^1.4.0", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.17.0", + "yarn": ">=1.16.0" + }, + "terascope": { + "testSuite": "k8se2e" + } +} diff --git a/k8se2e/priorityClass.yaml b/k8se2e/priorityClass.yaml new file mode 100644 index 00000000000..0747825c87d --- /dev/null +++ b/k8se2e/priorityClass.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000000 +globalDefault: false +description: "This priority class is for Teraslice pods." diff --git a/k8se2e/role.yaml b/k8se2e/role.yaml new file mode 100644 index 00000000000..725cec875a5 --- /dev/null +++ b/k8se2e/role.yaml @@ -0,0 +1,9 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] \ No newline at end of file diff --git a/k8se2e/roleBinding.yaml b/k8se2e/roleBinding.yaml new file mode 100644 index 00000000000..b46683aeb41 --- /dev/null +++ b/k8se2e/roleBinding.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +subjects: + - kind: ServiceAccount + name: default + namespace: ts-dev1 +roleRef: + kind: Role + name: teraslice-all-ts-dev1 + apiGroup: "rbac.authorization.k8s.io" \ No newline at end of file diff --git a/k8se2e/workerConfig/teraslice.yaml b/k8se2e/workerConfig/teraslice.yaml new file mode 100644 index 00000000000..6615cc43757 --- /dev/null +++ b/k8se2e/workerConfig/teraslice.yaml @@ -0,0 +1,29 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: false + master_hostname: "teraslice-master" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 diff --git a/package.json b/package.json index 9e1ac806a9e..5044388055b 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "tests": { "suites": { "e2e": [], + "k8se2e": [], "elasticsearch": [], "search": [], "restrained": [], diff --git a/packages/scripts/src/helpers/packages.ts b/packages/scripts/src/helpers/packages.ts index 0e7197730f4..75ae9b384de 100644 --- a/packages/scripts/src/helpers/packages.ts +++ b/packages/scripts/src/helpers/packages.ts @@ -15,6 +15,7 @@ import * as i from './interfaces'; let _packages: i.PackageInfo[] = []; let _e2eDir: string|undefined; +let _k8se2eDir: string|undefined; export function getE2EDir(): string|undefined { if (_e2eDir) return _e2eDir; @@ -27,6 +28,17 @@ export function getE2EDir(): string|undefined { return undefined; } +export function getK8SE2EDir(): string|undefined { + if (_k8se2eDir) return _k8se2eDir; + + if (fs.existsSync(path.join(misc.getRootDir(), 'k8se2e'))) { + _k8se2eDir = path.join(misc.getRootDir(), 'k8se2e'); + return _k8se2eDir; + } + + return undefined; +} + function _loadPackage(packagePath: string): i.PackageInfo|undefined { const pkgJsonPath = path.join(packagePath, 'package.json'); if (fs.existsSync(pkgJsonPath)) { diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 3118ed56c26..4c97be2f891 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -532,3 +532,85 @@ export async function yarnPublish( } }); } + +export async function createKindCluster( + k8se2eDir: string, + kindConfigFileName: string +): Promise { + const configPath = path.join(k8se2eDir, kindConfigFileName); + const subprocess = await execa.command(`kind create cluster --config ${configPath}`); + signale.log(subprocess.stderr); + // const test = await execa.command('echo hello'); + // console.log('test: ', test); +} + +export async function destroyKindCluster(): Promise { + // TODO: pass cluster name in as variable + const subprocess = await execa.command('kind delete cluster --name k8se2e'); + signale.log(subprocess.stderr); +} + +export async function isKindInstalled(): Promise { + try { + const subprocess = await execa.command('command -v kind'); + // console.log('kind subprocess: ', subprocess); + return !!subprocess.stdout; + } catch (err) { + // console.log(err); + return false; + } +} + +export async function isKubectlInstalled(): Promise { + try { + const subprocess = await execa.command('command -v kubectl'); + // console.log('kubectl subprocess: ', subprocess); + return !!subprocess.stdout; + } catch (err) { + return false; + } +} + +export async function loadTerasliceImage(terasliceImage: string): Promise { + const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); + console.log('load teraslice image subprocess: ', subprocess); +} + +export async function createNamespace() { + const subprocess = await execa.command('kubectl create namespace ts-dev1'); + console.log('namespace subprocess: ', subprocess); +} + +export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: string) { + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); + console.log('esDeploy subprocess: ', subprocess); +} + +export async function k8sSetup( + k8se2eDir: string, + roleYaml: string, + roleBindingYaml: string, + priorityClassYaml: string +): Promise { + const subprocess1 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleYaml)}`); + const subprocess2 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleBindingYaml)}`); + const subprocess3 = await execa.command(`kubectl apply -f ${path.join(k8se2eDir, priorityClassYaml)}`); + console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); +} + +export async function deployk8sTeraslice( + k8se2eDir: string, + masterDeploymentYaml: string +) { + /// Creates configmap for terasclice-master + let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(k8se2eDir, 'masterConfig', 'teraslice.yaml')}`); + console.log('masterConfig subprocess: ', subprocess); + + /// Creates configmap for teraslice-worker + subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(k8se2eDir, 'workerConfig', 'teraslice.yaml')}`); + console.log('workerConfig subprocess: ', subprocess); + + /// Creates deployment for teraslice + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); + console.log('masterDeploy subprocess: ', subprocess); +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 45d26a3628c..8307176728a 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -9,20 +9,35 @@ import { import { ensureServices, pullServices } from './services'; import { PackageInfo } from '../interfaces'; import { TestOptions } from './interfaces'; -import { runJest, dockerTag } from '../scripts'; +import { + createKindCluster, + destroyKindCluster, + loadTerasliceImage, + createNamespace, + deployElasticSearch, + k8sSetup, + deployk8sTeraslice, + runJest, + dockerTag, + isKindInstalled, + isKubectlInstalled +} from '../scripts'; import { getArgs, filterBySuite, globalTeardown, reportCoverage, logE2E, getEnv, groupBySuite } from './utils'; import signale from '../signale'; -import { getE2EDir, readPackageInfo, listPackages } from '../packages'; +import { + getE2EDir, readPackageInfo, listPackages, getK8SE2EDir +} from '../packages'; import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; import { TestTracker } from './tracker'; import { MAX_PROJECTS_PER_BATCH, - SKIP_DOCKER_BUILD_IN_E2E + SKIP_DOCKER_BUILD_IN_E2E, + SKIP_DOCKER_BUILD_K8S_E2E } from '../config'; const logger = debugLogger('ts-scripts:cmd:test'); @@ -65,6 +80,11 @@ async function _runTests( return; } + if (options.suite?.includes('k8se2e')) { + await runk8sE2ETest(options, tracker); + return; + } + const filtered = filterBySuite(pkgInfos, options); if (!filtered.length) { signale.warn('No tests found.'); @@ -300,3 +320,111 @@ function printAndGetEnv(suite: string, options: TestOptions) { } return env; } + +async function runk8sE2ETest( + options: TestOptions, tracker: TestTracker +): Promise { + console.log('options: ', options); + tracker.expected++; + + const k8se2eDir = getK8SE2EDir(); + if (!k8se2eDir) { + throw new Error('Missing k8se2e test directory'); + } + + const kindInstalled = await isKindInstalled(); + if (!kindInstalled) { + signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); + process.exit(1); + } + + const kubectlInstalled = await isKubectlInstalled(); + if (!kubectlInstalled) { + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + process.exit(1); + } + // TODO: pass kind config file in as a variable + await createKindCluster(k8se2eDir, 'kindConfig.yaml'); + + const suite = 'k8se2e'; + let startedTest = false; + + const rootInfo = getRootInfo(); + const k8se2eImage = `${rootInfo.name}:k8se2e`; + + // if (isCI) { + // // pull the services first in CI + // await pullServices(suite, options); + // } + + try { + if (SKIP_DOCKER_BUILD_K8S_E2E) { + const devImage = getDevDockerImage(); + await dockerTag(devImage, k8se2eImage); + await loadTerasliceImage(k8se2eImage); + } else { + const devImage = await buildDevDockerImage(); + await dockerTag(devImage, k8se2eImage); + await loadTerasliceImage(k8se2eImage); + } + } catch (err) { + tracker.addError(err); + } + + // TODO: add tracker + await createNamespace(); + await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); + await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); + await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); + + if (!tracker.hasErrors()) { + const timeLabel = `test suite "${suite}"`; + signale.time(timeLabel); + startedTest = true; + + const env = printAndGetEnv(suite, options); + + tracker.started++; + try { + await runJest( + k8se2eDir, + getArgs(options), + env, + options.jestArgs, + options.debug + ); + tracker.ended++; + } catch (err) { + tracker.ended++; + tracker.addError(err.message); + } + + signale.timeEnd(timeLabel); + } + + if (!startedTest) return; + + // if (!options.keepOpen) { + // try { + // await logE2E(k8se2eDir, tracker.hasErrors()); + // } catch (err) { + // signale.error( + // new TSError(err, { + // reason: `Writing the "${suite}" logs failed`, + // }) + // ); + // } + // } + + // if (tracker.hasErrors()) { + // tracker.addCleanup('e2e:teardown', async () => { + // options.keepOpen = false; + // await globalTeardown(options, [{ + // name: suite, + // dir: k8se2eDir, + // suite, + // }]); + // }); + // } + // await destroyKindCluster(); +} From 775498476dea26924b7af2aed9a9951740bea387 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 13 Oct 2023 07:42:07 -0700 Subject: [PATCH 002/142] update buildDevDockerImage to take node version. --- k8se2e/package.json | 8 +-- k8se2e/testJob.json | 12 ++++ k8se2e/testReaderFile.json | 8 +++ packages/scripts/src/helpers/config.ts | 2 + packages/scripts/src/helpers/scripts.ts | 16 +++++ .../scripts/src/helpers/test-runner/index.ts | 60 +++++++++++-------- 6 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 k8se2e/testJob.json create mode 100644 k8se2e/testReaderFile.json diff --git a/k8se2e/package.json b/k8se2e/package.json index 0b261e4873f..64302c49c46 100644 --- a/k8se2e/package.json +++ b/k8se2e/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "private": true, "description": "Teraslice in Kubernetes integration test suite", - "keywords": [ // ??? + "keywords": [ "docker-compose", "elasticsearch", "teraslice" @@ -19,7 +19,7 @@ }, "license": "MIT", "author": "Terascope, LLC ", - "scripts": { // check these + "scripts": { "clean": "docker-compose down --volumes --remove-orphans --timeout=5", "logs": "./scripts/logs.sh", "logs-follow": "./scripts/logs.sh -f", @@ -32,10 +32,10 @@ "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --watch --" }, - "resolutions": { // ??? + "resolutions": { "ms": "^2.1.3" }, - "devDependencies": { // ??? + "devDependencies": { "bunyan": "^1.8.15", "elasticsearch-store": "^0.72.0", "fs-extra": "^10.1.0", diff --git a/k8se2e/testJob.json b/k8se2e/testJob.json new file mode 100644 index 00000000000..f688bdcb954 --- /dev/null +++ b/k8se2e/testJob.json @@ -0,0 +1,12 @@ +{ + "name": "Test Job", + "lifecycle": "once", + "workers": 1, + "assets": [], + "operations": [ + { + "_op": "test-reader", + "fetcher_data_file_path": "./testReaderFile.json" + } + ] +} diff --git a/k8se2e/testReaderFile.json b/k8se2e/testReaderFile.json new file mode 100644 index 00000000000..b0126017515 --- /dev/null +++ b/k8se2e/testReaderFile.json @@ -0,0 +1,8 @@ +[ + { + "foo": "bar" + }, + { + "foo": "baz" + } +] diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index e6c880989c2..25879be6542 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -98,6 +98,8 @@ export const DEV_DOCKER_IMAGE = process.env.DEV_DOCKER_IMAGE || undefined; */ export const SKIP_DOCKER_BUILD_IN_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_IN_E2E ?? false); +export const SKIP_DOCKER_BUILD_K8S_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_K8S_E2E ?? false); + export const SKIP_E2E_OUTPUT_LOGS = toBoolean(process.env.SKIP_E2E_OUTPUT_LOGS ?? !isCI); /** diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 4c97be2f891..1e0e3670129 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -614,3 +614,19 @@ export async function deployk8sTeraslice( subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); console.log('masterDeploy subprocess: ', subprocess); } + +export async function setAlias() { + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); + console.log('setAlias subprocess: ', subprocess1, subprocess2); +} + +export async function registerTestJob(k8se2eDir: string) { + const subprocess = await execa.command(`earl tjm register localhost ${path.join(k8se2eDir, 'testJob.json')}`); + console.log('registerTestJob subprocess: ', subprocess); +} + +export async function startTestJob(k8se2eDir: string) { + const subprocess = await execa.command(`earl tjm start ${path.join(k8se2eDir, 'testJob.json')}`); + console.log('registerTestJob subprocess: ', subprocess); +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 8307176728a..39da75774b1 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -20,7 +20,9 @@ import { runJest, dockerTag, isKindInstalled, - isKubectlInstalled + isKubectlInstalled, + registerTestJob, + startTestJob } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -359,11 +361,16 @@ async function runk8sE2ETest( try { if (SKIP_DOCKER_BUILD_K8S_E2E) { - const devImage = getDevDockerImage(); + const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; await dockerTag(devImage, k8se2eImage); await loadTerasliceImage(k8se2eImage); } else { - const devImage = await buildDevDockerImage(); + const publishOptions: PublishOptions = { + dryRun: true, + nodeVersion: options.nodeVersion, + type: PublishType.Dev + }; + const devImage = await buildDevDockerImage(publishOptions); await dockerTag(devImage, k8se2eImage); await loadTerasliceImage(k8se2eImage); } @@ -377,32 +384,35 @@ async function runk8sE2ETest( await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); - if (!tracker.hasErrors()) { - const timeLabel = `test suite "${suite}"`; - signale.time(timeLabel); - startedTest = true; + // await registerTestJob(k8se2eDir); + // await startTestJob(k8se2eDir); - const env = printAndGetEnv(suite, options); + // if (!tracker.hasErrors()) { + // const timeLabel = `test suite "${suite}"`; + // signale.time(timeLabel); + // startedTest = true; - tracker.started++; - try { - await runJest( - k8se2eDir, - getArgs(options), - env, - options.jestArgs, - options.debug - ); - tracker.ended++; - } catch (err) { - tracker.ended++; - tracker.addError(err.message); - } + // const env = printAndGetEnv(suite, options); - signale.timeEnd(timeLabel); - } + // tracker.started++; + // try { + // await runJest( + // k8se2eDir, + // getArgs(options), + // env, + // options.jestArgs, + // options.debug + // ); + // tracker.ended++; + // } catch (err) { + // tracker.ended++; + // tracker.addError(err.message); + // } - if (!startedTest) return; + // signale.timeEnd(timeLabel); + // } + + // if (!startedTest) return; // if (!options.keepOpen) { // try { From 1804317fb82ca4e0baf2394af2d1aa46f831e0e4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 13 Oct 2023 13:13:24 -0700 Subject: [PATCH 003/142] test that elasticsearch and teraslice are running. --- packages/scripts/src/helpers/scripts.ts | 93 ++++++++++++++++++- .../scripts/src/helpers/test-runner/index.ts | 8 +- 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 1e0e3670129..ddf1871072a 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -14,6 +14,7 @@ import { TSCommands, PackageInfo } from './interfaces'; import { getRootDir } from './misc'; import signale from './signale'; import * as config from './config'; +import { setTimeout } from 'timers'; const logger = debugLogger('ts-scripts:cmd'); @@ -584,6 +585,23 @@ export async function createNamespace() { export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: string) { const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); console.log('esDeploy subprocess: ', subprocess); + + let elasticsearchRunning = false; + // TODO: add curl check to ES before continuing + while (!elasticsearchRunning) { + const ESResponse = execa.command('curl localhost:9200').stdout; + if (ESResponse) { + ESResponse.on('data', (data) => { + const jsonData = JSON.parse(data); + console.log(`response: ${JSON.stringify(jsonData)}`); + if (jsonData.tagline === 'You Know, for Search') { + console.log('jsonData.tagline === You Know, for Search') + elasticsearchRunning = true; + } + }); + } + await PromiseTimeout(3000); + } } export async function k8sSetup( @@ -613,6 +631,62 @@ export async function deployk8sTeraslice( /// Creates deployment for teraslice subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); console.log('masterDeploy subprocess: ', subprocess); + + subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); + const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); + console.log('podName: ', podName); + + // await waitForTerasliceRunning(120000); + + let terasliceRunning = false; + + while (!terasliceRunning) { + const TSMasterResponse = execa.command('curl localhost:5678').stdout; + if (TSMasterResponse) { + TSMasterResponse.on('data', (data) => { + const jsonData = JSON.parse(data); + console.log(`response: ${JSON.stringify(jsonData)}`); + if (jsonData.clustering_type === 'kubernetes') { + console.log('jsonData.clusteringType === kubernetes') + terasliceRunning = true; + } + }); + } + await PromiseTimeout(3000); + } + // function waitForTerasliceRunning(timeoutMs = 120000) : Promise { + // const endAt = Date.now() + timeoutMs; + + // const _waitForTerasliceRunning = async () : Promise => { + // if (Date.now() > endAt) { + // throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); + // } + + // let terasliceRunning = false; + // // let nodes = -1; + // try { + // const TSMasterResponse = execa.command('curl localhost:5678').stdout; + // if (TSMasterResponse) { + // TSMasterResponse.on('data', (data) => { + // const jsonData = JSON.parse(data); + // console.log(`response: ${JSON.stringify(jsonData)}`); + // if (jsonData.clustering_type === 'kubernetes') { + // console.log('jsonData.clusteringType === kubernetes'); + // terasliceRunning = true; + // } + // }); + // } + // } catch (err) { + // await PromiseTimeout(3000); + // return _waitForTerasliceRunning(); + // } + + // if (terasliceRunning) return true; + // return _waitForTerasliceRunning(); + // }; + + // return _waitForTerasliceRunning(); + // } } export async function setAlias() { @@ -621,12 +695,23 @@ export async function setAlias() { console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function registerTestJob(k8se2eDir: string) { - const subprocess = await execa.command(`earl tjm register localhost ${path.join(k8se2eDir, 'testJob.json')}`); +export async function registerTestJob() { + const subprocess = await execa.command('earl tjm register localhost testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } -export async function startTestJob(k8se2eDir: string) { - const subprocess = await execa.command(`earl tjm start ${path.join(k8se2eDir, 'testJob.json')}`); +export async function startTestJob() { + const subprocess = await execa.command('earl tjm start testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } + +export async function showState() { + const subprocess = await execa.command('kubectl -n ts-dev1 get deployments,po,svc -o wide'); + console.log('showState subprocess: ', subprocess); +} + +function PromiseTimeout(delayms: number) { + return new Promise((resolve) => { + setTimeout(resolve, delayms); + }); +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 39da75774b1..ceba05ba9bd 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -22,7 +22,8 @@ import { isKindInstalled, isKubectlInstalled, registerTestJob, - startTestJob + startTestJob, + showState } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -383,9 +384,10 @@ async function runk8sE2ETest( await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); + await showState(); - // await registerTestJob(k8se2eDir); - // await startTestJob(k8se2eDir); + await registerTestJob(); + await startTestJob(); // if (!tracker.hasErrors()) { // const timeLabel = `test suite "${suite}"`; From a143ecb05e62534f8a441fcfc8e3c5833013d725 Mon Sep 17 00:00:00 2001 From: sotojn Date: Fri, 13 Oct 2023 15:04:09 -0700 Subject: [PATCH 004/142] update testReaderFile & testJob for k8s e2e --- k8se2e/testJob.json | 22 ++++++++++-- k8se2e/testReaderFile.json | 72 ++++++++++++++++++++++++++++++++++---- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/k8se2e/testJob.json b/k8se2e/testJob.json index f688bdcb954..10baeb47de4 100644 --- a/k8se2e/testJob.json +++ b/k8se2e/testJob.json @@ -2,11 +2,27 @@ "name": "Test Job", "lifecycle": "once", "workers": 1, - "assets": [], + "assets": [ + "elasticsearch" + ], "operations": [ { "_op": "test-reader", - "fetcher_data_file_path": "./testReaderFile.json" + "fetcher_data_file_path": "./k8se2e/testReaderFile.json" + }, + { + "_op": "elasticsearch_bulk", + "index": "terak8s-example-data", + "type": "events", + "size": 1 + } + ], + "__metadata": { + "cli": { + "cluster": "http://localhost:5678", + "version": "0.55.1", + "job_id": "7c30c1f9-31b1-4ec9-b5c0-da4624fed030", + "updated": "2023-10-13T21:58:14.228Z" } - ] + } } diff --git a/k8se2e/testReaderFile.json b/k8se2e/testReaderFile.json index b0126017515..bada14e3ebf 100644 --- a/k8se2e/testReaderFile.json +++ b/k8se2e/testReaderFile.json @@ -1,8 +1,68 @@ [ - { - "foo": "bar" - }, - { - "foo": "baz" - } + { "name": "John Doe", "age": 44, "accountId": 43394838 }, + { "name": "Jane Smith", "age": 28, "accountId": 82736482 }, + { "name": "Michael Johnson", "age": 35, "accountId": 12874938 }, + { "name": "Emily Davis", "age": 52, "accountId": 29384729 }, + { "name": "William Brown", "age": 29, "accountId": 74839274 }, + { "name": "Sarah Wilson", "age": 41, "accountId": 49283747 }, + { "name": "David Lee", "age": 37, "accountId": 23847283 }, + { "name": "Jennifer Harris", "age": 32, "accountId": 58374635 }, + { "name": "Joseph Clark", "age": 50, "accountId": 37483748 }, + { "name": "Linda Lewis", "age": 45, "accountId": 93847564 }, + { "name": "Robert Hall", "age": 33, "accountId": 47298374 }, + { "name": "Karen Turner", "age": 49, "accountId": 73847293 }, + { "name": "Richard Garcia", "age": 39, "accountId": 94872347 }, + { "name": "Patricia Rodriguez", "age": 56, "accountId": 64827364 }, + { "name": "Charles Martinez", "age": 43, "accountId": 87348634 }, + { "name": "Nancy Jackson", "age": 30, "accountId": 74623847 }, + { "name": "Thomas Allen", "age": 47, "accountId": 38274692 }, + { "name": "Betty Anderson", "age": 36, "accountId": 92384728 }, + { "name": "Daniel Moore", "age": 53, "accountId": 37483947 }, + { "name": "Deborah White", "age": 40, "accountId": 48374839 }, + { "name": "Paul Martin", "age": 31, "accountId": 73847283 }, + { "name": "Sandra Taylor", "age": 55, "accountId": 23847236 }, + { "name": "Mark Scott", "age": 38, "accountId": 82734628 }, + { "name": "Laura King", "age": 34, "accountId": 74628374 }, + { "name": "Kevin Young", "age": 46, "accountId": 23846283 }, + { "name": "Cynthia Turner", "age": 42, "accountId": 27486374 }, + { "name": "Christopher Thomas", "age": 48, "accountId": 34872634 }, + { "name": "Susan Adams", "age": 29, "accountId": 78374628 }, + { "name": "Kenneth Walker", "age": 57, "accountId": 87263847 }, + { "name": "Ashley Baker", "age": 33, "accountId": 37462836 }, + { "name": "George White", "age": 52, "accountId": 48364782 }, + { "name": "Jessica Robinson", "age": 35, "accountId": 34628347 }, + { "name": "Edward Perez", "age": 41, "accountId": 82736482 }, + { "name": "Amanda Mitchell", "age": 44, "accountId": 73647283 }, + { "name": "Brian Turner", "age": 37, "accountId": 48327462 }, + { "name": "Maria Evans", "age": 48, "accountId": 38274636 }, + { "name": "James Davis", "age": 31, "accountId": 47283947 }, + { "name": "Karen Hall", "age": 56, "accountId": 74638274 }, + { "name": "Jason Martinez", "age": 39, "accountId": 28374683 }, + { "name": "Donna Johnson", "age": 32, "accountId": 74826374 }, + { "name": "Ronald Brown", "age": 50, "accountId": 82734682 }, + { "name": "Carol Lewis", "age": 43, "accountId": 34786374 }, + { "name": "William Taylor", "age": 38, "accountId": 73647283 }, + { "name": "Sharon Robinson", "age": 47, "accountId": 47364826 }, + { "name": "Michael Scott", "age": 30, "accountId": 82374683 }, + { "name": "Ruth Harris", "age": 45, "accountId": 48374628 }, + { "name": "Kevin Anderson", "age": 42, "accountId": 73648236 }, + { "name": "Deborah Perez", "age": 53, "accountId": 23874623 }, + { "name": "Richard Turner", "age": 34, "accountId": 37482364 }, + { "name": "Lisa Baker", "age": 49, "accountId": 48236478 }, + { "name": "David Mitchell", "age": 46, "accountId": 37468236 }, + { "name": "Mary Adams", "age": 52, "accountId": 84623847 }, + { "name": "Joseph King", "age": 41, "accountId": 23874623 }, + { "name": "Susan Harris", "age": 37, "accountId": 47364826 }, + { "name": "Christopher Moore", "age": 35, "accountId": 73648236 }, + { "name": "Patricia Davis", "age": 54, "accountId": 36478236 }, + { "name": "Matthew Garcia", "age": 44, "accountId": 74823647 }, + { "name": "Linda Turner", "age": 33, "accountId": 82364826 }, + { "name": "Charles Scott", "age": 47, "accountId": 36478236 }, + { "name": "Nancy Robinson", "age": 36, "accountId": 74826347 }, + { "name": "Daniel Evans", "age": 51, "accountId": 48374682 }, + { "name": "Betty Perez", "age": 40, "accountId": 73648236 }, + { "name": "Michael Lewis", "age": 38, "accountId": 48364782 }, + { "name": "Karen Hall", "age": 46, "accountId": 36478236 }, + { "name": "William Davis", "age": 49, "accountId": 74823647 } + ] From b273a9c3273e424fe222e4b4e59f2239f53793f9 Mon Sep 17 00:00:00 2001 From: sotojn Date: Mon, 16 Oct 2023 08:26:57 -0700 Subject: [PATCH 005/142] add setAlias and registerElasticsearch method to k8e2e tests. Update Dockerfile --- Dockerfile | 1 + packages/scripts/src/helpers/scripts.ts | 13 ++++++++++--- packages/scripts/src/helpers/test-runner/index.ts | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a7ddfb5b162..9b5789fe6cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ COPY .yarn /app/source/.yarn COPY packages /app/source/packages COPY scripts /app/source/scripts COPY types /app/source/types +COPY k8se2e /app/source/k8se2e RUN yarn --prod=false --frozen-lockfile \ && yarn build \ diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index ddf1871072a..e87624261d5 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -690,19 +690,26 @@ export async function deployk8sTeraslice( } export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + // const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } export async function registerTestJob() { - const subprocess = await execa.command('earl tjm register localhost testJob.json'); + const subprocess = await execa.command('earl tjm register ts-k8s-e2e testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } +export async function registerElasticsearch() { + const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/elasticsearch-assets'); + console.log('registerElasticsearch asset subprocess: ', subprocess); +} + export async function startTestJob() { const subprocess = await execa.command('earl tjm start testJob.json'); - console.log('registerTestJob subprocess: ', subprocess); + console.log('Run earl tjm start testJob.json'); + console.log(subprocess.stdout); } export async function showState() { diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index ceba05ba9bd..9eb6aa55255 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -23,7 +23,9 @@ import { isKubectlInstalled, registerTestJob, startTestJob, - showState + showState, + registerElasticsearch, + setAlias } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -386,6 +388,8 @@ async function runk8sE2ETest( await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); await showState(); + await setAlias(); + await registerElasticsearch(); await registerTestJob(); await startTestJob(); From abb0c137ac788d4ecc2c13b793bb095b6ca4c89c Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 16 Oct 2023 08:40:04 -0700 Subject: [PATCH 006/142] Rework waitForElasticsearch and waitForTeraslice --- packages/scripts/src/helpers/scripts.ts | 125 +++++++++++++----------- 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index e87624261d5..ff4b1e3c0ba 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -14,7 +14,6 @@ import { TSCommands, PackageInfo } from './interfaces'; import { getRootDir } from './misc'; import signale from './signale'; import * as config from './config'; -import { setTimeout } from 'timers'; const logger = debugLogger('ts-scripts:cmd'); @@ -586,22 +585,44 @@ export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); console.log('esDeploy subprocess: ', subprocess); - let elasticsearchRunning = false; - // TODO: add curl check to ES before continuing - while (!elasticsearchRunning) { - const ESResponse = execa.command('curl localhost:9200').stdout; - if (ESResponse) { - ESResponse.on('data', (data) => { - const jsonData = JSON.parse(data); + const elasticsearchReady = await waitForESRunning(240000); + if (elasticsearchReady) { + signale.success('Elasticsearch is ready to go'); + } +} + +function waitForESRunning(timeoutMs = 120000): Promise { + const endAt = Date.now() + timeoutMs; + + const _waitForESRunning = async (): Promise => { + if (Date.now() > endAt) { + throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); + } + + let elasticsearchRunning = false; + try { + const ESResponse = await execa.command('curl localhost:9200'); + if (ESResponse.stdout) { + const jsonData = JSON.parse(ESResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); if (jsonData.tagline === 'You Know, for Search') { - console.log('jsonData.tagline === You Know, for Search') + console.log('jsonData.tagline === You Know, for Search'); elasticsearchRunning = true; } - }); + } + } catch (err) { + await PromiseTimeout(3000); + return _waitForESRunning(); + } + + if (elasticsearchRunning) { + return true; } await PromiseTimeout(3000); - } + return _waitForESRunning(); + }; + + return _waitForESRunning(); } export async function k8sSetup( @@ -636,66 +657,58 @@ export async function deployk8sTeraslice( const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); console.log('podName: ', podName); - // await waitForTerasliceRunning(120000); + const terasliceReady = await waitForTerasliceRunning(240000); + if (terasliceReady) { + signale.success('Teraslice is ready to go'); + } +} + +function waitForTerasliceRunning(timeoutMs = 120000): Promise { + const endAt = Date.now() + timeoutMs; - let terasliceRunning = false; + const _waitForTerasliceRunning = async (): Promise => { + if (Date.now() > endAt) { + throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); + } - while (!terasliceRunning) { - const TSMasterResponse = execa.command('curl localhost:5678').stdout; - if (TSMasterResponse) { - TSMasterResponse.on('data', (data) => { - const jsonData = JSON.parse(data); + let terasliceRunning = false; + try { + const TSMasterResponse = await execa.command('curl localhost:5678'); + if (TSMasterResponse.stdout) { + const jsonData = JSON.parse(TSMasterResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); if (jsonData.clustering_type === 'kubernetes') { - console.log('jsonData.clusteringType === kubernetes') + console.log('jsonData.clusteringType === kubernetes'); terasliceRunning = true; } - }); + } + } catch (err) { + await PromiseTimeout(3000); + return _waitForTerasliceRunning(); + } + + if (terasliceRunning) { + return true; } + await PromiseTimeout(3000); - } - // function waitForTerasliceRunning(timeoutMs = 120000) : Promise { - // const endAt = Date.now() + timeoutMs; - - // const _waitForTerasliceRunning = async () : Promise => { - // if (Date.now() > endAt) { - // throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); - // } - - // let terasliceRunning = false; - // // let nodes = -1; - // try { - // const TSMasterResponse = execa.command('curl localhost:5678').stdout; - // if (TSMasterResponse) { - // TSMasterResponse.on('data', (data) => { - // const jsonData = JSON.parse(data); - // console.log(`response: ${JSON.stringify(jsonData)}`); - // if (jsonData.clustering_type === 'kubernetes') { - // console.log('jsonData.clusteringType === kubernetes'); - // terasliceRunning = true; - // } - // }); - // } - // } catch (err) { - // await PromiseTimeout(3000); - // return _waitForTerasliceRunning(); - // } - - // if (terasliceRunning) return true; - // return _waitForTerasliceRunning(); - // }; - - // return _waitForTerasliceRunning(); - // } + return _waitForTerasliceRunning(); + }; + + return _waitForTerasliceRunning(); } export async function setAlias() { - // const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } +export async function deployESAsset() { + const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --blocking --bundle terascope/elasticsearch-assets'); + console.log('deployESAsset subprocess: ', subprocess); +} + export async function registerTestJob() { const subprocess = await execa.command('earl tjm register ts-k8s-e2e testJob.json'); console.log('registerTestJob subprocess: ', subprocess); From 51f4ac8136578ce0d7eab8d79ff50fd05a11c0e5 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 16 Oct 2023 09:03:58 -0700 Subject: [PATCH 007/142] Fix merge error in setAlias function. Use shell. --- packages/scripts/src/helpers/scripts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index ff4b1e3c0ba..45a3b465da3 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -699,7 +699,7 @@ function waitForTerasliceRunning(timeoutMs = 120000): Promise { } export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } From 18914baab364623001b5e9175e52eb3d9dcb0b42 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 08:20:55 -0700 Subject: [PATCH 008/142] Remove k8se2e directory and all its files --- k8se2e/docker-compose.yml | 32 -------------- k8se2e/elasticsearchDeployment.yaml | 45 ------------------- k8se2e/jest.config.js | 7 --- k8se2e/kindConfig.yaml | 10 ----- k8se2e/masterConfig/teraslice.yaml | 31 ------------- k8se2e/masterDeployment.yaml | 53 ---------------------- k8se2e/ns.yaml | 4 -- k8se2e/package.json | 55 ----------------------- k8se2e/priorityClass.yaml | 7 --- k8se2e/role.yaml | 9 ---- k8se2e/roleBinding.yaml | 13 ------ k8se2e/testJob.json | 28 ------------ k8se2e/testReaderFile.json | 68 ----------------------------- k8se2e/workerConfig/teraslice.yaml | 29 ------------ 14 files changed, 391 deletions(-) delete mode 100644 k8se2e/docker-compose.yml delete mode 100644 k8se2e/elasticsearchDeployment.yaml delete mode 100644 k8se2e/jest.config.js delete mode 100644 k8se2e/kindConfig.yaml delete mode 100644 k8se2e/masterConfig/teraslice.yaml delete mode 100644 k8se2e/masterDeployment.yaml delete mode 100644 k8se2e/ns.yaml delete mode 100644 k8se2e/package.json delete mode 100644 k8se2e/priorityClass.yaml delete mode 100644 k8se2e/role.yaml delete mode 100644 k8se2e/roleBinding.yaml delete mode 100644 k8se2e/testJob.json delete mode 100644 k8se2e/testReaderFile.json delete mode 100644 k8se2e/workerConfig/teraslice.yaml diff --git a/k8se2e/docker-compose.yml b/k8se2e/docker-compose.yml deleted file mode 100644 index e8cd69ed047..00000000000 --- a/k8se2e/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '2.4' -services: - teraslice-master: - image: teraslice-workspace:e2e - ports: - - '45678:45678' - scale: 1 - restart: 'no' - stop_grace_period: 30s - environment: - - TERAFOUNDATION_CONFIG=/app/config/teraslice-master.json - - NODE_OPTIONS=--max-old-space-size=256 - volumes: - - ./logs:/app/logs:delegated - - ./autoload:/app/autoload:ro - - ./.config:/app/config:ro - - ./.assets:/app/assets:delegated - teraslice-worker: - image: teraslice-workspace:e2e - scale: 2 - restart: 'no' - stop_grace_period: 30s - links: - - teraslice-master - environment: - - TERAFOUNDATION_CONFIG=/app/config/teraslice-worker.json - - NODE_OPTIONS=--max-old-space-size=256 - volumes: - - ./logs:/app/logs:delegated - - ./autoload:/app/autoload:ro - - ./.config:/app/config:ro - - ./.assets:/app/assets:delegated diff --git a/k8se2e/elasticsearchDeployment.yaml b/k8se2e/elasticsearchDeployment.yaml deleted file mode 100644 index 5ea8287b1aa..00000000000 --- a/k8se2e/elasticsearchDeployment.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: elasticsearch - labels: - app: elasticsearch - nodeType: master -spec: - replicas: 1 - selector: - matchLabels: - app: elasticsearch - nodeType: master - template: - metadata: - labels: - app: elasticsearch - nodeType: master - spec: - containers: - - name: elasticsearch - image: elasticsearch:7.9.3 - ports: - - containerPort: 9200 - env: - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - - name: discovery.type - value: single-node ---- -kind: Service -apiVersion: v1 -metadata: - name: elasticsearch - labels: - app: elasticsearch -spec: - selector: - app: elasticsearch - nodeType: master - ports: - - port: 9200 - targetPort: 9200 - nodePort: 30200 # the external port teraslice can be accessed on - type: NodePort diff --git a/k8se2e/jest.config.js b/k8se2e/jest.config.js deleted file mode 100644 index 0a4be3a992a..00000000000 --- a/k8se2e/jest.config.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -const config = require('../jest.config.base')(__dirname); - -config.collectCoverage = false; -delete config.transform; -module.exports = config; diff --git a/k8se2e/kindConfig.yaml b/k8se2e/kindConfig.yaml deleted file mode 100644 index e9d87e52e73..00000000000 --- a/k8se2e/kindConfig.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: Cluster -name: k8se2e -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 30200 - hostPort: 9200 - - containerPort: 30678 - hostPort: 5678 diff --git a/k8se2e/masterConfig/teraslice.yaml b/k8se2e/masterConfig/teraslice.yaml deleted file mode 100644 index 8f1aaddea3e..00000000000 --- a/k8se2e/masterConfig/teraslice.yaml +++ /dev/null @@ -1,31 +0,0 @@ -terafoundation: - environment: 'development' - log_level: debug - connectors: - elasticsearch: - default: - apiVersion: "5.6" - host: - - "elasticsearch:9200" - elasticsearch-next: - default: - node: - - "http://elasticsearch:9200" -teraslice: - worker_disconnect_timeout: 60000 - node_disconnect_timeout: 60000 - slicer_timeout: 60000 - shutdown_timeout: 30000 - assets_directory: '/app/assets/' - cluster_manager_type: "kubernetes" - master: true - master_hostname: "127.0.0.1" - kubernetes_image: "teraslice-workspace:k8se2e" - kubernetes_image_pull_secrets: - - "docker-tera1-secret" - kubernetes_namespace: "ts-dev1" - kubernetes_overrides_enabled: true - kubernetes_priority_class_name: 'high-priority' - name: "ts-dev1" - cpu: 1 - memory: 536870912 diff --git a/k8se2e/masterDeployment.yaml b/k8se2e/masterDeployment.yaml deleted file mode 100644 index 14433977e66..00000000000 --- a/k8se2e/masterDeployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: teraslice-master - labels: - app: teraslice - nodeType: master -spec: - replicas: 1 - selector: - matchLabels: - app: teraslice - nodeType: master - template: - metadata: - labels: - app: teraslice - nodeType: master - clusterName: ts-dev1 - spec: - containers: - - name: teraslice-master - image: teraslice-workspace:k8se2e - ports: - - containerPort: 5678 - volumeMounts: - - mountPath: /app/config # defines the directory - name: config - volumes: - - name: config - configMap: - name: teraslice-master - items: - - key: teraslice.yaml - path: teraslice.yaml # the filename that the configMap gets written to, inside the matching mountPath - imagePullSecrets: - - name: docker-tera1-secret ---- -kind: Service -apiVersion: v1 -metadata: - name: teraslice-master - labels: - app: teraslice -spec: - selector: - app: teraslice - nodeType: master - ports: - - port: 5678 - targetPort: 5678 - nodePort: 30678 # the external port teraslice can be accessed on - type: NodePort diff --git a/k8se2e/ns.yaml b/k8se2e/ns.yaml deleted file mode 100644 index 1af803c0c4f..00000000000 --- a/k8se2e/ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: ts-dev1 \ No newline at end of file diff --git a/k8se2e/package.json b/k8se2e/package.json deleted file mode 100644 index 64302c49c46..00000000000 --- a/k8se2e/package.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "k8se2e", - "displayName": "K8S E2E Tests", - "version": "0.0.1", - "private": true, - "description": "Teraslice in Kubernetes integration test suite", - "keywords": [ - "docker-compose", - "elasticsearch", - "teraslice" - ], - "homepage": "https://github.com/terascope/teraslice/tree/master/k8se2e/#readme", - "bugs": { - "url": "https://github.com/terascope/teraslice/issues" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/terascope/teraslice.git" - }, - "license": "MIT", - "author": "Terascope, LLC ", - "scripts": { - "clean": "docker-compose down --volumes --remove-orphans --timeout=5", - "logs": "./scripts/logs.sh", - "logs-follow": "./scripts/logs.sh -f", - "setup": "yarn --silent", - "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --debug --", - "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --watch --" - }, - "resolutions": { - "ms": "^2.1.3" - }, - "devDependencies": { - "bunyan": "^1.8.15", - "elasticsearch-store": "^0.72.0", - "fs-extra": "^10.1.0", - "ms": "^2.1.3", - "nanoid": "^3.3.4", - "semver": "^7.3.8", - "signale": "^1.4.0", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14.17.0", - "yarn": ">=1.16.0" - }, - "terascope": { - "testSuite": "k8se2e" - } -} diff --git a/k8se2e/priorityClass.yaml b/k8se2e/priorityClass.yaml deleted file mode 100644 index 0747825c87d..00000000000 --- a/k8se2e/priorityClass.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: high-priority -value: 1000000 -globalDefault: false -description: "This priority class is for Teraslice pods." diff --git a/k8se2e/role.yaml b/k8se2e/role.yaml deleted file mode 100644 index 725cec875a5..00000000000 --- a/k8se2e/role.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: teraslice-all-ts-dev1 - namespace: ts-dev1 -rules: - - apiGroups: ["*"] - resources: ["*"] - verbs: ["*"] \ No newline at end of file diff --git a/k8se2e/roleBinding.yaml b/k8se2e/roleBinding.yaml deleted file mode 100644 index b46683aeb41..00000000000 --- a/k8se2e/roleBinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: teraslice-all-ts-dev1 - namespace: ts-dev1 -subjects: - - kind: ServiceAccount - name: default - namespace: ts-dev1 -roleRef: - kind: Role - name: teraslice-all-ts-dev1 - apiGroup: "rbac.authorization.k8s.io" \ No newline at end of file diff --git a/k8se2e/testJob.json b/k8se2e/testJob.json deleted file mode 100644 index 10baeb47de4..00000000000 --- a/k8se2e/testJob.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "Test Job", - "lifecycle": "once", - "workers": 1, - "assets": [ - "elasticsearch" - ], - "operations": [ - { - "_op": "test-reader", - "fetcher_data_file_path": "./k8se2e/testReaderFile.json" - }, - { - "_op": "elasticsearch_bulk", - "index": "terak8s-example-data", - "type": "events", - "size": 1 - } - ], - "__metadata": { - "cli": { - "cluster": "http://localhost:5678", - "version": "0.55.1", - "job_id": "7c30c1f9-31b1-4ec9-b5c0-da4624fed030", - "updated": "2023-10-13T21:58:14.228Z" - } - } -} diff --git a/k8se2e/testReaderFile.json b/k8se2e/testReaderFile.json deleted file mode 100644 index bada14e3ebf..00000000000 --- a/k8se2e/testReaderFile.json +++ /dev/null @@ -1,68 +0,0 @@ -[ - { "name": "John Doe", "age": 44, "accountId": 43394838 }, - { "name": "Jane Smith", "age": 28, "accountId": 82736482 }, - { "name": "Michael Johnson", "age": 35, "accountId": 12874938 }, - { "name": "Emily Davis", "age": 52, "accountId": 29384729 }, - { "name": "William Brown", "age": 29, "accountId": 74839274 }, - { "name": "Sarah Wilson", "age": 41, "accountId": 49283747 }, - { "name": "David Lee", "age": 37, "accountId": 23847283 }, - { "name": "Jennifer Harris", "age": 32, "accountId": 58374635 }, - { "name": "Joseph Clark", "age": 50, "accountId": 37483748 }, - { "name": "Linda Lewis", "age": 45, "accountId": 93847564 }, - { "name": "Robert Hall", "age": 33, "accountId": 47298374 }, - { "name": "Karen Turner", "age": 49, "accountId": 73847293 }, - { "name": "Richard Garcia", "age": 39, "accountId": 94872347 }, - { "name": "Patricia Rodriguez", "age": 56, "accountId": 64827364 }, - { "name": "Charles Martinez", "age": 43, "accountId": 87348634 }, - { "name": "Nancy Jackson", "age": 30, "accountId": 74623847 }, - { "name": "Thomas Allen", "age": 47, "accountId": 38274692 }, - { "name": "Betty Anderson", "age": 36, "accountId": 92384728 }, - { "name": "Daniel Moore", "age": 53, "accountId": 37483947 }, - { "name": "Deborah White", "age": 40, "accountId": 48374839 }, - { "name": "Paul Martin", "age": 31, "accountId": 73847283 }, - { "name": "Sandra Taylor", "age": 55, "accountId": 23847236 }, - { "name": "Mark Scott", "age": 38, "accountId": 82734628 }, - { "name": "Laura King", "age": 34, "accountId": 74628374 }, - { "name": "Kevin Young", "age": 46, "accountId": 23846283 }, - { "name": "Cynthia Turner", "age": 42, "accountId": 27486374 }, - { "name": "Christopher Thomas", "age": 48, "accountId": 34872634 }, - { "name": "Susan Adams", "age": 29, "accountId": 78374628 }, - { "name": "Kenneth Walker", "age": 57, "accountId": 87263847 }, - { "name": "Ashley Baker", "age": 33, "accountId": 37462836 }, - { "name": "George White", "age": 52, "accountId": 48364782 }, - { "name": "Jessica Robinson", "age": 35, "accountId": 34628347 }, - { "name": "Edward Perez", "age": 41, "accountId": 82736482 }, - { "name": "Amanda Mitchell", "age": 44, "accountId": 73647283 }, - { "name": "Brian Turner", "age": 37, "accountId": 48327462 }, - { "name": "Maria Evans", "age": 48, "accountId": 38274636 }, - { "name": "James Davis", "age": 31, "accountId": 47283947 }, - { "name": "Karen Hall", "age": 56, "accountId": 74638274 }, - { "name": "Jason Martinez", "age": 39, "accountId": 28374683 }, - { "name": "Donna Johnson", "age": 32, "accountId": 74826374 }, - { "name": "Ronald Brown", "age": 50, "accountId": 82734682 }, - { "name": "Carol Lewis", "age": 43, "accountId": 34786374 }, - { "name": "William Taylor", "age": 38, "accountId": 73647283 }, - { "name": "Sharon Robinson", "age": 47, "accountId": 47364826 }, - { "name": "Michael Scott", "age": 30, "accountId": 82374683 }, - { "name": "Ruth Harris", "age": 45, "accountId": 48374628 }, - { "name": "Kevin Anderson", "age": 42, "accountId": 73648236 }, - { "name": "Deborah Perez", "age": 53, "accountId": 23874623 }, - { "name": "Richard Turner", "age": 34, "accountId": 37482364 }, - { "name": "Lisa Baker", "age": 49, "accountId": 48236478 }, - { "name": "David Mitchell", "age": 46, "accountId": 37468236 }, - { "name": "Mary Adams", "age": 52, "accountId": 84623847 }, - { "name": "Joseph King", "age": 41, "accountId": 23874623 }, - { "name": "Susan Harris", "age": 37, "accountId": 47364826 }, - { "name": "Christopher Moore", "age": 35, "accountId": 73648236 }, - { "name": "Patricia Davis", "age": 54, "accountId": 36478236 }, - { "name": "Matthew Garcia", "age": 44, "accountId": 74823647 }, - { "name": "Linda Turner", "age": 33, "accountId": 82364826 }, - { "name": "Charles Scott", "age": 47, "accountId": 36478236 }, - { "name": "Nancy Robinson", "age": 36, "accountId": 74826347 }, - { "name": "Daniel Evans", "age": 51, "accountId": 48374682 }, - { "name": "Betty Perez", "age": 40, "accountId": 73648236 }, - { "name": "Michael Lewis", "age": 38, "accountId": 48364782 }, - { "name": "Karen Hall", "age": 46, "accountId": 36478236 }, - { "name": "William Davis", "age": 49, "accountId": 74823647 } - -] diff --git a/k8se2e/workerConfig/teraslice.yaml b/k8se2e/workerConfig/teraslice.yaml deleted file mode 100644 index 6615cc43757..00000000000 --- a/k8se2e/workerConfig/teraslice.yaml +++ /dev/null @@ -1,29 +0,0 @@ -terafoundation: - environment: 'development' - log_level: debug - connectors: - elasticsearch: - default: - apiVersion: "5.6" - host: - - "elasticsearch:9200" - elasticsearch-next: - default: - node: - - "http://elasticsearch:9200" -teraslice: - worker_disconnect_timeout: 60000 - node_disconnect_timeout: 60000 - slicer_timeout: 60000 - shutdown_timeout: 30000 - assets_directory: '/app/assets/' - cluster_manager_type: "kubernetes" - master: false - master_hostname: "teraslice-master" - kubernetes_image: "teraslice-workspace:k8se2e" - kubernetes_namespace: "ts-dev1" - kubernetes_overrides_enabled: true - kubernetes_priority_class_name: 'high-priority' - name: "ts-dev1" - cpu: 1 - memory: 536870912 From 5ab754eca915381d3a33bd32b9d22961d1919112 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 08:21:39 -0700 Subject: [PATCH 009/142] Add k8se2e to top level package.json's workspaces --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5044388055b..9db7ce63b2d 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ }, "workspaces": [ "packages/*", - "e2e" + "e2e", + "k8se2e" ], "scripts": { "prebuild": "./packages/xlucene-parser/scripts/generate-engine.js", From d370c69bf3b8e23bb03b7f9ee23afbe3d0e8dcba Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 08:23:32 -0700 Subject: [PATCH 010/142] Add k8s directory and kind files to e2e --- e2e/k8s/Dockerfile | 43 ++++++++++++++++++++++ e2e/k8s/elasticsearchDeployment.yaml | 45 +++++++++++++++++++++++ e2e/k8s/kindConfig.yaml | 10 ++++++ e2e/k8s/masterConfig/teraslice.yaml | 31 ++++++++++++++++ e2e/k8s/masterDeployment.yaml | 53 ++++++++++++++++++++++++++++ e2e/k8s/ns.yaml | 4 +++ e2e/k8s/priorityClass.yaml | 7 ++++ e2e/k8s/role.yaml | 9 +++++ e2e/k8s/roleBinding.yaml | 13 +++++++ e2e/k8s/workerConfig/teraslice.yaml | 29 +++++++++++++++ 10 files changed, 244 insertions(+) create mode 100644 e2e/k8s/Dockerfile create mode 100644 e2e/k8s/elasticsearchDeployment.yaml create mode 100644 e2e/k8s/kindConfig.yaml create mode 100644 e2e/k8s/masterConfig/teraslice.yaml create mode 100644 e2e/k8s/masterDeployment.yaml create mode 100644 e2e/k8s/ns.yaml create mode 100644 e2e/k8s/priorityClass.yaml create mode 100644 e2e/k8s/role.yaml create mode 100644 e2e/k8s/roleBinding.yaml create mode 100644 e2e/k8s/workerConfig/teraslice.yaml diff --git a/e2e/k8s/Dockerfile b/e2e/k8s/Dockerfile new file mode 100644 index 00000000000..9b5789fe6cc --- /dev/null +++ b/e2e/k8s/Dockerfile @@ -0,0 +1,43 @@ +# NODE_VERSION is set by default in the config.ts, the following value will only +# be used if you build images by default with docker build +ARG NODE_VERSION=14.21.3 +FROM terascope/node-base:${NODE_VERSION} + +ENV NODE_ENV production + +ENV YARN_SETUP_ARGS "--prod=false --silent --frozen-lockfile" + +COPY package.json yarn.lock tsconfig.json .yarnrc /app/source/ +COPY .yarnclean.ci /app/source/.yarnclean +COPY .yarn /app/source/.yarn +COPY packages /app/source/packages +COPY scripts /app/source/scripts +COPY types /app/source/types +COPY k8se2e /app/source/k8se2e + +RUN yarn --prod=false --frozen-lockfile \ + && yarn build \ + && yarn \ + --prod=true \ + --silent \ + --frozen-lockfile \ + --skip-integrity-check \ + --ignore-scripts \ + && yarn cache clean + + +COPY service.js /app/source/ + +# verify node-rdkafka is installed right +RUN node -e "require('node-rdkafka')" + +# verify teraslice is installed right +RUN node -e "require('teraslice')" + +EXPOSE 5678 + +# set up the volumes +VOLUME /app/config /app/logs /app/assets +ENV TERAFOUNDATION_CONFIG /app/config/teraslice.yaml + +CMD ["node", "service.js"] diff --git a/e2e/k8s/elasticsearchDeployment.yaml b/e2e/k8s/elasticsearchDeployment.yaml new file mode 100644 index 00000000000..5ea8287b1aa --- /dev/null +++ b/e2e/k8s/elasticsearchDeployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + nodeType: master + template: + metadata: + labels: + app: elasticsearch + nodeType: master + spec: + containers: + - name: elasticsearch + image: elasticsearch:7.9.3 + ports: + - containerPort: 9200 + env: + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: single-node +--- +kind: Service +apiVersion: v1 +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + nodeType: master + ports: + - port: 9200 + targetPort: 9200 + nodePort: 30200 # the external port teraslice can be accessed on + type: NodePort diff --git a/e2e/k8s/kindConfig.yaml b/e2e/k8s/kindConfig.yaml new file mode 100644 index 00000000000..e9d87e52e73 --- /dev/null +++ b/e2e/k8s/kindConfig.yaml @@ -0,0 +1,10 @@ +kind: Cluster +name: k8se2e +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30200 + hostPort: 9200 + - containerPort: 30678 + hostPort: 5678 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml new file mode 100644 index 00000000000..8f1aaddea3e --- /dev/null +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -0,0 +1,31 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: true + master_hostname: "127.0.0.1" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image_pull_secrets: + - "docker-tera1-secret" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml new file mode 100644 index 00000000000..14433977e66 --- /dev/null +++ b/e2e/k8s/masterDeployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: teraslice-master + labels: + app: teraslice + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: teraslice + nodeType: master + template: + metadata: + labels: + app: teraslice + nodeType: master + clusterName: ts-dev1 + spec: + containers: + - name: teraslice-master + image: teraslice-workspace:k8se2e + ports: + - containerPort: 5678 + volumeMounts: + - mountPath: /app/config # defines the directory + name: config + volumes: + - name: config + configMap: + name: teraslice-master + items: + - key: teraslice.yaml + path: teraslice.yaml # the filename that the configMap gets written to, inside the matching mountPath + imagePullSecrets: + - name: docker-tera1-secret +--- +kind: Service +apiVersion: v1 +metadata: + name: teraslice-master + labels: + app: teraslice +spec: + selector: + app: teraslice + nodeType: master + ports: + - port: 5678 + targetPort: 5678 + nodePort: 30678 # the external port teraslice can be accessed on + type: NodePort diff --git a/e2e/k8s/ns.yaml b/e2e/k8s/ns.yaml new file mode 100644 index 00000000000..1af803c0c4f --- /dev/null +++ b/e2e/k8s/ns.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: ts-dev1 \ No newline at end of file diff --git a/e2e/k8s/priorityClass.yaml b/e2e/k8s/priorityClass.yaml new file mode 100644 index 00000000000..0747825c87d --- /dev/null +++ b/e2e/k8s/priorityClass.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000000 +globalDefault: false +description: "This priority class is for Teraslice pods." diff --git a/e2e/k8s/role.yaml b/e2e/k8s/role.yaml new file mode 100644 index 00000000000..725cec875a5 --- /dev/null +++ b/e2e/k8s/role.yaml @@ -0,0 +1,9 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] \ No newline at end of file diff --git a/e2e/k8s/roleBinding.yaml b/e2e/k8s/roleBinding.yaml new file mode 100644 index 00000000000..b46683aeb41 --- /dev/null +++ b/e2e/k8s/roleBinding.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +subjects: + - kind: ServiceAccount + name: default + namespace: ts-dev1 +roleRef: + kind: Role + name: teraslice-all-ts-dev1 + apiGroup: "rbac.authorization.k8s.io" \ No newline at end of file diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml new file mode 100644 index 00000000000..6615cc43757 --- /dev/null +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -0,0 +1,29 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: false + master_hostname: "teraslice-master" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 From d618b144134e5c3c80ced095c53173c67cbd29e2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:19:25 -0700 Subject: [PATCH 011/142] revert changes to root dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9b5789fe6cc..a7ddfb5b162 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY .yarn /app/source/.yarn COPY packages /app/source/packages COPY scripts /app/source/scripts COPY types /app/source/types -COPY k8se2e /app/source/k8se2e RUN yarn --prod=false --frozen-lockfile \ && yarn build \ From ae9383a394b268fcb1741a005cd8de69dead615f Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:20:23 -0700 Subject: [PATCH 012/142] modify k8s yaml file ports --- e2e/k8s/Dockerfile | 2 +- e2e/k8s/kindConfig.yaml | 8 ++++---- e2e/k8s/masterConfig/teraslice.yaml | 4 ++-- e2e/k8s/masterDeployment.yaml | 8 ++++---- e2e/k8s/workerConfig/teraslice.yaml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/e2e/k8s/Dockerfile b/e2e/k8s/Dockerfile index 9b5789fe6cc..d61a1a66e8a 100644 --- a/e2e/k8s/Dockerfile +++ b/e2e/k8s/Dockerfile @@ -13,7 +13,7 @@ COPY .yarn /app/source/.yarn COPY packages /app/source/packages COPY scripts /app/source/scripts COPY types /app/source/types -COPY k8se2e /app/source/k8se2e +COPY e2e/k8s /app/source/e2e/k8s RUN yarn --prod=false --frozen-lockfile \ && yarn build \ diff --git a/e2e/k8s/kindConfig.yaml b/e2e/k8s/kindConfig.yaml index e9d87e52e73..9a344500de0 100644 --- a/e2e/k8s/kindConfig.yaml +++ b/e2e/k8s/kindConfig.yaml @@ -4,7 +4,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - - containerPort: 30200 - hostPort: 9200 - - containerPort: 30678 - hostPort: 5678 + - containerPort: 9200 + hostPort: 49200 + - containerPort: 45678 + hostPort: 45678 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 8f1aaddea3e..2a2a972caa4 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -6,11 +6,11 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:9200" + - "elasticsearch:49200" elasticsearch-next: default: node: - - "http://elasticsearch:9200" + - "http://elasticsearch:49200" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 14433977e66..6a3827dfaec 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -22,7 +22,7 @@ spec: - name: teraslice-master image: teraslice-workspace:k8se2e ports: - - containerPort: 5678 + - containerPort: 45678 volumeMounts: - mountPath: /app/config # defines the directory name: config @@ -47,7 +47,7 @@ spec: app: teraslice nodeType: master ports: - - port: 5678 - targetPort: 5678 - nodePort: 30678 # the external port teraslice can be accessed on + - port: 45678 + targetPort: 45678 + nodePort: 45678 # the external port teraslice can be accessed on type: NodePort diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 6615cc43757..4c0a872bc55 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -6,11 +6,11 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:9200" + - "elasticsearch:49200" elasticsearch-next: default: node: - - "http://elasticsearch:9200" + - "http://elasticsearch:49200" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 From 9e39381b2ef2250c66b0d9e1ad6ff6d79b7e6732 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:21:20 -0700 Subject: [PATCH 013/142] update port in masterDeployment yaml file --- e2e/k8s/masterDeployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 6a3827dfaec..4b5eb01c947 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -49,5 +49,5 @@ spec: ports: - port: 45678 targetPort: 45678 - nodePort: 45678 # the external port teraslice can be accessed on + nodePort: 30678 # the external port teraslice can be accessed on type: NodePort From 1b01319f3d6c5708dbce9aac9f07d9c5ca12f17c Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:22:41 -0700 Subject: [PATCH 014/142] Add checks for k8s test environment --- e2e/test/global.setup.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index bcd66773d55..bf9ddf3e237 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -1,6 +1,7 @@ 'use strict'; const { pDelay } = require('@terascope/utils'); +const { getE2eK8sDir } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); const globalTeardown = require('./global.teardown'); @@ -9,13 +10,17 @@ const signale = require('./signale'); const setupTerasliceConfig = require('./setup-config'); const downloadAssets = require('./download-assets'); const { CONFIG_PATH, ASSETS_PATH } = require('./config'); +const { createKindCluster, destroyKindCluster } = require('../../packages/scripts/src/helpers/scripts.ts');// FIXME: is this right? module.exports = async () => { const teraslice = new TerasliceHarness(); + await teraslice.init();// create TS and ES or OS clients - await teraslice.init(); - - await globalTeardown(teraslice.client); + if (process.env.TEST_PLATFORM === 'kubernetes') { + await destroyKindCluster(); + } else { + await globalTeardown(teraslice.client); // docker compose down and ES or OS teardown + } await teraslice.resetLogs(); process.stdout.write('\n'); @@ -35,7 +40,12 @@ module.exports = async () => { await Promise.all([setupTerasliceConfig(), downloadAssets()]); - await dockerUp(); + if (process.env.TEST_PLATFORM === 'kubernetes') { + const e2eK8sDir = getE2eK8sDir(); + await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); + } else { + await dockerUp(); // create TS master and workers from docker-compose file + } await teraslice.waitForTeraslice(); await pDelay(2000); await teraslice.resetState(); From 1ddf937d93fe8c73936006c3f17cab890384dff7 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:25:36 -0700 Subject: [PATCH 015/142] Add deployKafka, and kindLoadServiceImage functions. Add registerStandardAssets function --- packages/scripts/src/helpers/scripts.ts | 67 ++++++++++++++++++++----- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 45a3b465da3..bee455073ae 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -14,6 +14,7 @@ import { TSCommands, PackageInfo } from './interfaces'; import { getRootDir } from './misc'; import signale from './signale'; import * as config from './config'; +import { getE2eK8sDir } from '../helpers/packages'; const logger = debugLogger('ts-scripts:cmd'); @@ -165,6 +166,12 @@ export async function runJest( signale.debug(`executing: jest ${args.join(' ')}`); } + console.log('######## cwd: ', cwd); + console.log('######## argsMap: ', argsMap); + console.log('######## args: ', args); + console.log('######## env: ', env); + console.log('######## extraArgs: ', extraArgs); + await fork({ cmd: 'jest', cwd, @@ -534,10 +541,10 @@ export async function yarnPublish( } export async function createKindCluster( - k8se2eDir: string, + e2eK8sDir: string, kindConfigFileName: string ): Promise { - const configPath = path.join(k8se2eDir, kindConfigFileName); + const configPath = path.join(e2eK8sDir, kindConfigFileName); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); signale.log(subprocess.stderr); // const test = await execa.command('echo hello'); @@ -576,13 +583,29 @@ export async function loadTerasliceImage(terasliceImage: string): Promise console.log('load teraslice image subprocess: ', subprocess); } +export async function kindLoadServiceImage(serviceName: string): Promise { + console.log('@@@@@ kindLoadServiceImage'); + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + if (serviceName === 'elasticsearch') { + await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); + } else if (serviceName === 'kafka') { + await deployKafka(e2eK8sDir, 'elasticsearchDeployment.yaml'); + } else { + signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); + } +} + export async function createNamespace() { const subprocess = await execa.command('kubectl create namespace ts-dev1'); console.log('namespace subprocess: ', subprocess); } -export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: string) { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); +export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { + console.log('@@@@@ deployElasticsearch'); + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); console.log('esDeploy subprocess: ', subprocess); const elasticsearchReady = await waitForESRunning(240000); @@ -625,32 +648,43 @@ function waitForESRunning(timeoutMs = 120000): Promise { return _waitForESRunning(); } +function deployKafka(e2eK8sDir: string, arg1: string) { + throw new Error('Function not implemented.'); + // const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); + // console.log('esDeploy subprocess: ', subprocess); + + // const elasticsearchReady = await waitForESRunning(240000); + // if (elasticsearchReady) { + // signale.success('Elasticsearch is ready to go'); + // } +} + export async function k8sSetup( - k8se2eDir: string, + e2eK8sDir: string, roleYaml: string, roleBindingYaml: string, priorityClassYaml: string ): Promise { - const subprocess1 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleYaml)}`); - const subprocess2 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleBindingYaml)}`); - const subprocess3 = await execa.command(`kubectl apply -f ${path.join(k8se2eDir, priorityClassYaml)}`); + const subprocess1 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleYaml)}`); + const subprocess2 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleBindingYaml)}`); + const subprocess3 = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, priorityClassYaml)}`); console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); } export async function deployk8sTeraslice( - k8se2eDir: string, + e2eK8sDir: string, masterDeploymentYaml: string ) { /// Creates configmap for terasclice-master - let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(k8se2eDir, 'masterConfig', 'teraslice.yaml')}`); + let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); console.log('masterConfig subprocess: ', subprocess); /// Creates configmap for teraslice-worker - subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(k8se2eDir, 'workerConfig', 'teraslice.yaml')}`); + subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); console.log('workerConfig subprocess: ', subprocess); /// Creates deployment for teraslice - subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); console.log('masterDeploy subprocess: ', subprocess); subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); @@ -714,11 +748,16 @@ export async function registerTestJob() { console.log('registerTestJob subprocess: ', subprocess); } -export async function registerElasticsearch() { +export async function registerElasticsearchAssets() { const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/elasticsearch-assets'); console.log('registerElasticsearch asset subprocess: ', subprocess); } +export async function registerStandardAssets() { + const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/standard-assets'); + console.log('registerStandardAssets subprocess: ', subprocess); +} + export async function startTestJob() { const subprocess = await execa.command('earl tjm start testJob.json'); console.log('Run earl tjm start testJob.json'); @@ -730,8 +769,10 @@ export async function showState() { console.log('showState subprocess: ', subprocess); } +// TODO: utils/src/promises/.ts pDelay already does this function PromiseTimeout(delayms: number) { return new Promise((resolve) => { setTimeout(resolve, delayms); }); } + From 0fbf4b7b7d50d097b0fcebaf595c97e5c8d9810d Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:26:41 -0700 Subject: [PATCH 016/142] remove unneeded K8S E2E variable --- packages/scripts/src/helpers/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 25879be6542..43c108d4390 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -98,8 +98,6 @@ export const DEV_DOCKER_IMAGE = process.env.DEV_DOCKER_IMAGE || undefined; */ export const SKIP_DOCKER_BUILD_IN_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_IN_E2E ?? false); -export const SKIP_DOCKER_BUILD_K8S_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_K8S_E2E ?? false); - export const SKIP_E2E_OUTPUT_LOGS = toBoolean(process.env.SKIP_E2E_OUTPUT_LOGS ?? !isCI); /** @@ -160,3 +158,5 @@ export const SEARCH_TEST_HOST = testHost; // https://github.com/terascope/base-docker-image // This overrides the value in the Dockerfile export const NODE_VERSION = process.env.NODE_VERSION || '14.21.3'; + +export const { TEST_PLATFORM = 'native' } = process.env; // FIXME: test this From 6703b7965447959f98f7ca5c00ab3ec15e11abf1 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:27:20 -0700 Subject: [PATCH 017/142] rename function that gets e2e directory --- packages/scripts/src/helpers/packages.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/scripts/src/helpers/packages.ts b/packages/scripts/src/helpers/packages.ts index 75ae9b384de..b1bca1e2621 100644 --- a/packages/scripts/src/helpers/packages.ts +++ b/packages/scripts/src/helpers/packages.ts @@ -15,7 +15,7 @@ import * as i from './interfaces'; let _packages: i.PackageInfo[] = []; let _e2eDir: string|undefined; -let _k8se2eDir: string|undefined; +let _e2e_k8s_dir: string|undefined; export function getE2EDir(): string|undefined { if (_e2eDir) return _e2eDir; @@ -28,12 +28,12 @@ export function getE2EDir(): string|undefined { return undefined; } -export function getK8SE2EDir(): string|undefined { - if (_k8se2eDir) return _k8se2eDir; +export function getE2eK8sDir(): string|undefined { + if (_e2e_k8s_dir) return _e2e_k8s_dir; - if (fs.existsSync(path.join(misc.getRootDir(), 'k8se2e'))) { - _k8se2eDir = path.join(misc.getRootDir(), 'k8se2e'); - return _k8se2eDir; + if (fs.existsSync(path.join(misc.getRootDir(), 'e2e/k8s'))) { + _e2e_k8s_dir = path.join(misc.getRootDir(), 'e2e/k8s'); + return _e2e_k8s_dir; } return undefined; From 82f01f1e84d3dabd989653065f23bd54f8780b9f Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:51:26 -0700 Subject: [PATCH 018/142] Check test platform to call service in proper env --- .../src/helpers/test-runner/services.ts | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 5e2f67f96f2..77d5980f460 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -10,7 +10,8 @@ import { DockerRunOptions, getContainerInfo, dockerStop, - dockerPull + dockerPull, + kindLoadServiceImage } from '../scripts'; import { TestOptions } from './interfaces'; import { Service } from '../interfaces'; @@ -135,6 +136,8 @@ const services: Readonly>> = { }; export async function pullServices(suite: string, options: TestOptions): Promise { + console.log('@@@ in pull services'); + const launchServices = getServicesForSuite(suite); try { @@ -189,6 +192,7 @@ export async function pullServices(suite: string, options: TestOptions): Promise } export async function ensureServices(suite: string, options: TestOptions): Promise<() => void> { + console.log('@@@ in ensure services'); const launchServices = getServicesForSuite(suite); const promises: Promise<(() => void)>[] = []; @@ -242,7 +246,7 @@ export async function ensureServices(suite: string, options: TestOptions): Promi } export async function ensureKafka(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Kafka); await checkKafka(options, startTime); @@ -250,7 +254,7 @@ export async function ensureKafka(options: TestOptions): Promise<() => void> { } export async function ensureMinio(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Minio); await checkMinio(options, startTime); @@ -258,7 +262,7 @@ export async function ensureMinio(options: TestOptions): Promise<() => void> { } export async function ensureElasticsearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Elasticsearch); await checkElasticsearch(options, startTime); @@ -266,7 +270,7 @@ export async function ensureElasticsearch(options: TestOptions): Promise<() => v } export async function ensureRestrainedElasticsearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.RestrainedElasticsearch); await checkRestrainedElasticsearch(options, startTime); @@ -274,7 +278,7 @@ export async function ensureRestrainedElasticsearch(options: TestOptions): Promi } export async function ensureRestrainedOpensearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.RestrainedOpensearch); await checkRestrainedOpensearch(options, startTime); @@ -282,7 +286,7 @@ export async function ensureRestrainedOpensearch(options: TestOptions): Promise< } export async function ensureOpensearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Opensearch); await checkOpensearch(options, startTime); @@ -290,7 +294,7 @@ export async function ensureOpensearch(options: TestOptions): Promise<() => void } export async function ensureRabbitMQ(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.RabbitMQ); await checkRabbitMQ(options, startTime); @@ -680,6 +684,8 @@ async function checkKafka(options: TestOptions, startTime: number) { } async function startService(options: TestOptions, service: Service): Promise<() => void> { + console.log('in startService: ', service); + let serviceName = service; if (serviceName === 'restrained_elasticsearch') { @@ -693,13 +699,21 @@ async function startService(options: TestOptions, service: Service): Promise<() const version = options[`${serviceName}Version`] as string; if (options.useExistingServices) { signale.warn(`expecting ${service}@${version} to be running (this can be dangerous)...`); - return () => {}; + return () => { }; } signale.pending(`starting ${service}@${version} service...`); await stopService(service); + if (process.env.TEST_PLATFORM === 'kubernetes') { + await stopService(service); // FIXME: does this use docker + console.log(`@@@@@@@ loading ${service} via kind`); + // load via kind + kindLoadServiceImage(service); + return () => { }; + } + console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], version, @@ -707,6 +721,7 @@ async function startService(options: TestOptions, service: Service): Promise<() options.debug || options.trace ); + console.log('@@@@@ fn(): ', fn.toString()); return () => { try { fn(); @@ -718,4 +733,5 @@ async function startService(options: TestOptions, service: Service): Promise<() ); } }; + } From 70aef331702c9fd041be3e88e7088c8f6fa7b92b Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:55:09 -0700 Subject: [PATCH 019/142] check test platform before starting docker or kind --- packages/scripts/src/cmds/test.ts | 2 +- .../scripts/src/helpers/test-runner/index.ts | 290 +++++++++--------- .../src/helpers/test-runner/interfaces.ts | 2 +- 3 files changed, 149 insertions(+), 145 deletions(-) diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index 9f3c24c7f75..435932ae56b 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -26,7 +26,7 @@ type Options = { 'node-version': string; 'use-existing-services': boolean; packages?: PackageInfo[]; - 'ignore-mount': boolean + 'ignore-mount': boolean; }; const jestArgs = getExtraArgs(); diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 9eb6aa55255..47c342b3d2d 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -11,21 +11,12 @@ import { PackageInfo } from '../interfaces'; import { TestOptions } from './interfaces'; import { createKindCluster, - destroyKindCluster, - loadTerasliceImage, - createNamespace, - deployElasticSearch, - k8sSetup, - deployk8sTeraslice, runJest, dockerTag, isKindInstalled, isKubectlInstalled, - registerTestJob, - startTestJob, - showState, - registerElasticsearch, - setAlias + createNamespace, + k8sSetup, } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -34,16 +25,12 @@ import { } from './utils'; import signale from '../signale'; import { - getE2EDir, readPackageInfo, listPackages, getK8SE2EDir + getE2EDir, readPackageInfo, listPackages, getE2eK8sDir } from '../packages'; import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; import { TestTracker } from './tracker'; -import { - MAX_PROJECTS_PER_BATCH, - SKIP_DOCKER_BUILD_IN_E2E, - SKIP_DOCKER_BUILD_K8S_E2E -} from '../config'; +import { MAX_PROJECTS_PER_BATCH, SKIP_DOCKER_BUILD_IN_E2E } from '../config'; const logger = debugLogger('ts-scripts:cmd:test'); @@ -85,11 +72,6 @@ async function _runTests( return; } - if (options.suite?.includes('k8se2e')) { - await runk8sE2ETest(options, tracker); - return; - } - const filtered = filterBySuite(pkgInfos, options); if (!filtered.length) { signale.warn('No tests found.'); @@ -216,6 +198,7 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { + console.log('options: ', options); tracker.expected++; const suite = 'e2e'; @@ -226,13 +209,34 @@ async function runE2ETest( throw new Error('Missing e2e test directory'); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const kindInstalled = await isKindInstalled(); + if (!kindInstalled) { + signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); + process.exit(1); + } + + const kubectlInstalled = await isKubectlInstalled(); + if (!kubectlInstalled) { + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + process.exit(1); + } + // TODO: pass kind config file in as a variable + await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); + await createNamespace(); + await k8sSetup(e2eK8sDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); + } + const rootInfo = getRootInfo(); - // const e2eImage = `${rootInfo.name}:e2e-nodev${options.nodeVersion}`; const e2eImage = `${rootInfo.name}:e2e`; if (isCI) { // pull the services first in CI - await pullServices(suite, options); + await pullServices(suite, options); // FIXME: if in k8s run different function } try { @@ -255,7 +259,7 @@ async function runE2ETest( try { tracker.addCleanup( 'e2e:services', - await ensureServices(suite, options) + await ensureServices(suite, options) // FIXME: if in k8s run different function ); } catch (err) { tracker.addError(err); @@ -326,121 +330,121 @@ function printAndGetEnv(suite: string, options: TestOptions) { return env; } -async function runk8sE2ETest( - options: TestOptions, tracker: TestTracker -): Promise { - console.log('options: ', options); - tracker.expected++; - - const k8se2eDir = getK8SE2EDir(); - if (!k8se2eDir) { - throw new Error('Missing k8se2e test directory'); - } - - const kindInstalled = await isKindInstalled(); - if (!kindInstalled) { - signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); - process.exit(1); - } - - const kubectlInstalled = await isKubectlInstalled(); - if (!kubectlInstalled) { - signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); - process.exit(1); - } - // TODO: pass kind config file in as a variable - await createKindCluster(k8se2eDir, 'kindConfig.yaml'); - - const suite = 'k8se2e'; - let startedTest = false; - - const rootInfo = getRootInfo(); - const k8se2eImage = `${rootInfo.name}:k8se2e`; - - // if (isCI) { - // // pull the services first in CI - // await pullServices(suite, options); - // } - - try { - if (SKIP_DOCKER_BUILD_K8S_E2E) { - const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; - await dockerTag(devImage, k8se2eImage); - await loadTerasliceImage(k8se2eImage); - } else { - const publishOptions: PublishOptions = { - dryRun: true, - nodeVersion: options.nodeVersion, - type: PublishType.Dev - }; - const devImage = await buildDevDockerImage(publishOptions); - await dockerTag(devImage, k8se2eImage); - await loadTerasliceImage(k8se2eImage); - } - } catch (err) { - tracker.addError(err); - } - - // TODO: add tracker - await createNamespace(); - await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); - await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); - await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); - await showState(); - - await setAlias(); - await registerElasticsearch(); - await registerTestJob(); - await startTestJob(); - - // if (!tracker.hasErrors()) { - // const timeLabel = `test suite "${suite}"`; - // signale.time(timeLabel); - // startedTest = true; - - // const env = printAndGetEnv(suite, options); - - // tracker.started++; - // try { - // await runJest( - // k8se2eDir, - // getArgs(options), - // env, - // options.jestArgs, - // options.debug - // ); - // tracker.ended++; - // } catch (err) { - // tracker.ended++; - // tracker.addError(err.message); - // } - - // signale.timeEnd(timeLabel); - // } - - // if (!startedTest) return; - - // if (!options.keepOpen) { - // try { - // await logE2E(k8se2eDir, tracker.hasErrors()); - // } catch (err) { - // signale.error( - // new TSError(err, { - // reason: `Writing the "${suite}" logs failed`, - // }) - // ); - // } - // } - - // if (tracker.hasErrors()) { - // tracker.addCleanup('e2e:teardown', async () => { - // options.keepOpen = false; - // await globalTeardown(options, [{ - // name: suite, - // dir: k8se2eDir, - // suite, - // }]); - // }); - // } - // await destroyKindCluster(); -} +// async function runk8sE2ETest( +// options: TestOptions, tracker: TestTracker +// ): Promise { +// console.log('options: ', options); +// tracker.expected++; + +// const k8se2eDir = getK8SE2EDir(); +// if (!k8se2eDir) { +// throw new Error('Missing k8se2e test directory'); +// } + +// const kindInstalled = await isKindInstalled(); +// if (!kindInstalled) { +// signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); +// process.exit(1); +// } + +// const kubectlInstalled = await isKubectlInstalled(); +// if (!kubectlInstalled) { +// signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); +// process.exit(1); +// } +// // TODO: pass kind config file in as a variable +// await createKindCluster(k8se2eDir, 'kindConfig.yaml'); + +// const suite = 'k8se2e'; +// let startedTest = false; + +// const rootInfo = getRootInfo(); +// const k8se2eImage = `${rootInfo.name}:k8se2e`; + +// // if (isCI) { +// // // pull the services first in CI +// // await pullServices(suite, options); +// // } + +// try { +// if (SKIP_DOCKER_BUILD_IN_E2E) { +// const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; +// await dockerTag(devImage, k8se2eImage); +// await loadTerasliceImage(k8se2eImage); +// } else { +// const publishOptions: PublishOptions = { +// dryRun: true, +// nodeVersion: options.nodeVersion, +// type: PublishType.Dev +// }; +// const devImage = await buildDevDockerImage(publishOptions); +// await dockerTag(devImage, k8se2eImage); +// await loadTerasliceImage(k8se2eImage); +// } +// } catch (err) { +// tracker.addError(err); +// } + +// TODO: add tracker +// await createNamespace(); +// await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); +// await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); +// await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); +// await showState(); + +// await setAlias(); +// await registerElasticsearchAssets(); +// await registerStandardAssets(); +// await registerTestJob(); +// await startTestJob(); + +// if (!tracker.hasErrors()) { +// const timeLabel = `test suite "${suite}"`; +// signale.time(timeLabel); +// startedTest = true; + +// const env = printAndGetEnv(suite, options); + +// tracker.started++; +// try { +// await runJest( +// k8se2eDir, +// getArgs(options), +// env, +// options.jestArgs, +// options.debug +// ); +// tracker.ended++; +// } catch (err) { +// tracker.ended++; +// tracker.addError(err.message); +// } + +// signale.timeEnd(timeLabel); +// } + +// if (!startedTest) return; + +// if (!options.keepOpen) { +// try { +// await logE2E(k8se2eDir, tracker.hasErrors()); +// } catch (err) { +// signale.error( +// new TSError(err, { +// reason: `Writing the "${suite}" logs failed`, +// }) +// ); +// } +// } + +// if (tracker.hasErrors()) { +// tracker.addCleanup('e2e:teardown', async () => { +// options.keepOpen = false; +// await globalTeardown(options, [{ +// name: suite, +// dir: k8se2eDir, +// suite, +// }]); +// }); +// } +// } diff --git a/packages/scripts/src/helpers/test-runner/interfaces.ts b/packages/scripts/src/helpers/test-runner/interfaces.ts index 5ce208e7a3e..fdb40b8a7af 100644 --- a/packages/scripts/src/helpers/test-runner/interfaces.ts +++ b/packages/scripts/src/helpers/test-runner/interfaces.ts @@ -19,7 +19,7 @@ export type TestOptions = { opensearchVersion: string; nodeVersion: string; jestArgs?: string[]; - ignoreMount: boolean + ignoreMount: boolean; }; export type GroupedPackages = { From 70702ca9e83e7b0821fde8969356c36aa6a8cd71 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:55:33 -0700 Subject: [PATCH 020/142] create script to run k8s e2e tests --- e2e/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/package.json b/e2e/package.json index e4135d21a58..fd16ea10a24 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,6 +25,7 @@ "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", + "test:k8s": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", From d7187cb8e990ef64d769b5152ba8133a0d7a4fcd Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:08:09 -0700 Subject: [PATCH 021/142] Add kafka to teraslice.yaml files --- e2e/k8s/masterConfig/teraslice.yaml | 9 +++++++-- e2e/k8s/workerConfig/teraslice.yaml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 2a2a972caa4..22458f4958c 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -6,11 +6,16 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:49200" + - "elasticsearch:9200" elasticsearch-next: default: node: - - "http://elasticsearch:49200" + - "http://elasticsearch:9200" + kafka: + default: + brokers: + - "kafka:9092" + teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 4c0a872bc55..647d1c05870 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -6,11 +6,16 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:49200" + - "elasticsearch:9200" elasticsearch-next: default: node: - - "http://elasticsearch:49200" + - "http://elasticsearch:9200" + kafka: + default: + brokers: + - "kafka:9092" + teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 From a7ac89604686135f65deee41d255e12b04c104a0 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:08:31 -0700 Subject: [PATCH 022/142] Add kafkaDeployment.yaml file --- e2e/k8s/kafkaDeployment.yaml | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 e2e/k8s/kafkaDeployment.yaml diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml new file mode 100644 index 00000000000..37197548860 --- /dev/null +++ b/e2e/k8s/kafkaDeployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka + labels: + app: kafka + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: kafka + nodeType: master + template: + metadata: + labels: + app: kafka + nodeType: master + spec: + containers: + - name: kafka + image: kafka:3.3 + ports: + - containerPort: 9092 + env: + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: single-node +--- +kind: Service +apiVersion: v1 +metadata: + name: kafka + labels: + app: kafka +spec: + selector: + app: kafka + nodeType: master + ports: + - port: 9092 + targetPort: 9092 + nodePort: 30092 # the external port teraslice can be accessed on + type: NodePort From c849388607934b46de9e8dfda54c4171b084fe08 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:09:09 -0700 Subject: [PATCH 023/142] finally get ports forwarded correctly --- e2e/k8s/kindConfig.yaml | 6 ++++-- e2e/k8s/masterDeployment.yaml | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/e2e/k8s/kindConfig.yaml b/e2e/k8s/kindConfig.yaml index 9a344500de0..54fd983a1f4 100644 --- a/e2e/k8s/kindConfig.yaml +++ b/e2e/k8s/kindConfig.yaml @@ -4,7 +4,9 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - - containerPort: 9200 + - containerPort: 30200 hostPort: 49200 - - containerPort: 45678 + - containerPort: 30678 hostPort: 45678 + - containerPort: 30092 + hostPort: 49092 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 4b5eb01c947..14433977e66 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -22,7 +22,7 @@ spec: - name: teraslice-master image: teraslice-workspace:k8se2e ports: - - containerPort: 45678 + - containerPort: 5678 volumeMounts: - mountPath: /app/config # defines the directory name: config @@ -47,7 +47,7 @@ spec: app: teraslice nodeType: master ports: - - port: 45678 - targetPort: 45678 + - port: 5678 + targetPort: 5678 nodePort: 30678 # the external port teraslice can be accessed on type: NodePort From 689b9cfec6528499e1cd41e1e227b9d3f4135491 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:09:30 -0700 Subject: [PATCH 024/142] remove unneeeded dockerfile --- e2e/k8s/Dockerfile | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 e2e/k8s/Dockerfile diff --git a/e2e/k8s/Dockerfile b/e2e/k8s/Dockerfile deleted file mode 100644 index d61a1a66e8a..00000000000 --- a/e2e/k8s/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# NODE_VERSION is set by default in the config.ts, the following value will only -# be used if you build images by default with docker build -ARG NODE_VERSION=14.21.3 -FROM terascope/node-base:${NODE_VERSION} - -ENV NODE_ENV production - -ENV YARN_SETUP_ARGS "--prod=false --silent --frozen-lockfile" - -COPY package.json yarn.lock tsconfig.json .yarnrc /app/source/ -COPY .yarnclean.ci /app/source/.yarnclean -COPY .yarn /app/source/.yarn -COPY packages /app/source/packages -COPY scripts /app/source/scripts -COPY types /app/source/types -COPY e2e/k8s /app/source/e2e/k8s - -RUN yarn --prod=false --frozen-lockfile \ - && yarn build \ - && yarn \ - --prod=true \ - --silent \ - --frozen-lockfile \ - --skip-integrity-check \ - --ignore-scripts \ - && yarn cache clean - - -COPY service.js /app/source/ - -# verify node-rdkafka is installed right -RUN node -e "require('node-rdkafka')" - -# verify teraslice is installed right -RUN node -e "require('teraslice')" - -EXPOSE 5678 - -# set up the volumes -VOLUME /app/config /app/logs /app/assets -ENV TERAFOUNDATION_CONFIG /app/config/teraslice.yaml - -CMD ["node", "service.js"] From 535c531d4f01d241b063f71059b07f414c1c0803 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:12:24 -0700 Subject: [PATCH 025/142] wip: update global.setup to be k8s aware --- e2e/test/global.setup.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index bf9ddf3e237..c61c698fb75 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -1,7 +1,7 @@ 'use strict'; const { pDelay } = require('@terascope/utils'); -const { getE2eK8sDir } = require('@terascope/scripts'); +const { getE2eK8sDir, deployK8sTeraslice, destroyKindCluster } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); const globalTeardown = require('./global.teardown'); @@ -10,7 +10,6 @@ const signale = require('./signale'); const setupTerasliceConfig = require('./setup-config'); const downloadAssets = require('./download-assets'); const { CONFIG_PATH, ASSETS_PATH } = require('./config'); -const { createKindCluster, destroyKindCluster } = require('../../packages/scripts/src/helpers/scripts.ts');// FIXME: is this right? module.exports = async () => { const teraslice = new TerasliceHarness(); @@ -42,7 +41,7 @@ module.exports = async () => { if (process.env.TEST_PLATFORM === 'kubernetes') { const e2eK8sDir = getE2eK8sDir(); - await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); + await deployK8sTeraslice(e2eK8sDir, 'kindConfig.yaml'); } else { await dockerUp(); // create TS master and workers from docker-compose file } From 97855718b083595d71ecdedfd77cf2cd6124d117 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:14:18 -0700 Subject: [PATCH 026/142] update ES port, rename variables --- packages/scripts/src/helpers/scripts.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index bee455073ae..bfccade00bd 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -592,7 +592,7 @@ export async function kindLoadServiceImage(serviceName: string): Promise { if (serviceName === 'elasticsearch') { await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); } else if (serviceName === 'kafka') { - await deployKafka(e2eK8sDir, 'elasticsearchDeployment.yaml'); + await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml'); } else { signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); } @@ -606,7 +606,7 @@ export async function createNamespace() { export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { console.log('@@@@@ deployElasticsearch'); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - console.log('esDeploy subprocess: ', subprocess); + console.log('deployElasticserach subprocess: ', subprocess); const elasticsearchReady = await waitForESRunning(240000); if (elasticsearchReady) { @@ -624,7 +624,7 @@ function waitForESRunning(timeoutMs = 120000): Promise { let elasticsearchRunning = false; try { - const ESResponse = await execa.command('curl localhost:9200'); + const ESResponse = await execa.command('curl http://localhost:49200'); if (ESResponse.stdout) { const jsonData = JSON.parse(ESResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); @@ -648,10 +648,9 @@ function waitForESRunning(timeoutMs = 120000): Promise { return _waitForESRunning(); } -function deployKafka(e2eK8sDir: string, arg1: string) { - throw new Error('Function not implemented.'); - // const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - // console.log('esDeploy subprocess: ', subprocess); +export async function deployKafka(e2eK8sDir: string, kafkaYaml: string) { + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaYaml)}`); + console.log('deployKafka subprocess: ', subprocess); // const elasticsearchReady = await waitForESRunning(240000); // if (elasticsearchReady) { @@ -671,7 +670,7 @@ export async function k8sSetup( console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); } -export async function deployk8sTeraslice( +export async function deployK8sTeraslice( e2eK8sDir: string, masterDeploymentYaml: string ) { From 4612a4acff0d66d56717433934a6c8989a822dd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 00:04:31 +0000 Subject: [PATCH 027/142] Bump fs-extra and @types/fs-extra Bumps [fs-extra](https://github.com/jprichardson/node-fs-extra) and [@types/fs-extra](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/fs-extra). These dependencies needed to be updated together. Updates `fs-extra` from 10.1.0 to 11.1.1 - [Changelog](https://github.com/jprichardson/node-fs-extra/blob/master/CHANGELOG.md) - [Commits](https://github.com/jprichardson/node-fs-extra/compare/10.1.0...11.1.1) Updates `@types/fs-extra` from 9.0.13 to 11.0.2 - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/fs-extra) --- updated-dependencies: - dependency-name: fs-extra dependency-type: direct:production update-type: version-update:semver-major - dependency-name: "@types/fs-extra" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- e2e/package.json | 2 +- package.json | 2 +- packages/job-components/package.json | 2 +- packages/scripts/package.json | 2 +- packages/teraslice-cli/package.json | 2 +- packages/teraslice-test-harness/package.json | 2 +- packages/teraslice/package.json | 2 +- yarn.lock | 25 ++++++++++---------- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index fd16ea10a24..df97dc8b293 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -39,7 +39,7 @@ "devDependencies": { "bunyan": "^1.8.15", "elasticsearch-store": "^0.72.0", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.1", "ms": "^2.1.3", "nanoid": "^3.3.4", "semver": "^7.3.8", diff --git a/package.json b/package.json index 9db7ce63b2d..02eca751642 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "@types/bluebird": "^3.5.38", "@types/convict": "^6.1.2", "@types/elasticsearch": "^5.0.40", - "@types/fs-extra": "^9.0.9", + "@types/fs-extra": "^11.0.2", "@types/jest": "^29.5.2", "@types/lodash": "^4.14.195", "@types/node": "^18.14.2", diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 462073c1119..7219e24107e 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -40,7 +40,7 @@ }, "devDependencies": { "benchmark": "^2.1.4", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.1", "jest-fixtures": "^0.6.0" }, "engines": { diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 53ed002be6e..129e53b5335 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -34,7 +34,7 @@ "@terascope/utils": "^0.51.0", "codecov": "^3.8.3", "execa": "^5.1.0", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.1", "globby": "^11.0.4", "got": "^11.8.3", "ip": "^1.1.8", diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index e411a01b4f4..e61ef34bddb 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -44,7 +44,7 @@ "esbuild": "^0.15.9", "events": "^3.3.0", "execa": "^5.1.0", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.1", "glob": "^8.0.3", "glob-promise": "5.0.0", "got": "^11.8.3", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index e84509947f3..f8b9efc9e54 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -33,7 +33,7 @@ "@terascope/fetch-github-release": "^0.8.7", "@terascope/teraslice-op-test-harness": "^1.24.1", "decompress": "^4.2.1", - "fs-extra": "^10.1.0" + "fs-extra": "^11.1.1" }, "devDependencies": { "@terascope/job-components": "^0.64.0" diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index fd08076f9a8..bdbd2b0d29e 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -53,7 +53,7 @@ "easy-table": "^1.2.0", "event-loop-stats": "^1.4.1", "express": "^4.18.2", - "fs-extra": "^10.1.0", + "fs-extra": "^11.1.1", "gc-stats": "^1.4.0", "got": "^11.8.3", "ip": "^1.1.8", diff --git a/yarn.lock b/yarn.lock index 5a0a1230ece..4933a242dac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2601,11 +2601,12 @@ resolved "https://registry.yarnpkg.com/@types/expect/-/expect-1.20.4.tgz#8288e51737bf7e3ab5d7c77bfa695883745264e5" integrity sha512-Q5Vn3yjTDyCMV50TB6VRIbQNxSE4OmZR86VSbGaNpfUolm0iePBB4KdEEHmxoY5sT2+2DIvXW0rvMDP2nHZ4Mg== -"@types/fs-extra@^9.0.9": - version "9.0.13" - resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-9.0.13.tgz#7594fbae04fe7f1918ce8b3d213f74ff44ac1f45" - integrity sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA== +"@types/fs-extra@^11.0.2": + version "11.0.2" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.2.tgz#23dc1ed7b2eba8ccd75568ac34e7a4e48aa2d087" + integrity sha512-c0hrgAOVYr21EX8J0jBMXGLMgJqVf/v6yxi0dLaJboW9aQPh16Id+z6w2Tx1hm+piJOLv8xPfVKZCLfjPw/IMQ== dependencies: + "@types/jsonfile" "*" "@types/node" "*" "@types/geojson@7946.0.8": @@ -2729,6 +2730,13 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/jsonfile@*": + version "6.1.2" + resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.2.tgz#d3b8a3536c5bb272ebee0f784180e456b7691c8f" + integrity sha512-8t92P+oeW4d/CRQfJaSqEwXujrhH4OEeHRjGU3v1Q8mUS8GPF3yiX26sw4svv6faL2HfBtGTe2xWIoVgN3dy9w== + dependencies: + "@types/node" "*" + "@types/keyv@^3.1.4": version "3.1.4" resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6" @@ -6203,15 +6211,6 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@^10.1.0: - version "10.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" - integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs-extra@^11.1.1: version "11.1.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.1.tgz#da69f7c39f3b002378b0954bb6ae7efdc0876e2d" From 4117a26925940534b0c3cf2049f742688fd89f57 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 13:40:19 -0700 Subject: [PATCH 028/142] Update images for kafka and master deployments --- e2e/k8s/kafkaDeployment.yaml | 8 ++------ e2e/k8s/masterDeployment.yaml | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index 37197548860..a1ac36cccf0 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -19,14 +19,10 @@ spec: spec: containers: - name: kafka - image: kafka:3.3 + image: sotojn6/kafka:3.3 ports: - containerPort: 9092 - env: - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - - name: discovery.type - value: single-node + --- kind: Service apiVersion: v1 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 14433977e66..20b31f7c188 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: teraslice-master - image: teraslice-workspace:k8se2e + image: teraslice-workspace:e2e ports: - containerPort: 5678 volumeMounts: From 67d70e89e97d9ed442936e034c532d868948755d Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:45:45 -0700 Subject: [PATCH 029/142] fix kubernetes image tag --- e2e/k8s/masterConfig/teraslice.yaml | 3 +-- e2e/k8s/workerConfig/teraslice.yaml | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 22458f4958c..924d4bc6dae 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -15,7 +15,6 @@ terafoundation: default: brokers: - "kafka:9092" - teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 @@ -25,7 +24,7 @@ teraslice: cluster_manager_type: "kubernetes" master: true master_hostname: "127.0.0.1" - kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image: "teraslice-workspace:e2e" kubernetes_image_pull_secrets: - "docker-tera1-secret" kubernetes_namespace: "ts-dev1" diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 647d1c05870..af04736d632 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -11,11 +11,10 @@ terafoundation: default: node: - "http://elasticsearch:9200" - kafka: + kafka: default: brokers: - "kafka:9092" - teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 @@ -25,7 +24,7 @@ teraslice: cluster_manager_type: "kubernetes" master: false master_hostname: "teraslice-master" - kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image: "teraslice-workspace:e2e" kubernetes_namespace: "ts-dev1" kubernetes_overrides_enabled: true kubernetes_priority_class_name: 'high-priority' From 36723869f52e23dff10c376d9faa93d7024eab5c Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:46:21 -0700 Subject: [PATCH 030/142] add env variables and change image used --- e2e/k8s/kafkaDeployment.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index a1ac36cccf0..d2f7626ccb7 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -19,10 +19,14 @@ spec: spec: containers: - name: kafka - image: sotojn6/kafka:3.3 + image: terascope/kafka-zookeeper:v1.1.0 + env: + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://kafka:9092 + - name: host.name + value: kafka ports: - containerPort: 9092 - --- kind: Service apiVersion: v1 From edd611dde27deb29de5630acc0e1ee3af4814f17 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:46:37 -0700 Subject: [PATCH 031/142] change cluster name to k8se2e --- e2e/k8s/masterDeployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 20b31f7c188..b5a521560de 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: teraslice nodeType: master - clusterName: ts-dev1 + clusterName: k8se2e spec: containers: - name: teraslice-master From 36c2b4297453cd0afd4c20dbdf7620be6c4aa733 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:49:50 -0700 Subject: [PATCH 032/142] changes to test setup just to get things running --- e2e/test/global.setup.js | 24 +++++++++++++++++------- e2e/test/global.teardown.js | 13 +++++++++++-- e2e/test/teraslice-harness.js | 28 +++++++++++++++++++++------- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index c61c698fb75..84fafa6d6b6 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -1,7 +1,9 @@ 'use strict'; const { pDelay } = require('@terascope/utils'); -const { getE2eK8sDir, deployK8sTeraslice, destroyKindCluster } = require('@terascope/scripts'); +const { + getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets +} = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); const globalTeardown = require('./global.teardown'); @@ -15,10 +17,8 @@ module.exports = async () => { const teraslice = new TerasliceHarness(); await teraslice.init();// create TS and ES or OS clients - if (process.env.TEST_PLATFORM === 'kubernetes') { - await destroyKindCluster(); - } else { - await globalTeardown(teraslice.client); // docker compose down and ES or OS teardown + if (process.env.TEST_PLATFORM === 'native') { + await globalTeardown(teraslice.client); // docker compose down and ES teardown FIXME for k8s } await teraslice.resetLogs(); @@ -37,18 +37,28 @@ module.exports = async () => { fse.ensureDir(CONFIG_PATH), ]); + // FIXME: config diff between k8s and native await Promise.all([setupTerasliceConfig(), downloadAssets()]); + // await pDelay(10000); + if (process.env.TEST_PLATFORM === 'kubernetes') { const e2eK8sDir = getE2eK8sDir(); - await deployK8sTeraslice(e2eK8sDir, 'kindConfig.yaml'); + await deployK8sTeraslice(e2eK8sDir, 'masterDeployment.yaml'); } else { - await dockerUp(); // create TS master and workers from docker-compose file + await dockerUp(); } + await teraslice.waitForTeraslice(); await pDelay(2000); await teraslice.resetState(); + if (process.env.TEST_PLATFORM === 'kubernetes') { + await setAlias(); + await deployElasticsearchAssets(); + await deployStandardAssets(); + } + try { await teraslice.generateTestData(); } catch (err) { diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index e59a84a9462..806d293d99c 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -1,6 +1,7 @@ 'use strict'; const { ElasticsearchTestHelpers } = require('elasticsearch-store'); +const { k8sTearDown } = require('@terascope/scripts'); const fse = require('fs-extra'); const { KEEP_OPEN, CONFIG_PATH, ASSETS_PATH } = require('./config'); const { tearDown, TEST_INDEX_PREFIX } = require('./docker-helpers'); @@ -14,16 +15,24 @@ async function getClient(client) { } async function globalTeardown(testClient) { - const client = await getClient(testClient); + console.log('@@@@@@@@ testClient: ', testClient); + console.log('@@@@@@@@ KEEP_OPEN?: ', KEEP_OPEN); if (KEEP_OPEN) { return; } + console.log('@@@@@@@@ past KEEP_OPEN'); + + const client = await getClient(testClient); const errors = []; try { - await tearDown(); + if (process.env.TEST_PATTERN === 'kubernetes') { + await k8sTearDown(); + } else { + await tearDown(); + } } catch (err) { errors.push(err); } diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 36a96061aaf..37610617b0f 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -9,6 +9,7 @@ const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store' const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); const fse = require('fs-extra'); +const { resetTeraslice } = require('@terascope/scripts'); const { TEST_HOST, HOST_IP, SPEC_INDEX_PREFIX, DEFAULT_NODES, newId, DEFAULT_WORKERS, GENERATE_ONLY, @@ -23,7 +24,7 @@ const generateOnly = GENERATE_ONLY ? parseInt(GENERATE_ONLY, 10) : null; module.exports = class TerasliceHarness { async init() { - const { client } = await createClient({ node: TEST_HOST }); + const { client } = await createClient({ node: TEST_HOST });// create ES or OS db this.client = client; this.teraslice = new TerasliceClient({ host: `http://${HOST_IP}:45678`, @@ -113,11 +114,16 @@ module.exports = class TerasliceHarness { ); })(), (async () => { - const count = Object.keys(state).length; - if (count !== DEFAULT_NODES) { - signale.warn(`resetting cluster state of ${count} nodes`); - await scaleWorkers(); - await this.forWorkers(); + if (process.env.TEST_PLATFORM === 'kubernetes') { + // FIXME: scale workers in k8s???????? + } else { + console.log('@@@@@@@ else'); + const count = Object.keys(state).length; + if (count !== DEFAULT_NODES) { + signale.warn(`resetting cluster state of ${count} nodes`); + await scaleWorkers(); + await this.forWorkers(); + } } })() ]); @@ -350,6 +356,9 @@ module.exports = class TerasliceHarness { return _waitForClusterState(); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + if (nodes === 0) return nodes; + } if (nodes >= DEFAULT_NODES) return nodes; return _waitForClusterState(); }; @@ -390,7 +399,12 @@ module.exports = class TerasliceHarness { signale.pending('Waiting for Teraslice...'); const nodes = await this.waitForClusterState(); - signale.success(`Teraslice is ready to go with ${nodes} nodes`, getElapsed(startTime)); + + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.success('Teraslice is ready to go', getElapsed(startTime)); + } else { + signale.success(`Teraslice is ready to go with ${nodes} nodes`, getElapsed(startTime)); + } } async postJob(jobSpec) { From 81b581348e221739cbd8f88d421df0a4d6fbe621 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:52:27 -0700 Subject: [PATCH 033/142] new functions: kindStopService, takeDownTeraslice --- packages/scripts/src/helpers/scripts.ts | 163 ++++++++++++++++++------ 1 file changed, 122 insertions(+), 41 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index bfccade00bd..2242850157c 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -547,8 +547,6 @@ export async function createKindCluster( const configPath = path.join(e2eK8sDir, kindConfigFileName); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); signale.log(subprocess.stderr); - // const test = await execa.command('echo hello'); - // console.log('test: ', test); } export async function destroyKindCluster(): Promise { @@ -583,6 +581,25 @@ export async function loadTerasliceImage(terasliceImage: string): Promise console.log('load teraslice image subprocess: ', subprocess); } +export async function kindStopService(serviceName: string): Promise { + console.log('@@@@@ kindStopService'); + // const e2eK8sDir = getE2eK8sDir(); + // if (!e2eK8sDir) { + // throw new Error('Missing k8s e2e test directory'); + // } + // if (serviceName === 'elasticsearch') { + // const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'elasticsearchDeployment.yaml')}`); + // console.log('stopElasticsearch subprocess: ', subprocess); + // } else if (serviceName === 'kafka') { + // let subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaDeployment.yaml')}`); + // console.log('stopKafkaDeployment subprocess: ', subprocess); + // // subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaService.yaml')}`); + // // console.log('stopKafkaService subprocess: ', subprocess); + // } else { + // signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); + // } +} + export async function kindLoadServiceImage(serviceName: string): Promise { console.log('@@@@@ kindLoadServiceImage'); const e2eK8sDir = getE2eK8sDir(); @@ -598,15 +615,15 @@ export async function kindLoadServiceImage(serviceName: string): Promise { } } -export async function createNamespace() { - const subprocess = await execa.command('kubectl create namespace ts-dev1'); +export async function createNamespace(e2eK8sDir: string, namespaceYaml: string) { + const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); console.log('namespace subprocess: ', subprocess); } export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { console.log('@@@@@ deployElasticsearch'); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - console.log('deployElasticserach subprocess: ', subprocess); + console.log('deployElasticsearch subprocess: ', subprocess); const elasticsearchReady = await waitForESRunning(240000); if (elasticsearchReady) { @@ -648,16 +665,56 @@ function waitForESRunning(timeoutMs = 120000): Promise { return _waitForESRunning(); } -export async function deployKafka(e2eK8sDir: string, kafkaYaml: string) { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaYaml)}`); +export async function deployKafka( + e2eK8sDir: string, + kafkaDeploymentYaml: string, +) { + let subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); console.log('deployKafka subprocess: ', subprocess); - // const elasticsearchReady = await waitForESRunning(240000); - // if (elasticsearchReady) { - // signale.success('Elasticsearch is ready to go'); + // subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaServiceYaml)}`); + // console.log('deployKafkaService subprocess: ', subprocess); + + // const kafkaReady = await waitForKafkaRunning(240000); + // if (kafkaReady) { + signale.success('Kafka *might* be ready to go'); // } } +function waitForKafkaRunning(timeoutMs = 120000): Promise { + const endAt = Date.now() + timeoutMs; + + const _waitForKafkaRunning = async (): Promise => { + if (Date.now() > endAt) { + throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); + } + + let elasticsearchRunning = false; + try { + const kafkaResponse = await execa.command('kubectl -n ts-dev1 get po '); + if (kafkaResponse.stdout) { + const jsonData = JSON.parse(kafkaResponse.stdout); + console.log(`response: ${JSON.stringify(jsonData)}`); + if (jsonData.tagline === 'You Know, for Search') { + console.log('jsonData.tagline === You Know, for Search'); + elasticsearchRunning = true; + } + } + } catch (err) { + await PromiseTimeout(3000); + return _waitForKafkaRunning(); + } + + if (elasticsearchRunning) { + return true; + } + await PromiseTimeout(3000); + return _waitForKafkaRunning(); + }; + + return _waitForKafkaRunning(); +} + export async function k8sSetup( e2eK8sDir: string, roleYaml: string, @@ -674,25 +731,29 @@ export async function deployK8sTeraslice( e2eK8sDir: string, masterDeploymentYaml: string ) { - /// Creates configmap for terasclice-master - let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); - console.log('masterConfig subprocess: ', subprocess); + try { + /// Creates configmap for terasclice-master + let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); + console.log('masterConfig subprocess: ', subprocess); - /// Creates configmap for teraslice-worker - subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); - console.log('workerConfig subprocess: ', subprocess); + /// Creates configmap for teraslice-worker + subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); + console.log('workerConfig subprocess: ', subprocess); - /// Creates deployment for teraslice - subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); - console.log('masterDeploy subprocess: ', subprocess); + /// Creates deployment for teraslice + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); + console.log('masterDeploy subprocess: ', subprocess); - subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); - const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); - console.log('podName: ', podName); + subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); + const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); + console.log('podName: ', podName); - const terasliceReady = await waitForTerasliceRunning(240000); - if (terasliceReady) { - signale.success('Teraslice is ready to go'); + const terasliceReady = await waitForTerasliceRunning(240000); + if (terasliceReady) { + signale.success('Teraslice is ready to go'); + } + } catch (error) { + console.log('@@@@@@@@ deployK8sTeraslice error: ', error); } } @@ -706,7 +767,7 @@ function waitForTerasliceRunning(timeoutMs = 120000): Promise { let terasliceRunning = false; try { - const TSMasterResponse = await execa.command('curl localhost:5678'); + const TSMasterResponse = await execa.command('curl localhost:45678'); if (TSMasterResponse.stdout) { const jsonData = JSON.parse(TSMasterResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); @@ -732,29 +793,29 @@ function waitForTerasliceRunning(timeoutMs = 120000): Promise { } export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); - const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); + const subprocess1 = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); + const subprocess2 = await execa.command('earl aliases add k8se2e http://localhost:45678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function deployESAsset() { - const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --blocking --bundle terascope/elasticsearch-assets'); - console.log('deployESAsset subprocess: ', subprocess); -} - export async function registerTestJob() { - const subprocess = await execa.command('earl tjm register ts-k8s-e2e testJob.json'); + const subprocess = await execa.command('earl tjm register k8se2e testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } -export async function registerElasticsearchAssets() { - const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/elasticsearch-assets'); - console.log('registerElasticsearch asset subprocess: ', subprocess); +export async function deployElasticsearchAssets() { + const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/elasticsearch-assets'); // FIXME: specify version + console.log('deployElasticsearchAssets subprocess: ', subprocess); +} + +export async function deployStandardAssets() { + const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/standard-assets'); // FIXME: specify version + console.log('deployStandardAssets subprocess: ', subprocess); } -export async function registerStandardAssets() { - const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/standard-assets'); - console.log('registerStandardAssets subprocess: ', subprocess); +export async function deployKafkaAssets() { + const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/kafka-assets'); // FIXME: specify version + console.log('deployKafkaAssets subprocess: ', subprocess); } export async function startTestJob() { @@ -768,10 +829,30 @@ export async function showState() { console.log('showState subprocess: ', subprocess); } -// TODO: utils/src/promises/.ts pDelay already does this +// FIXME: utils/src/promises/.ts pDelay already does this function PromiseTimeout(delayms: number) { return new Promise((resolve) => { setTimeout(resolve, delayms); }); } +export async function takeDownTeraslice() { + const subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); + console.log('resetTeraslice subprocess: ', subprocess); +} + +export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { + await takeDownTeraslice(); + await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); +} + +export async function k8sTearDown() { + await takeDownTeraslice(); // FIXME: or reset + /// Creates configmap for teraslice-worker + let subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); + console.log('workerConfig subprocess: ', subprocess); + + /// Creates deployment for teraslice + subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-worker || echo "* it is okay..."'); + console.log('masterDeploy subprocess: ', subprocess); +} From cb0e1ab64363841a52e9d42d20ea56aad9b2eb14 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:53:23 -0700 Subject: [PATCH 034/142] temp change to get k8s tests running --- .../scripts/src/helpers/test-runner/index.ts | 21 ++++++++++++++----- .../src/helpers/test-runner/services.ts | 12 ++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 47c342b3d2d..1e24901a8df 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -17,6 +17,8 @@ import { isKubectlInstalled, createNamespace, k8sSetup, + loadTerasliceImage, + destroyKindCluster, } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -225,18 +227,19 @@ async function runE2ETest( signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); process.exit(1); } - // TODO: pass kind config file in as a variable + // TODO: pass kind config files in as variables + // FIXME: error handling await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); - await createNamespace(); + await createNamespace(e2eK8sDir, 'ns.yaml'); await k8sSetup(e2eK8sDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); } const rootInfo = getRootInfo(); const e2eImage = `${rootInfo.name}:e2e`; - if (isCI) { + if (isCI && process.env.TEST_PLATFORM === 'native') { // pull the services first in CI - await pullServices(suite, options); // FIXME: if in k8s run different function + await pullServices(suite, options); } try { @@ -256,10 +259,14 @@ async function runE2ETest( tracker.addError(err); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + await loadTerasliceImage(e2eImage); // FIXME: move to global.setup? + } + try { tracker.addCleanup( 'e2e:services', - await ensureServices(suite, options) // FIXME: if in k8s run different function + await ensureServices(suite, options) ); } catch (err) { tracker.addError(err); @@ -314,6 +321,10 @@ async function runE2ETest( }]); }); } + + // if (process.env.TEST_PLATFORM === 'kubernetes') { + // await destroyKindCluster(); + // } } function printAndGetEnv(suite: string, options: TestOptions) { diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 77d5980f460..80dacf3cda9 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -11,7 +11,8 @@ import { getContainerInfo, dockerStop, dockerPull, - kindLoadServiceImage + kindLoadServiceImage, + kindStopService } from '../scripts'; import { TestOptions } from './interfaces'; import { Service } from '../interfaces'; @@ -704,15 +705,16 @@ async function startService(options: TestOptions, service: Service): Promise<() signale.pending(`starting ${service}@${version} service...`); - await stopService(service); if (process.env.TEST_PLATFORM === 'kubernetes') { - await stopService(service); // FIXME: does this use docker - console.log(`@@@@@@@ loading ${service} via kind`); + await kindStopService(service); // load via kind - kindLoadServiceImage(service); + console.log(`@@@@@@@ loading ${service} via kind`); + await kindLoadServiceImage(service); return () => { }; } + await stopService(service); + console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], From fd402ab5f405c0e3904f34ab43758b724039c2d2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:59:27 -0700 Subject: [PATCH 035/142] deploy kafka assets --- e2e/test/global.setup.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 84fafa6d6b6..110463c3eb3 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -2,7 +2,7 @@ const { pDelay } = require('@terascope/utils'); const { - getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets + getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets, deployKafkaAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -57,6 +57,7 @@ module.exports = async () => { await setAlias(); await deployElasticsearchAssets(); await deployStandardAssets(); + await deployKafkaAssets(); } try { From 2e1216c3dc47e0c122a66d9667b59dc957223113 Mon Sep 17 00:00:00 2001 From: sotojn Date: Fri, 20 Oct 2023 10:20:46 -0700 Subject: [PATCH 036/142] update kafka connections for k8s --- e2e/k8s/kafkaDeployment.yaml | 17 +++++++++-------- e2e/k8s/masterConfig/teraslice.yaml | 2 +- e2e/k8s/workerConfig/teraslice.yaml | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index d2f7626ccb7..1aaf60374e1 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -21,12 +21,12 @@ spec: - name: kafka image: terascope/kafka-zookeeper:v1.1.0 env: - - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka:9092 - - name: host.name + - name: ADVERTISED_HOST value: kafka + - name: ADVERTISED_PORT + value: "9093" ports: - - containerPort: 9092 + - containerPort: 9093 --- kind: Service apiVersion: v1 @@ -35,11 +35,12 @@ metadata: labels: app: kafka spec: + type: NodePort selector: app: kafka nodeType: master ports: - - port: 9092 - targetPort: 9092 - nodePort: 30092 # the external port teraslice can be accessed on - type: NodePort + - port: 9093 + name: kafka + targetPort: 9093 + nodePort: 30092 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 924d4bc6dae..5868d2c46a6 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9092" + - "kafka:9093" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index af04736d632..1e887b17795 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9092" + - "kafka:9093" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 From ba5140728c4720988249a5ee3635545af2280209 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Oct 2023 16:58:30 -0700 Subject: [PATCH 037/142] Add ignore pattern for testing in k8s --- e2e/jest.config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/jest.config.js b/e2e/jest.config.js index 0a4be3a992a..f211efc9d58 100644 --- a/e2e/jest.config.js +++ b/e2e/jest.config.js @@ -2,6 +2,8 @@ const config = require('../jest.config.base')(__dirname); +// FIXME update arrays to run tests specific to platform +config.testPathIgnorePatterns = process.env.TEST_PLATFORM === 'kubernetes' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : []; config.collectCoverage = false; delete config.transform; module.exports = config; From 83f99158618955f60fab247fb5aaa09434df3991 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Oct 2023 16:59:41 -0700 Subject: [PATCH 038/142] Add version variables for kafka and elasticsearch --- e2e/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/package.json b/e2e/package.json index 0b7182e9f99..b5291bb2862 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,7 +25,7 @@ "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", From a1c09e796adfd9de2508fa84ae7917e265052a76 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Oct 2023 17:00:18 -0700 Subject: [PATCH 039/142] Refactoring and cleanup. --- e2e/test/global.setup.js | 15 ++++++++------- e2e/test/global.teardown.js | 16 +++++++--------- e2e/test/teraslice-harness.js | 11 ++++++----- packages/scripts/src/helpers/scripts.ts | 21 ++++++--------------- 4 files changed, 27 insertions(+), 36 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 110463c3eb3..b5549f72443 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -2,7 +2,12 @@ const { pDelay } = require('@terascope/utils'); const { - getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets, deployKafkaAssets + getE2eK8sDir, + deployK8sTeraslice, + setAlias, + deployElasticsearchAssets, + deployStandardAssets, + deployKafkaAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -15,7 +20,7 @@ const { CONFIG_PATH, ASSETS_PATH } = require('./config'); module.exports = async () => { const teraslice = new TerasliceHarness(); - await teraslice.init();// create TS and ES or OS clients + await teraslice.init(); if (process.env.TEST_PLATFORM === 'native') { await globalTeardown(teraslice.client); // docker compose down and ES teardown FIXME for k8s @@ -37,15 +42,11 @@ module.exports = async () => { fse.ensureDir(CONFIG_PATH), ]); - // FIXME: config diff between k8s and native - await Promise.all([setupTerasliceConfig(), downloadAssets()]); - - // await pDelay(10000); - if (process.env.TEST_PLATFORM === 'kubernetes') { const e2eK8sDir = getE2eK8sDir(); await deployK8sTeraslice(e2eK8sDir, 'masterDeployment.yaml'); } else { + await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); } diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index 806d293d99c..17b2663a41e 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -1,10 +1,12 @@ 'use strict'; const { ElasticsearchTestHelpers } = require('elasticsearch-store'); -const { k8sTearDown } = require('@terascope/scripts'); +const { tearDownTerasliceK8s } = require('@terascope/scripts'); const fse = require('fs-extra'); -const { KEEP_OPEN, CONFIG_PATH, ASSETS_PATH } = require('./config'); -const { tearDown, TEST_INDEX_PREFIX } = require('./docker-helpers'); +const { + KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, TEST_INDEX_PREFIX +} = require('./config'); +const { tearDown } = require('./docker-helpers'); const signale = require('./signale'); const { cleanupIndex, makeClient } = ElasticsearchTestHelpers; @@ -15,13 +17,9 @@ async function getClient(client) { } async function globalTeardown(testClient) { - console.log('@@@@@@@@ testClient: ', testClient); - console.log('@@@@@@@@ KEEP_OPEN?: ', KEEP_OPEN); - if (KEEP_OPEN) { return; } - console.log('@@@@@@@@ past KEEP_OPEN'); const client = await getClient(testClient); @@ -29,14 +27,14 @@ async function globalTeardown(testClient) { try { if (process.env.TEST_PATTERN === 'kubernetes') { - await k8sTearDown(); + await tearDownTerasliceK8s(); } else { await tearDown(); } } catch (err) { errors.push(err); } - + console.log('@@@@@ global.teardown, TEST_INDEX_PREFIX: ', TEST_INDEX_PREFIX); await cleanupIndex(client, `${TEST_INDEX_PREFIX}*`); if (fse.existsSync(CONFIG_PATH)) { diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 37610617b0f..87eae115c17 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -9,7 +9,6 @@ const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store' const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); const fse = require('fs-extra'); -const { resetTeraslice } = require('@terascope/scripts'); const { TEST_HOST, HOST_IP, SPEC_INDEX_PREFIX, DEFAULT_NODES, newId, DEFAULT_WORKERS, GENERATE_ONLY, @@ -114,16 +113,18 @@ module.exports = class TerasliceHarness { ); })(), (async () => { - if (process.env.TEST_PLATFORM === 'kubernetes') { - // FIXME: scale workers in k8s???????? - } else { - console.log('@@@@@@@ else'); + if (process.env.TEST_PLATFORM === 'native') { const count = Object.keys(state).length; if (count !== DEFAULT_NODES) { signale.warn(`resetting cluster state of ${count} nodes`); await scaleWorkers(); await this.forWorkers(); } + } else { + // Do nothing + // TODO: If tests are ever implemented to scale nodes in Kind, + // a scaleWorkers implementation will need to be created that works with Kind. + // As of 10/2023 Kind doesn't let you scale nodes w/o restarting the cluster. } })() ]); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 2242850157c..030ff876018 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -166,12 +166,6 @@ export async function runJest( signale.debug(`executing: jest ${args.join(' ')}`); } - console.log('######## cwd: ', cwd); - console.log('######## argsMap: ', argsMap); - console.log('######## args: ', args); - console.log('######## env: ', env); - console.log('######## extraArgs: ', extraArgs); - await fork({ cmd: 'jest', cwd, @@ -836,20 +830,17 @@ function PromiseTimeout(delayms: number) { }); } -export async function takeDownTeraslice() { - const subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); - console.log('resetTeraslice subprocess: ', subprocess); -} - export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { - await takeDownTeraslice(); + await tearDownTerasliceK8s(); await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); } -export async function k8sTearDown() { - await takeDownTeraslice(); // FIXME: or reset +export async function tearDownTerasliceK8s() { + let subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); + console.log('resetTeraslice subprocess: ', subprocess); + /// Creates configmap for teraslice-worker - let subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); + subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); console.log('workerConfig subprocess: ', subprocess); /// Creates deployment for teraslice From 02639efc6ab41f59db42257de4180b75b86d23c4 Mon Sep 17 00:00:00 2001 From: sotojn Date: Tue, 24 Oct 2023 07:26:38 -0700 Subject: [PATCH 040/142] update docker compose yaml to use cp-kafka and cp-zookeeper images --- docker-compose.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fcbcecdd76b..5c57163ffd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,16 +73,32 @@ services: cap_add: - IPC_LOCK kafka: - image: terascope/kafka-zookeeper:v1.1.0 - platform: linux/amd64 # platform specified to force amd64 on arm MacOS docker + image: confluentinc/cp-kafka:7.2.0 # Has kafka 3.2.0 + ports: + - "9094:9094" + restart: unless-stopped + depends_on: + - zookeeper + networks: + - cluster + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://0.0.0.0:9094 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://localhost:9094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + zookeeper: + image: confluentinc/cp-zookeeper:7.2.0 ports: - "2181:2181" - - "9092:9092" restart: unless-stopped networks: - cluster - volumes: - - kafka-data:/kafka + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 minio: image: minio/minio:RELEASE.2023-09-30T07-02-29Z ports: From 7d042521095cc7ec9e044a5cfed681d1446f307a Mon Sep 17 00:00:00 2001 From: sotojn Date: Tue, 24 Oct 2023 09:05:29 -0700 Subject: [PATCH 041/142] update k8s tests to use cp-kafka and cp-zkeeper --- e2e/k8s/kafkaDeployment.yaml | 42 +++++++++++++--------- e2e/k8s/masterConfig/teraslice.yaml | 2 +- e2e/k8s/workerConfig/teraslice.yaml | 2 +- e2e/k8s/zookeeperDeployment.yaml | 46 +++++++++++++++++++++++++ packages/scripts/src/helpers/scripts.ts | 4 ++- 5 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 e2e/k8s/zookeeperDeployment.yaml diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index 1aaf60374e1..90220bf0d6d 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -1,46 +1,54 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: kafka + name: cpkafka labels: - app: kafka + app: cpkafka nodeType: master spec: replicas: 1 selector: matchLabels: - app: kafka + app: cpkafka nodeType: master template: metadata: labels: - app: kafka + app: cpkafka nodeType: master spec: containers: - - name: kafka - image: terascope/kafka-zookeeper:v1.1.0 + - name: cpkafka + image: confluentinc/cp-kafka:7.2.0 env: - - name: ADVERTISED_HOST - value: kafka - - name: ADVERTISED_PORT - value: "9093" + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: zookeeper:2181 + - name: KAFKA_ADVERTISED_LISTENERS + value: INTERNAL://cpkafka:9092 + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: INTERNAL:PLAINTEXT + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: INTERNAL + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" ports: - - containerPort: 9093 + - containerPort: 9092 --- kind: Service apiVersion: v1 metadata: - name: kafka + name: cpkafka labels: - app: kafka + app: cpkafka spec: type: NodePort selector: - app: kafka + app: cpkafka nodeType: master ports: - - port: 9093 - name: kafka - targetPort: 9093 + - port: 9092 + name: cpkafka + targetPort: 9092 nodePort: 30092 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 5868d2c46a6..457d537d514 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9093" + - "cpkafka:9092" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 1e887b17795..3b1870c3d27 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9093" + - "cpkafka:9092" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/zookeeperDeployment.yaml b/e2e/k8s/zookeeperDeployment.yaml new file mode 100644 index 00000000000..d8069babb38 --- /dev/null +++ b/e2e/k8s/zookeeperDeployment.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper + labels: + app: zookeeper + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + nodeType: master + template: + metadata: + labels: + app: zookeeper + nodeType: master + spec: + containers: + - name: zookeeper + image: confluentinc/cp-zookeeper:7.2.0 + env: + - name: ZOOKEEPER_CLIENT_PORT + value: "2181" + - name: ZOOKEEPER_TICK_TIME + value: "2000" + ports: + - containerPort: 2181 +--- +kind: Service +apiVersion: v1 +metadata: + name: zookeeper + labels: + app: zookeeper +spec: + type: NodePort + selector: + app: zookeeper + nodeType: master + ports: + - port: 2181 + name: zookeeper + targetPort: 2181 + nodePort: 32181 \ No newline at end of file diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 030ff876018..d5011277f36 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -603,7 +603,7 @@ export async function kindLoadServiceImage(serviceName: string): Promise { if (serviceName === 'elasticsearch') { await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); } else if (serviceName === 'kafka') { - await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml'); + await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml', 'zookeeperDeployment.yaml'); } else { signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); } @@ -662,7 +662,9 @@ function waitForESRunning(timeoutMs = 120000): Promise { export async function deployKafka( e2eK8sDir: string, kafkaDeploymentYaml: string, + zookeeperDeploymentYaml: string ) { + await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, zookeeperDeploymentYaml)}`); let subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); console.log('deployKafka subprocess: ', subprocess); From d81451dbbe4d7b48047d0ce7c1123da139a5fd6c Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 23 Oct 2023 16:25:51 -0700 Subject: [PATCH 042/142] Refactoring, add error handling, add comments. --- e2e/test/global.setup.js | 14 +- e2e/test/global.teardown.js | 1 - e2e/test/teraslice-harness.js | 2 + packages/scripts/src/helpers/scripts.ts | 315 +++++++----------- .../scripts/src/helpers/test-runner/index.ts | 167 ++-------- .../src/helpers/test-runner/services.ts | 13 +- 6 files changed, 161 insertions(+), 351 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index b5549f72443..0f3f315264f 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -5,9 +5,7 @@ const { getE2eK8sDir, deployK8sTeraslice, setAlias, - deployElasticsearchAssets, - deployStandardAssets, - deployKafkaAssets + deployAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -22,9 +20,7 @@ module.exports = async () => { const teraslice = new TerasliceHarness(); await teraslice.init(); - if (process.env.TEST_PLATFORM === 'native') { - await globalTeardown(teraslice.client); // docker compose down and ES teardown FIXME for k8s - } + await globalTeardown(teraslice.client); await teraslice.resetLogs(); process.stdout.write('\n'); @@ -56,9 +52,9 @@ module.exports = async () => { if (process.env.TEST_PLATFORM === 'kubernetes') { await setAlias(); - await deployElasticsearchAssets(); - await deployStandardAssets(); - await deployKafkaAssets(); + await deployAssets('elasticsearch'); + await deployAssets('standard'); + await deployAssets('kafka'); } try { diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index 17b2663a41e..e0605afb025 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -34,7 +34,6 @@ async function globalTeardown(testClient) { } catch (err) { errors.push(err); } - console.log('@@@@@ global.teardown, TEST_INDEX_PREFIX: ', TEST_INDEX_PREFIX); await cleanupIndex(client, `${TEST_INDEX_PREFIX}*`); if (fse.existsSync(CONFIG_PATH)) { diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 87eae115c17..bde5676b4f5 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -358,6 +358,8 @@ module.exports = class TerasliceHarness { } if (process.env.TEST_PLATFORM === 'kubernetes') { + // A get request to 'cluster/state' will return an empty object in kubernetes. + // Therefore nodes will be 0. if (nodes === 0) return nodes; } if (nodes >= DEFAULT_NODES) return nodes; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index d5011277f36..0ccf6f1ea92 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -534,19 +534,20 @@ export async function yarnPublish( }); } -export async function createKindCluster( - e2eK8sDir: string, - kindConfigFileName: string -): Promise { - const configPath = path.join(e2eK8sDir, kindConfigFileName); +export async function createKindCluster(): Promise { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const configPath = path.join(e2eK8sDir, 'kindConfig.yaml'); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); - signale.log(subprocess.stderr); + signale.info(subprocess.stderr); } export async function destroyKindCluster(): Promise { // TODO: pass cluster name in as variable const subprocess = await execa.command('kind delete cluster --name k8se2e'); - signale.log(subprocess.stderr); + signale.info(subprocess.stderr); } export async function isKindInstalled(): Promise { @@ -572,30 +573,68 @@ export async function isKubectlInstalled(): Promise { export async function loadTerasliceImage(terasliceImage: string): Promise { const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); - console.log('load teraslice image subprocess: ', subprocess); + // console.log('load teraslice image subprocess: ', subprocess); + signale.info(subprocess.stderr); + + // FIXME: is it necessary to wait? + // const imageLoaded = await waitForTerasliceImageLoaded(); + // if (imageLoaded) { + // signale.success('Teraslice image successfully loaded.'); + // } } +// function waitForTerasliceImageLoaded(timeoutMs = 120000): Promise { +// const endAt = Date.now() + timeoutMs; + +// const _waitForTerasliceImageLoaded = async (): Promise => { +// if (Date.now() > endAt) { +// throw new Error(`Failure to load teraslice image after ${timeoutMs}ms`); +// } + +// let terasliceLoaded = false; +// try { +// // docker exec into kind-control-plane +// // crictl images -o json +// // find repo tag "docker.io/library/teraslice-workspace:e2e" +// const kubectlResponse = await execa.command(''); +// const kindFinished = kubectlResponse.stdout; +// console.log('kubectl response: ', kubectlResponse); +// if (kindFinished === '"true"') { +// terasliceLoaded = true; +// } +// } catch (err) { +// await pDelay(3000); +// return _waitForTerasliceImageLoaded(); +// } + +// if (terasliceLoaded) { +// return true; +// } +// await pDelay(3000); +// return _waitForTerasliceImageLoaded(); +// }; + +// return _waitForTerasliceImageLoaded(); +// } + export async function kindStopService(serviceName: string): Promise { - console.log('@@@@@ kindStopService'); - // const e2eK8sDir = getE2eK8sDir(); - // if (!e2eK8sDir) { - // throw new Error('Missing k8s e2e test directory'); - // } - // if (serviceName === 'elasticsearch') { - // const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'elasticsearchDeployment.yaml')}`); - // console.log('stopElasticsearch subprocess: ', subprocess); - // } else if (serviceName === 'kafka') { - // let subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaDeployment.yaml')}`); - // console.log('stopKafkaDeployment subprocess: ', subprocess); - // // subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaService.yaml')}`); - // // console.log('stopKafkaService subprocess: ', subprocess); - // } else { - // signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); - // } + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + try { + // Any new service's yaml file must be named 'Deployment.yaml' + const yamlFile = `${serviceName}Deployment.yaml`; + const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + signale.info(subprocess.stdout); + // console.log('stopElasticsearch subprocess: ', subprocess); + } catch (err) { + signale.warn(`The service ${serviceName} could not be deleted because it does not exist.`); + } } +// FIXME: refactor export async function kindLoadServiceImage(serviceName: string): Promise { - console.log('@@@@@ kindLoadServiceImage'); const e2eK8sDir = getE2eK8sDir(); if (!e2eK8sDir) { throw new Error('Missing k8s e2e test directory'); @@ -609,54 +648,20 @@ export async function kindLoadServiceImage(serviceName: string): Promise { } } -export async function createNamespace(e2eK8sDir: string, namespaceYaml: string) { - const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); - console.log('namespace subprocess: ', subprocess); +export async function createNamespace() { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'ns.yaml')}`); + signale.info(subprocess.stdout); + // console.log('namespace subprocess: ', subprocess); } export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { - console.log('@@@@@ deployElasticsearch'); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - console.log('deployElasticsearch subprocess: ', subprocess); - - const elasticsearchReady = await waitForESRunning(240000); - if (elasticsearchReady) { - signale.success('Elasticsearch is ready to go'); - } -} - -function waitForESRunning(timeoutMs = 120000): Promise { - const endAt = Date.now() + timeoutMs; - - const _waitForESRunning = async (): Promise => { - if (Date.now() > endAt) { - throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); - } - - let elasticsearchRunning = false; - try { - const ESResponse = await execa.command('curl http://localhost:49200'); - if (ESResponse.stdout) { - const jsonData = JSON.parse(ESResponse.stdout); - console.log(`response: ${JSON.stringify(jsonData)}`); - if (jsonData.tagline === 'You Know, for Search') { - console.log('jsonData.tagline === You Know, for Search'); - elasticsearchRunning = true; - } - } - } catch (err) { - await PromiseTimeout(3000); - return _waitForESRunning(); - } - - if (elasticsearchRunning) { - return true; - } - await PromiseTimeout(3000); - return _waitForESRunning(); - }; - - return _waitForESRunning(); + signale.info(subprocess.stdout); + // console.log('deployElasticsearch subprocess: ', subprocess); } export async function deployKafka( @@ -664,17 +669,11 @@ export async function deployKafka( kafkaDeploymentYaml: string, zookeeperDeploymentYaml: string ) { - await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, zookeeperDeploymentYaml)}`); - let subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); - console.log('deployKafka subprocess: ', subprocess); - - // subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaServiceYaml)}`); - // console.log('deployKafkaService subprocess: ', subprocess); + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); + signale.info(subprocess.stdout); + // console.log('deployKafka subprocess: ', subprocess); - // const kafkaReady = await waitForKafkaRunning(240000); - // if (kafkaReady) { - signale.success('Kafka *might* be ready to go'); - // } + await waitForKafkaRunning(240000); } function waitForKafkaRunning(timeoutMs = 120000): Promise { @@ -682,45 +681,44 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { const _waitForKafkaRunning = async (): Promise => { if (Date.now() > endAt) { - throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); + throw new Error(`Failure to communicate with kafka after ${timeoutMs}ms`); } - let elasticsearchRunning = false; + let kafkaRunning = false; try { - const kafkaResponse = await execa.command('kubectl -n ts-dev1 get po '); - if (kafkaResponse.stdout) { - const jsonData = JSON.parse(kafkaResponse.stdout); - console.log(`response: ${JSON.stringify(jsonData)}`); - if (jsonData.tagline === 'You Know, for Search') { - console.log('jsonData.tagline === You Know, for Search'); - elasticsearchRunning = true; - } + const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=kafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + const kafkaReady = kubectlResponse.stdout; + // console.log('kafka response: ', kafkaReady); + if (kafkaReady === '"true"') { + kafkaRunning = true; } } catch (err) { - await PromiseTimeout(3000); + await pDelay(3000); return _waitForKafkaRunning(); } - if (elasticsearchRunning) { + if (kafkaRunning) { return true; } - await PromiseTimeout(3000); + await pDelay(3000); return _waitForKafkaRunning(); }; return _waitForKafkaRunning(); } -export async function k8sSetup( - e2eK8sDir: string, - roleYaml: string, - roleBindingYaml: string, - priorityClassYaml: string -): Promise { - const subprocess1 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleYaml)}`); - const subprocess2 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleBindingYaml)}`); - const subprocess3 = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, priorityClassYaml)}`); - console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); +export async function k8sSetup(): Promise { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + + let subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'role.yaml')}`); + signale.info(subprocess.stdout); + subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'roleBinding.yaml')}`); + signale.info(subprocess.stdout); + subprocess = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, 'priorityClass.yaml')}`); + signale.info(subprocess.stdout); } export async function deployK8sTeraslice( @@ -730,108 +728,45 @@ export async function deployK8sTeraslice( try { /// Creates configmap for terasclice-master let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); - console.log('masterConfig subprocess: ', subprocess); + // console.log('masterConfig subprocess: ', subprocess); + signale.info(subprocess.stdout); /// Creates configmap for teraslice-worker subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); - console.log('workerConfig subprocess: ', subprocess); + // console.log('workerConfig subprocess: ', subprocess); + signale.info(subprocess.stdout); /// Creates deployment for teraslice subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); - console.log('masterDeploy subprocess: ', subprocess); - - subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); - const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); - console.log('podName: ', podName); - - const terasliceReady = await waitForTerasliceRunning(240000); - if (terasliceReady) { - signale.success('Teraslice is ready to go'); - } + // console.log('masterDeploy subprocess: ', subprocess); + signale.info(subprocess.stdout); } catch (error) { - console.log('@@@@@@@@ deployK8sTeraslice error: ', error); + signale.error('Error in deployK8sTeraslice function: ', error); } } -function waitForTerasliceRunning(timeoutMs = 120000): Promise { - const endAt = Date.now() + timeoutMs; - - const _waitForTerasliceRunning = async (): Promise => { - if (Date.now() > endAt) { - throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); - } - - let terasliceRunning = false; - try { - const TSMasterResponse = await execa.command('curl localhost:45678'); - if (TSMasterResponse.stdout) { - const jsonData = JSON.parse(TSMasterResponse.stdout); - console.log(`response: ${JSON.stringify(jsonData)}`); - if (jsonData.clustering_type === 'kubernetes') { - console.log('jsonData.clusteringType === kubernetes'); - terasliceRunning = true; - } - } - } catch (err) { - await PromiseTimeout(3000); - return _waitForTerasliceRunning(); - } - - if (terasliceRunning) { - return true; - } - - await PromiseTimeout(3000); - return _waitForTerasliceRunning(); - }; - - return _waitForTerasliceRunning(); -} - export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); - const subprocess2 = await execa.command('earl aliases add k8se2e http://localhost:45678'); - console.log('setAlias subprocess: ', subprocess1, subprocess2); -} - -export async function registerTestJob() { - const subprocess = await execa.command('earl tjm register k8se2e testJob.json'); - console.log('registerTestJob subprocess: ', subprocess); -} - -export async function deployElasticsearchAssets() { - const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/elasticsearch-assets'); // FIXME: specify version - console.log('deployElasticsearchAssets subprocess: ', subprocess); -} - -export async function deployStandardAssets() { - const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/standard-assets'); // FIXME: specify version - console.log('deployStandardAssets subprocess: ', subprocess); + let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); + signale.info(subprocess.stdout); + subprocess = await execa.command('earl aliases add k8se2e http://localhost:45678'); + signale.info(subprocess.stdout); + // console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function deployKafkaAssets() { - const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/kafka-assets'); // FIXME: specify version - console.log('deployKafkaAssets subprocess: ', subprocess); -} - -export async function startTestJob() { - const subprocess = await execa.command('earl tjm start testJob.json'); - console.log('Run earl tjm start testJob.json'); - console.log(subprocess.stdout); +export async function deployAssets(assetName: string) { + const subprocess = await execa.command(`earl assets deploy k8se2e --bundle --blocking terascope/${assetName}-assets`); + signale.info(subprocess.stdout); + // console.log('deployKafkaAssets subprocess: ', subprocess); } +// FIXME: delete before merging - for testing export async function showState() { const subprocess = await execa.command('kubectl -n ts-dev1 get deployments,po,svc -o wide'); - console.log('showState subprocess: ', subprocess); -} - -// FIXME: utils/src/promises/.ts pDelay already does this -function PromiseTimeout(delayms: number) { - return new Promise((resolve) => { - setTimeout(resolve, delayms); - }); + signale.info(subprocess.stdout); + // console.log('showState subprocess: ', subprocess); } +// FIXME: delete before merging if not needed in refactor export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { await tearDownTerasliceK8s(); await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); @@ -839,13 +774,13 @@ export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: st export async function tearDownTerasliceK8s() { let subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); - console.log('resetTeraslice subprocess: ', subprocess); + signale.info(subprocess.stdout); - /// Creates configmap for teraslice-worker subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); - console.log('workerConfig subprocess: ', subprocess); + signale.info(subprocess.stdout); + // console.log('workerConfig subprocess: ', subprocess); - /// Creates deployment for teraslice subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-worker || echo "* it is okay..."'); - console.log('masterDeploy subprocess: ', subprocess); + signale.info(subprocess.stdout); + // console.log('masterDeploy subprocess: ', subprocess); } diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 1e24901a8df..cd7227830fd 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -27,7 +27,7 @@ import { } from './utils'; import signale from '../signale'; import { - getE2EDir, readPackageInfo, listPackages, getE2eK8sDir + getE2EDir, readPackageInfo, listPackages } from '../packages'; import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; @@ -200,7 +200,7 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { - console.log('options: ', options); + // console.log('options: ', options); tracker.expected++; const suite = 'e2e'; @@ -212,26 +212,25 @@ async function runE2ETest( } if (process.env.TEST_PLATFORM === 'kubernetes') { - const e2eK8sDir = getE2eK8sDir(); - if (!e2eK8sDir) { - throw new Error('Missing k8s e2e test directory'); - } - const kindInstalled = await isKindInstalled(); - if (!kindInstalled) { - signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); - process.exit(1); - } + try { + const kindInstalled = await isKindInstalled(); + if (!kindInstalled) { + signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); + process.exit(1); + } - const kubectlInstalled = await isKubectlInstalled(); - if (!kubectlInstalled) { - signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); - process.exit(1); + const kubectlInstalled = await isKubectlInstalled(); + if (!kubectlInstalled) { + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + process.exit(1); + } + + await createKindCluster(); + await createNamespace(); + await k8sSetup(); + } catch (err) { + tracker.addError(err); } - // TODO: pass kind config files in as variables - // FIXME: error handling - await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); - await createNamespace(e2eK8sDir, 'ns.yaml'); - await k8sSetup(e2eK8sDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); } const rootInfo = getRootInfo(); @@ -260,13 +259,18 @@ async function runE2ETest( } if (process.env.TEST_PLATFORM === 'kubernetes') { - await loadTerasliceImage(e2eImage); // FIXME: move to global.setup? + try { + await loadTerasliceImage(e2eImage); + } catch (err) { + tracker.addError(err); + } } try { + const svcFn = await ensureServices(suite, options); tracker.addCleanup( 'e2e:services', - await ensureServices(suite, options) + svcFn ); } catch (err) { tracker.addError(err); @@ -340,122 +344,3 @@ function printAndGetEnv(suite: string, options: TestOptions) { } return env; } - -// async function runk8sE2ETest( -// options: TestOptions, tracker: TestTracker -// ): Promise { -// console.log('options: ', options); -// tracker.expected++; - -// const k8se2eDir = getK8SE2EDir(); -// if (!k8se2eDir) { -// throw new Error('Missing k8se2e test directory'); -// } - -// const kindInstalled = await isKindInstalled(); -// if (!kindInstalled) { -// signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); -// process.exit(1); -// } - -// const kubectlInstalled = await isKubectlInstalled(); -// if (!kubectlInstalled) { -// signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); -// process.exit(1); -// } -// // TODO: pass kind config file in as a variable -// await createKindCluster(k8se2eDir, 'kindConfig.yaml'); - -// const suite = 'k8se2e'; -// let startedTest = false; - -// const rootInfo = getRootInfo(); -// const k8se2eImage = `${rootInfo.name}:k8se2e`; - -// // if (isCI) { -// // // pull the services first in CI -// // await pullServices(suite, options); -// // } - -// try { -// if (SKIP_DOCKER_BUILD_IN_E2E) { -// const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; -// await dockerTag(devImage, k8se2eImage); -// await loadTerasliceImage(k8se2eImage); -// } else { -// const publishOptions: PublishOptions = { -// dryRun: true, -// nodeVersion: options.nodeVersion, -// type: PublishType.Dev -// }; -// const devImage = await buildDevDockerImage(publishOptions); -// await dockerTag(devImage, k8se2eImage); -// await loadTerasliceImage(k8se2eImage); -// } -// } catch (err) { -// tracker.addError(err); -// } - -// TODO: add tracker -// await createNamespace(); -// await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); -// await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); -// await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); -// await showState(); - -// await setAlias(); -// await registerElasticsearchAssets(); -// await registerStandardAssets(); -// await registerTestJob(); -// await startTestJob(); - -// if (!tracker.hasErrors()) { -// const timeLabel = `test suite "${suite}"`; -// signale.time(timeLabel); -// startedTest = true; - -// const env = printAndGetEnv(suite, options); - -// tracker.started++; -// try { -// await runJest( -// k8se2eDir, -// getArgs(options), -// env, -// options.jestArgs, -// options.debug -// ); -// tracker.ended++; -// } catch (err) { -// tracker.ended++; -// tracker.addError(err.message); -// } - -// signale.timeEnd(timeLabel); -// } - -// if (!startedTest) return; - -// if (!options.keepOpen) { -// try { -// await logE2E(k8se2eDir, tracker.hasErrors()); -// } catch (err) { -// signale.error( -// new TSError(err, { -// reason: `Writing the "${suite}" logs failed`, -// }) -// ); -// } -// } - -// if (tracker.hasErrors()) { -// tracker.addCleanup('e2e:teardown', async () => { -// options.keepOpen = false; -// await globalTeardown(options, [{ -// name: suite, -// dir: k8se2eDir, -// suite, -// }]); -// }); -// } -// } diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 80dacf3cda9..c5df90ac310 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -137,8 +137,6 @@ const services: Readonly>> = { }; export async function pullServices(suite: string, options: TestOptions): Promise { - console.log('@@@ in pull services'); - const launchServices = getServicesForSuite(suite); try { @@ -193,7 +191,6 @@ export async function pullServices(suite: string, options: TestOptions): Promise } export async function ensureServices(suite: string, options: TestOptions): Promise<() => void> { - console.log('@@@ in ensure services'); const launchServices = getServicesForSuite(suite); const promises: Promise<(() => void)>[] = []; @@ -685,8 +682,6 @@ async function checkKafka(options: TestOptions, startTime: number) { } async function startService(options: TestOptions, service: Service): Promise<() => void> { - console.log('in startService: ', service); - let serviceName = service; if (serviceName === 'restrained_elasticsearch') { @@ -707,15 +702,15 @@ async function startService(options: TestOptions, service: Service): Promise<() if (process.env.TEST_PLATFORM === 'kubernetes') { await kindStopService(service); - // load via kind - console.log(`@@@@@@@ loading ${service} via kind`); + + // console.log(`@@@@@@@ loading ${service} via kind`); await kindLoadServiceImage(service); return () => { }; } await stopService(service); - console.log(`@@@@@@@ loading ${service} via docker`); + // console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], version, @@ -723,7 +718,6 @@ async function startService(options: TestOptions, service: Service): Promise<() options.debug || options.trace ); - console.log('@@@@@ fn(): ', fn.toString()); return () => { try { fn(); @@ -735,5 +729,4 @@ async function startService(options: TestOptions, service: Service): Promise<() ); } }; - } From cb0a4303c4112cc68649f2807a1ec4bed6f5f4ef Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 08:24:49 -0700 Subject: [PATCH 043/142] update to latest esm module for streaming redirects in teraslice --- .../teraslice/lib/cluster/services/api.js | 23 ++-- packages/teraslice/package.json | 1 + yarn.lock | 105 +++++++++++++++++- 3 files changed, 118 insertions(+), 11 deletions(-) diff --git a/packages/teraslice/lib/cluster/services/api.js b/packages/teraslice/lib/cluster/services/api.js index 9c14760133c..41d5f9752e3 100644 --- a/packages/teraslice/lib/cluster/services/api.js +++ b/packages/teraslice/lib/cluster/services/api.js @@ -2,9 +2,7 @@ const { Router } = require('express'); const bodyParser = require('body-parser'); -const stream = require('stream'); -const { promisify } = require('util'); -const got = require('got'); +const { pipeline: streamPipeline } = require('node:stream/promises'); const { RecoveryCleanupType } = require('@terascope/job-components'); const { parseErrorInfo, parseList, logError, TSError, startsWith @@ -20,7 +18,15 @@ const { } = require('../../utils/api_utils'); const terasliceVersion = require('../../../package.json').version; -const pStreamPipeline = promisify(stream.pipeline); + +let gotESMModule; + +async function getGotESM() { + if (gotESMModule) return gotESMModule; + const module = await import('gotESM'); // eslint-disable-line + gotESMModule = module.default; + return module.default; +} module.exports = function apiService(context, { assetsUrl, app }) { const clusterConfig = context.sysconfig.teraslice; @@ -488,23 +494,24 @@ module.exports = function apiService(context, { assetsUrl, app }) { } async function _redirect(req, res) { + const module = await getGotESM() const options = { prefixUrl: assetsUrl, headers: req.headers, searchParams: req.query, throwHttpErrors: false, - timeout: clusterConfig.api_response_timeout, + timeout: { request: clusterConfig.api_response_timeout }, decompress: false, - retry: 0 + retry: { limit: 0 } }; const uri = req.url.replace(/^\//, ''); const method = req.method.toLowerCase(); try { - await pStreamPipeline( + await streamPipeline( req, - got.stream[method](uri, options), + module.stream[method](uri, options), res, ); } catch (err) { diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 00ab67c2abd..fa2bbbb621e 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -56,6 +56,7 @@ "fs-extra": "^11.1.1", "gc-stats": "^1.4.0", "got": "^11.8.3", + "gotESM": "npm:got@^13.0.0", "ip": "^1.1.8", "kubernetes-client": "^9.0.0", "lodash": "^4.17.21", diff --git a/yarn.lock b/yarn.lock index 4933a242dac..2b966165886 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1659,6 +1659,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== +"@sindresorhus/is@^5.2.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-5.6.0.tgz#41dd6093d34652cddb5d5bdeee04eafc33826668" + integrity sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g== + "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.8.1": version "1.8.6" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.6.tgz#80c516a4dc264c2a69115e7578d62581ff455ed9" @@ -2213,6 +2218,13 @@ dependencies: defer-to-connect "^2.0.0" +"@szmarczak/http-timer@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-5.0.1.tgz#c7c1bf1141cdd4751b0399c8fc7b8b664cd5be3a" + integrity sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw== + dependencies: + defer-to-connect "^2.0.1" + "@terascope/fetch-github-release@^0.8.7": version "0.8.7" resolved "https://registry.yarnpkg.com/@terascope/fetch-github-release/-/fetch-github-release-0.8.7.tgz#fd8aba265c06704d51a6ab70fd2c656c80f04156" @@ -2648,6 +2660,11 @@ resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz#0ea7b61496902b95890dc4c3a116b60cb8dae812" integrity sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ== +"@types/http-cache-semantics@^4.0.2": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.3.tgz#a3ff232bf7d5c55f38e4e45693eda2ebb545794d" + integrity sha512-V46MYLFp08Wf2mmaBhvgjStM3tPa+2GAdy/iqoX+noX1//zje2x4XmrIU0cAwyClATsTmahbtoQ2EwP7I5WSiA== + "@types/inquirer@^8": version "8.2.6" resolved "https://registry.yarnpkg.com/@types/inquirer/-/inquirer-8.2.6.tgz#abd41a5fb689c7f1acb12933d787d4262a02a0ab" @@ -4089,6 +4106,24 @@ cacheable-lookup@^5.0.3: resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005" integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA== +cacheable-lookup@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz#3476a8215d046e5a3202a9209dd13fec1f933a27" + integrity sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w== + +cacheable-request@^10.2.8: + version "10.2.14" + resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-10.2.14.tgz#eb915b665fda41b79652782df3f553449c406b9d" + integrity sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ== + dependencies: + "@types/http-cache-semantics" "^4.0.2" + get-stream "^6.0.1" + http-cache-semantics "^4.1.1" + keyv "^4.5.3" + mimic-response "^4.0.0" + normalize-url "^8.0.0" + responselike "^3.0.0" + cacheable-request@^2.1.1: version "2.1.4" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-2.1.4.tgz#0d808801b6342ad33c91df9d0b44dc09b91e5c3d" @@ -4973,7 +5008,7 @@ defer-to-connect@^1.0.1: resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== -defer-to-connect@^2.0.0: +defer-to-connect@^2.0.0, defer-to-connect@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587" integrity sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg== @@ -6170,6 +6205,11 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== +form-data-encoder@^2.1.2: + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-2.1.4.tgz#261ea35d2a70d48d30ec7a9603130fa5515e9cd5" + integrity sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw== + form-data@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" @@ -6460,7 +6500,7 @@ get-stream@^5.1.0: dependencies: pump "^3.0.0" -get-stream@^6.0.0: +get-stream@^6.0.0, get-stream@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== @@ -6726,6 +6766,23 @@ got@^9.6.0: to-readable-stream "^1.0.0" url-parse-lax "^3.0.0" +"gotESM@npm:got@^13.0.0": + version "13.0.0" + resolved "https://registry.yarnpkg.com/got/-/got-13.0.0.tgz#a2402862cef27a5d0d1b07c0fb25d12b58175422" + integrity sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA== + dependencies: + "@sindresorhus/is" "^5.2.0" + "@szmarczak/http-timer" "^5.0.1" + cacheable-lookup "^7.0.0" + cacheable-request "^10.2.8" + decompress-response "^6.0.0" + form-data-encoder "^2.1.2" + get-stream "^6.0.1" + http2-wrapper "^2.1.10" + lowercase-keys "^3.0.0" + p-cancelable "^3.0.0" + responselike "^3.0.0" + graceful-fs@^4.1.10, graceful-fs@^4.1.2, graceful-fs@^4.1.5, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.6, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -6972,6 +7029,14 @@ http2-wrapper@^1.0.0-beta.5.2: quick-lru "^5.1.1" resolve-alpn "^1.0.0" +http2-wrapper@^2.1.10: + version "2.2.0" + resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-2.2.0.tgz#b80ad199d216b7d3680195077bd7b9060fa9d7f3" + integrity sha512-kZB0wxMo0sh1PehyjJUWRFEd99KC5TLjZ2cULC4f9iqJBAmKQQXEICjxl5iPJRwP40dpeHFqqhm7tYCvODpqpQ== + dependencies: + quick-lru "^5.1.1" + resolve-alpn "^1.2.0" + https-proxy-agent@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" @@ -8353,6 +8418,13 @@ keyv@^4.0.0: dependencies: json-buffer "3.0.1" +keyv@^4.5.3: + version "4.5.4" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" + integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== + dependencies: + json-buffer "3.0.1" + kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" @@ -8615,6 +8687,11 @@ lowercase-keys@^2.0.0: resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== +lowercase-keys@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-3.0.0.tgz#c5e7d442e37ead247ae9db117a9d0a467c89d4f2" + integrity sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ== + lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -8896,6 +8973,11 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== +mimic-response@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-4.0.0.tgz#35468b19e7c75d10f5165ea25e75a5ceea7cf70f" + integrity sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg== + "minimatch@2 || 3", minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -9449,6 +9531,11 @@ normalize-url@^6.0.1: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +normalize-url@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-8.0.0.tgz#593dbd284f743e8dcf6a5ddf8fadff149c82701a" + integrity sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw== + npm-bundled@^1.0.1, npm-bundled@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.1.2.tgz#944c78789bd739035b70baa2ca5cc32b8d860bc1" @@ -9909,6 +9996,11 @@ p-cancelable@^2.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg== +p-cancelable@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-3.0.0.tgz#63826694b54d61ca1c20ebcb6d3ecf5e14cd8050" + integrity sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw== + p-defer@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-1.0.0.tgz#9f6eb182f6c9aa8cd743004a7d4f96b196b0fb0c" @@ -10890,7 +10982,7 @@ require-main-filename@^2.0.0: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" integrity sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg== -resolve-alpn@^1.0.0: +resolve-alpn@^1.0.0, resolve-alpn@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" integrity sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g== @@ -10956,6 +11048,13 @@ responselike@^2.0.0: dependencies: lowercase-keys "^2.0.0" +responselike@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/responselike/-/responselike-3.0.0.tgz#20decb6c298aff0dbee1c355ca95461d42823626" + integrity sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg== + dependencies: + lowercase-keys "^3.0.0" + restore-cursor@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-1.0.1.tgz#34661f46886327fed2991479152252df92daa541" From 0a4573138c5d85885a7f6ffe74a36849c228d4f9 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 08:28:58 -0700 Subject: [PATCH 044/142] remove node 14 from build and test as we migrate away from it --- .github/workflows/publish-tag.yml | 3 +-- .github/workflows/test.yml | 19 ++++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish-tag.yml b/.github/workflows/publish-tag.yml index 1b2caaaa68a..17e3398af4e 100644 --- a/.github/workflows/publish-tag.yml +++ b/.github/workflows/publish-tag.yml @@ -42,8 +42,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # node-version: [14.21.3, 16.19.1, 18.16.0] - node-version: [14.21.3, 16.19.1] + node-version: [16.19.1, 18.16.0] steps: # we login to docker to publish new teraslice image - name: Login to Docker Hub diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27e0eced78c..82abc4d29d5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 14.21.3 + node-version: 18.16.0 cache: 'yarn' - name: Install and build packages @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [14.21.3, 16.19.1, 18.16.0] + node-version: [16.19.1, 18.16.0] steps: - name: Check out code uses: actions/checkout@v3 @@ -57,7 +57,7 @@ jobs: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: [14.21.3, 16.19.1, 18.16.0] + node-version: [16.19.1, 18.16.0] search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code @@ -91,7 +91,7 @@ jobs: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: [14.21.3, 16.19.1, 18.16.0] + node-version: [16.19.1, 18.16.0] search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code @@ -129,7 +129,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 14.21.3 + node-version: 18.16.0 cache: 'yarn' # we login to docker to avoid docker pull limit rates @@ -154,7 +154,7 @@ jobs: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: [14.21.3, 16.19.1, 18.16.0] + node-version: [16.19.1, 18.16.0] search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code @@ -193,7 +193,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 14.21.3 + node-version: 18.16.0 cache: 'yarn' # we login to docker to avoid docker pull limit rates @@ -218,10 +218,7 @@ jobs: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - # temporarily remove node18 from the e2e tests since they fail, see: - # https://github.com/terascope/teraslice/issues/3434 - # node-version: [14.21.3, 16.19.1, 18.16.0] - node-version: [14.21.3, 16.19.1] + node-version: [16.19.1, 18.16.0] search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code From 212ca01dce4f0e23f711038275cef5cb60a195a3 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 08:30:27 -0700 Subject: [PATCH 045/142] release: (minor) teraslice@0.87.0 --- package.json | 2 +- packages/teraslice/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index f579c325538..6d7de255f72 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "0.86.5", + "version": "0.87.0", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index fa2bbbb621e..b7693521ac3 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "0.86.5", + "version": "0.87.0", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { From 31483f3ad42fd31bf406733c0945922dab0f24a7 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 08:35:44 -0700 Subject: [PATCH 046/142] change node engine version to drop supoprt for node 14 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6d7de255f72..c80facbbf97 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,7 @@ "registry": "https://registry.npmjs.org/" }, "engines": { - "node": ">=14.17.0", + "node": ">=16.19.0", "yarn": ">=1.22.19" }, "version": 1 From bee91365f78773c162a315cc23b3b587804ba367 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 10:48:36 -0700 Subject: [PATCH 047/142] update eslint to use latest parser --- packages/eslint-config/index.js | 2 +- packages/teraslice/lib/cluster/services/api.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/eslint-config/index.js b/packages/eslint-config/index.js index 7bf9cecb094..5f95233afb3 100644 --- a/packages/eslint-config/index.js +++ b/packages/eslint-config/index.js @@ -5,7 +5,7 @@ const { rules, overrides } = require('./lib'); module.exports = { extends: ['airbnb-base'], parserOptions: { - ecmaVersion: 2019, + ecmaVersion: 'latest', sourceType: 'script', }, env: { diff --git a/packages/teraslice/lib/cluster/services/api.js b/packages/teraslice/lib/cluster/services/api.js index 41d5f9752e3..d94b1af1de2 100644 --- a/packages/teraslice/lib/cluster/services/api.js +++ b/packages/teraslice/lib/cluster/services/api.js @@ -18,7 +18,6 @@ const { } = require('../../utils/api_utils'); const terasliceVersion = require('../../../package.json').version; - let gotESMModule; async function getGotESM() { @@ -494,7 +493,7 @@ module.exports = function apiService(context, { assetsUrl, app }) { } async function _redirect(req, res) { - const module = await getGotESM() + const module = await getGotESM(); const options = { prefixUrl: assetsUrl, headers: req.headers, From 918032a72d96518a96108630bc4166e59dec8ad8 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 10:48:59 -0700 Subject: [PATCH 048/142] bump: (minor) @terascope/eslint-config@0.8.0 --- packages/eslint-config/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eslint-config/package.json b/packages/eslint-config/package.json index 2e298aa8bc6..d6a7f48fa98 100644 --- a/packages/eslint-config/package.json +++ b/packages/eslint-config/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/eslint-config", "displayName": "Terascope ESLint Config", - "version": "0.7.1", + "version": "0.8.0", "description": "A shared eslint config based on eslint-config-airbnb", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/eslint-config#readme", "bugs": { From b05e1f530723356822827747b1e101f8662759cb Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Fri, 20 Oct 2023 11:31:15 -0700 Subject: [PATCH 049/142] fix test --- packages/eslint-config/test/index-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/eslint-config/test/index-spec.js b/packages/eslint-config/test/index-spec.js index 344dfbd5af8..1bff91e7307 100644 --- a/packages/eslint-config/test/index-spec.js +++ b/packages/eslint-config/test/index-spec.js @@ -12,7 +12,7 @@ describe('ESLint Config Index', () => { it('should export js parserOptions by default', () => { expect(Index).toHaveProperty('parserOptions'); expect(Index.parserOptions).toMatchObject({ - ecmaVersion: 2019, + ecmaVersion: 'latest', sourceType: 'script', },); }); From 85e0ac63fc280bcb597339f4b681182de840bdb6 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Mon, 23 Oct 2023 12:57:04 -0700 Subject: [PATCH 050/142] change default node version --- Dockerfile | 2 +- packages/scripts/src/helpers/config.ts | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a7ddfb5b162..78ead80ec7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # NODE_VERSION is set by default in the config.ts, the following value will only # be used if you build images by default with docker build -ARG NODE_VERSION=14.21.3 +ARG NODE_VERSION=18.16.0 FROM terascope/node-base:${NODE_VERSION} ENV NODE_ENV production diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 43c108d4390..889cce8bc74 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -157,6 +157,4 @@ export const SEARCH_TEST_HOST = testHost; // This should match a node version from the base-docker-image repo: // https://github.com/terascope/base-docker-image // This overrides the value in the Dockerfile -export const NODE_VERSION = process.env.NODE_VERSION || '14.21.3'; - -export const { TEST_PLATFORM = 'native' } = process.env; // FIXME: test this +export const NODE_VERSION = process.env.NODE_VERSION || '18.16.0'; From 93421d64142a607fe5d0dfee758af2f6dc8a9cf7 Mon Sep 17 00:00:00 2001 From: Jared Noble Date: Mon, 23 Oct 2023 12:57:19 -0700 Subject: [PATCH 051/142] bump: (minor) @terascope/scripts@0.60.0 --- packages/scripts/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 129e53b5335..cf4b187082f 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "0.59.0", + "version": "0.60.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { From 7509d0bc9483a0fad4b55d6fae1bae766b0c2d2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 21:18:15 +0000 Subject: [PATCH 052/142] Bump @babel/traverse from 7.22.5 to 7.23.2 Bumps [@babel/traverse](https://github.com/babel/babel/tree/HEAD/packages/babel-traverse) from 7.22.5 to 7.23.2. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.23.2/packages/babel-traverse) --- updated-dependencies: - dependency-name: "@babel/traverse" dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 81 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/yarn.lock b/yarn.lock index 2b966165886..91b133efeed 100644 --- a/yarn.lock +++ b/yarn.lock @@ -588,7 +588,7 @@ dependencies: "@babel/highlight" "^7.22.5" -"@babel/code-frame@^7.12.13": +"@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.13": version "7.22.13" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== @@ -632,6 +632,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-compilation-targets@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.5.tgz#fc7319fc54c5e2fa14b2909cf3c5fd3046813e02" @@ -643,18 +653,23 @@ lru-cache "^5.1.1" semver "^6.3.0" +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-environment-visitor@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98" integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q== -"@babel/helper-function-name@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz#ede300828905bb15e582c037162f99d5183af1be" - integrity sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ== +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== dependencies: - "@babel/template" "^7.22.5" - "@babel/types" "^7.22.5" + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" "@babel/helper-hoist-variables@^7.22.5": version "7.22.5" @@ -703,6 +718,13 @@ dependencies: "@babel/types" "^7.22.5" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-string-parser@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" @@ -741,6 +763,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.5.tgz#721fd042f3ce1896238cf1b341c77eb7dee7dbea" integrity sha512-DFZMC9LJUG9PLOclRC32G63UXwzqS2koQC8dkx+PLdmt1xSePYpbT/NbsrJy8Q/muXz7o/h/d4A7Fuyixm559Q== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" @@ -846,6 +873,15 @@ dependencies: regenerator-runtime "^0.13.11" +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + "@babel/template@^7.22.5", "@babel/template@^7.3.3": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.5.tgz#0c8c4d944509875849bd0344ff0050756eefc6ec" @@ -856,18 +892,18 @@ "@babel/types" "^7.22.5" "@babel/traverse@^7.22.5", "@babel/traverse@^7.7.2": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.5.tgz#44bd276690db6f4940fdb84e1cb4abd2f729ccd1" - integrity sha512-7DuIjPgERaNo6r+PZwItpjCZEa5vyw4eJGufeLxrPdBXBoLcCJCIasvK6pK/9DVNrLZTLFhUGqaC6X/PA007TQ== - dependencies: - "@babel/code-frame" "^7.22.5" - "@babel/generator" "^7.22.5" - "@babel/helper-environment-visitor" "^7.22.5" - "@babel/helper-function-name" "^7.22.5" + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" "@babel/helper-hoist-variables" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.5" - "@babel/parser" "^7.22.5" - "@babel/types" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -880,6 +916,15 @@ "@babel/helper-validator-identifier" "^7.22.5" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" From b54e70281e0f0da458b9a8951ed5909f9db7f116 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 12 Oct 2023 15:45:26 -0700 Subject: [PATCH 053/142] Build basic setup for k8s kind e2e tests. --- k8se2e/docker-compose.yml | 32 +++++ k8se2e/elasticsearchDeployment.yaml | 45 ++++++ k8se2e/jest.config.js | 7 + k8se2e/kindConfig.yaml | 10 ++ k8se2e/masterConfig/teraslice.yaml | 31 ++++ k8se2e/masterDeployment.yaml | 53 +++++++ k8se2e/ns.yaml | 4 + k8se2e/package.json | 55 +++++++ k8se2e/priorityClass.yaml | 7 + k8se2e/role.yaml | 9 ++ k8se2e/roleBinding.yaml | 13 ++ k8se2e/workerConfig/teraslice.yaml | 29 ++++ package.json | 1 + packages/scripts/src/helpers/packages.ts | 12 ++ packages/scripts/src/helpers/scripts.ts | 82 +++++++++++ .../scripts/src/helpers/test-runner/index.ts | 134 +++++++++++++++++- 16 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 k8se2e/docker-compose.yml create mode 100644 k8se2e/elasticsearchDeployment.yaml create mode 100644 k8se2e/jest.config.js create mode 100644 k8se2e/kindConfig.yaml create mode 100644 k8se2e/masterConfig/teraslice.yaml create mode 100644 k8se2e/masterDeployment.yaml create mode 100644 k8se2e/ns.yaml create mode 100644 k8se2e/package.json create mode 100644 k8se2e/priorityClass.yaml create mode 100644 k8se2e/role.yaml create mode 100644 k8se2e/roleBinding.yaml create mode 100644 k8se2e/workerConfig/teraslice.yaml diff --git a/k8se2e/docker-compose.yml b/k8se2e/docker-compose.yml new file mode 100644 index 00000000000..e8cd69ed047 --- /dev/null +++ b/k8se2e/docker-compose.yml @@ -0,0 +1,32 @@ +version: '2.4' +services: + teraslice-master: + image: teraslice-workspace:e2e + ports: + - '45678:45678' + scale: 1 + restart: 'no' + stop_grace_period: 30s + environment: + - TERAFOUNDATION_CONFIG=/app/config/teraslice-master.json + - NODE_OPTIONS=--max-old-space-size=256 + volumes: + - ./logs:/app/logs:delegated + - ./autoload:/app/autoload:ro + - ./.config:/app/config:ro + - ./.assets:/app/assets:delegated + teraslice-worker: + image: teraslice-workspace:e2e + scale: 2 + restart: 'no' + stop_grace_period: 30s + links: + - teraslice-master + environment: + - TERAFOUNDATION_CONFIG=/app/config/teraslice-worker.json + - NODE_OPTIONS=--max-old-space-size=256 + volumes: + - ./logs:/app/logs:delegated + - ./autoload:/app/autoload:ro + - ./.config:/app/config:ro + - ./.assets:/app/assets:delegated diff --git a/k8se2e/elasticsearchDeployment.yaml b/k8se2e/elasticsearchDeployment.yaml new file mode 100644 index 00000000000..5ea8287b1aa --- /dev/null +++ b/k8se2e/elasticsearchDeployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + nodeType: master + template: + metadata: + labels: + app: elasticsearch + nodeType: master + spec: + containers: + - name: elasticsearch + image: elasticsearch:7.9.3 + ports: + - containerPort: 9200 + env: + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: single-node +--- +kind: Service +apiVersion: v1 +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + nodeType: master + ports: + - port: 9200 + targetPort: 9200 + nodePort: 30200 # the external port teraslice can be accessed on + type: NodePort diff --git a/k8se2e/jest.config.js b/k8se2e/jest.config.js new file mode 100644 index 00000000000..0a4be3a992a --- /dev/null +++ b/k8se2e/jest.config.js @@ -0,0 +1,7 @@ +'use strict'; + +const config = require('../jest.config.base')(__dirname); + +config.collectCoverage = false; +delete config.transform; +module.exports = config; diff --git a/k8se2e/kindConfig.yaml b/k8se2e/kindConfig.yaml new file mode 100644 index 00000000000..e9d87e52e73 --- /dev/null +++ b/k8se2e/kindConfig.yaml @@ -0,0 +1,10 @@ +kind: Cluster +name: k8se2e +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30200 + hostPort: 9200 + - containerPort: 30678 + hostPort: 5678 diff --git a/k8se2e/masterConfig/teraslice.yaml b/k8se2e/masterConfig/teraslice.yaml new file mode 100644 index 00000000000..8f1aaddea3e --- /dev/null +++ b/k8se2e/masterConfig/teraslice.yaml @@ -0,0 +1,31 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: true + master_hostname: "127.0.0.1" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image_pull_secrets: + - "docker-tera1-secret" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 diff --git a/k8se2e/masterDeployment.yaml b/k8se2e/masterDeployment.yaml new file mode 100644 index 00000000000..14433977e66 --- /dev/null +++ b/k8se2e/masterDeployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: teraslice-master + labels: + app: teraslice + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: teraslice + nodeType: master + template: + metadata: + labels: + app: teraslice + nodeType: master + clusterName: ts-dev1 + spec: + containers: + - name: teraslice-master + image: teraslice-workspace:k8se2e + ports: + - containerPort: 5678 + volumeMounts: + - mountPath: /app/config # defines the directory + name: config + volumes: + - name: config + configMap: + name: teraslice-master + items: + - key: teraslice.yaml + path: teraslice.yaml # the filename that the configMap gets written to, inside the matching mountPath + imagePullSecrets: + - name: docker-tera1-secret +--- +kind: Service +apiVersion: v1 +metadata: + name: teraslice-master + labels: + app: teraslice +spec: + selector: + app: teraslice + nodeType: master + ports: + - port: 5678 + targetPort: 5678 + nodePort: 30678 # the external port teraslice can be accessed on + type: NodePort diff --git a/k8se2e/ns.yaml b/k8se2e/ns.yaml new file mode 100644 index 00000000000..1af803c0c4f --- /dev/null +++ b/k8se2e/ns.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: ts-dev1 \ No newline at end of file diff --git a/k8se2e/package.json b/k8se2e/package.json new file mode 100644 index 00000000000..0b261e4873f --- /dev/null +++ b/k8se2e/package.json @@ -0,0 +1,55 @@ +{ + "name": "k8se2e", + "displayName": "K8S E2E Tests", + "version": "0.0.1", + "private": true, + "description": "Teraslice in Kubernetes integration test suite", + "keywords": [ // ??? + "docker-compose", + "elasticsearch", + "teraslice" + ], + "homepage": "https://github.com/terascope/teraslice/tree/master/k8se2e/#readme", + "bugs": { + "url": "https://github.com/terascope/teraslice/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/terascope/teraslice.git" + }, + "license": "MIT", + "author": "Terascope, LLC ", + "scripts": { // check these + "clean": "docker-compose down --volumes --remove-orphans --timeout=5", + "logs": "./scripts/logs.sh", + "logs-follow": "./scripts/logs.sh -f", + "setup": "yarn --silent", + "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --debug --", + "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", + "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --watch --" + }, + "resolutions": { // ??? + "ms": "^2.1.3" + }, + "devDependencies": { // ??? + "bunyan": "^1.8.15", + "elasticsearch-store": "^0.72.0", + "fs-extra": "^10.1.0", + "ms": "^2.1.3", + "nanoid": "^3.3.4", + "semver": "^7.3.8", + "signale": "^1.4.0", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.17.0", + "yarn": ">=1.16.0" + }, + "terascope": { + "testSuite": "k8se2e" + } +} diff --git a/k8se2e/priorityClass.yaml b/k8se2e/priorityClass.yaml new file mode 100644 index 00000000000..0747825c87d --- /dev/null +++ b/k8se2e/priorityClass.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000000 +globalDefault: false +description: "This priority class is for Teraslice pods." diff --git a/k8se2e/role.yaml b/k8se2e/role.yaml new file mode 100644 index 00000000000..725cec875a5 --- /dev/null +++ b/k8se2e/role.yaml @@ -0,0 +1,9 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] \ No newline at end of file diff --git a/k8se2e/roleBinding.yaml b/k8se2e/roleBinding.yaml new file mode 100644 index 00000000000..b46683aeb41 --- /dev/null +++ b/k8se2e/roleBinding.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +subjects: + - kind: ServiceAccount + name: default + namespace: ts-dev1 +roleRef: + kind: Role + name: teraslice-all-ts-dev1 + apiGroup: "rbac.authorization.k8s.io" \ No newline at end of file diff --git a/k8se2e/workerConfig/teraslice.yaml b/k8se2e/workerConfig/teraslice.yaml new file mode 100644 index 00000000000..6615cc43757 --- /dev/null +++ b/k8se2e/workerConfig/teraslice.yaml @@ -0,0 +1,29 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: false + master_hostname: "teraslice-master" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 diff --git a/package.json b/package.json index a3cb7e0a674..6fa1e5ba81e 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "tests": { "suites": { "e2e": [], + "k8se2e": [], "elasticsearch": [], "search": [], "restrained": [], diff --git a/packages/scripts/src/helpers/packages.ts b/packages/scripts/src/helpers/packages.ts index 0e7197730f4..75ae9b384de 100644 --- a/packages/scripts/src/helpers/packages.ts +++ b/packages/scripts/src/helpers/packages.ts @@ -15,6 +15,7 @@ import * as i from './interfaces'; let _packages: i.PackageInfo[] = []; let _e2eDir: string|undefined; +let _k8se2eDir: string|undefined; export function getE2EDir(): string|undefined { if (_e2eDir) return _e2eDir; @@ -27,6 +28,17 @@ export function getE2EDir(): string|undefined { return undefined; } +export function getK8SE2EDir(): string|undefined { + if (_k8se2eDir) return _k8se2eDir; + + if (fs.existsSync(path.join(misc.getRootDir(), 'k8se2e'))) { + _k8se2eDir = path.join(misc.getRootDir(), 'k8se2e'); + return _k8se2eDir; + } + + return undefined; +} + function _loadPackage(packagePath: string): i.PackageInfo|undefined { const pkgJsonPath = path.join(packagePath, 'package.json'); if (fs.existsSync(pkgJsonPath)) { diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 3118ed56c26..4c97be2f891 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -532,3 +532,85 @@ export async function yarnPublish( } }); } + +export async function createKindCluster( + k8se2eDir: string, + kindConfigFileName: string +): Promise { + const configPath = path.join(k8se2eDir, kindConfigFileName); + const subprocess = await execa.command(`kind create cluster --config ${configPath}`); + signale.log(subprocess.stderr); + // const test = await execa.command('echo hello'); + // console.log('test: ', test); +} + +export async function destroyKindCluster(): Promise { + // TODO: pass cluster name in as variable + const subprocess = await execa.command('kind delete cluster --name k8se2e'); + signale.log(subprocess.stderr); +} + +export async function isKindInstalled(): Promise { + try { + const subprocess = await execa.command('command -v kind'); + // console.log('kind subprocess: ', subprocess); + return !!subprocess.stdout; + } catch (err) { + // console.log(err); + return false; + } +} + +export async function isKubectlInstalled(): Promise { + try { + const subprocess = await execa.command('command -v kubectl'); + // console.log('kubectl subprocess: ', subprocess); + return !!subprocess.stdout; + } catch (err) { + return false; + } +} + +export async function loadTerasliceImage(terasliceImage: string): Promise { + const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); + console.log('load teraslice image subprocess: ', subprocess); +} + +export async function createNamespace() { + const subprocess = await execa.command('kubectl create namespace ts-dev1'); + console.log('namespace subprocess: ', subprocess); +} + +export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: string) { + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); + console.log('esDeploy subprocess: ', subprocess); +} + +export async function k8sSetup( + k8se2eDir: string, + roleYaml: string, + roleBindingYaml: string, + priorityClassYaml: string +): Promise { + const subprocess1 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleYaml)}`); + const subprocess2 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleBindingYaml)}`); + const subprocess3 = await execa.command(`kubectl apply -f ${path.join(k8se2eDir, priorityClassYaml)}`); + console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); +} + +export async function deployk8sTeraslice( + k8se2eDir: string, + masterDeploymentYaml: string +) { + /// Creates configmap for terasclice-master + let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(k8se2eDir, 'masterConfig', 'teraslice.yaml')}`); + console.log('masterConfig subprocess: ', subprocess); + + /// Creates configmap for teraslice-worker + subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(k8se2eDir, 'workerConfig', 'teraslice.yaml')}`); + console.log('workerConfig subprocess: ', subprocess); + + /// Creates deployment for teraslice + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); + console.log('masterDeploy subprocess: ', subprocess); +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 45d26a3628c..8307176728a 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -9,20 +9,35 @@ import { import { ensureServices, pullServices } from './services'; import { PackageInfo } from '../interfaces'; import { TestOptions } from './interfaces'; -import { runJest, dockerTag } from '../scripts'; +import { + createKindCluster, + destroyKindCluster, + loadTerasliceImage, + createNamespace, + deployElasticSearch, + k8sSetup, + deployk8sTeraslice, + runJest, + dockerTag, + isKindInstalled, + isKubectlInstalled +} from '../scripts'; import { getArgs, filterBySuite, globalTeardown, reportCoverage, logE2E, getEnv, groupBySuite } from './utils'; import signale from '../signale'; -import { getE2EDir, readPackageInfo, listPackages } from '../packages'; +import { + getE2EDir, readPackageInfo, listPackages, getK8SE2EDir +} from '../packages'; import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; import { TestTracker } from './tracker'; import { MAX_PROJECTS_PER_BATCH, - SKIP_DOCKER_BUILD_IN_E2E + SKIP_DOCKER_BUILD_IN_E2E, + SKIP_DOCKER_BUILD_K8S_E2E } from '../config'; const logger = debugLogger('ts-scripts:cmd:test'); @@ -65,6 +80,11 @@ async function _runTests( return; } + if (options.suite?.includes('k8se2e')) { + await runk8sE2ETest(options, tracker); + return; + } + const filtered = filterBySuite(pkgInfos, options); if (!filtered.length) { signale.warn('No tests found.'); @@ -300,3 +320,111 @@ function printAndGetEnv(suite: string, options: TestOptions) { } return env; } + +async function runk8sE2ETest( + options: TestOptions, tracker: TestTracker +): Promise { + console.log('options: ', options); + tracker.expected++; + + const k8se2eDir = getK8SE2EDir(); + if (!k8se2eDir) { + throw new Error('Missing k8se2e test directory'); + } + + const kindInstalled = await isKindInstalled(); + if (!kindInstalled) { + signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); + process.exit(1); + } + + const kubectlInstalled = await isKubectlInstalled(); + if (!kubectlInstalled) { + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + process.exit(1); + } + // TODO: pass kind config file in as a variable + await createKindCluster(k8se2eDir, 'kindConfig.yaml'); + + const suite = 'k8se2e'; + let startedTest = false; + + const rootInfo = getRootInfo(); + const k8se2eImage = `${rootInfo.name}:k8se2e`; + + // if (isCI) { + // // pull the services first in CI + // await pullServices(suite, options); + // } + + try { + if (SKIP_DOCKER_BUILD_K8S_E2E) { + const devImage = getDevDockerImage(); + await dockerTag(devImage, k8se2eImage); + await loadTerasliceImage(k8se2eImage); + } else { + const devImage = await buildDevDockerImage(); + await dockerTag(devImage, k8se2eImage); + await loadTerasliceImage(k8se2eImage); + } + } catch (err) { + tracker.addError(err); + } + + // TODO: add tracker + await createNamespace(); + await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); + await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); + await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); + + if (!tracker.hasErrors()) { + const timeLabel = `test suite "${suite}"`; + signale.time(timeLabel); + startedTest = true; + + const env = printAndGetEnv(suite, options); + + tracker.started++; + try { + await runJest( + k8se2eDir, + getArgs(options), + env, + options.jestArgs, + options.debug + ); + tracker.ended++; + } catch (err) { + tracker.ended++; + tracker.addError(err.message); + } + + signale.timeEnd(timeLabel); + } + + if (!startedTest) return; + + // if (!options.keepOpen) { + // try { + // await logE2E(k8se2eDir, tracker.hasErrors()); + // } catch (err) { + // signale.error( + // new TSError(err, { + // reason: `Writing the "${suite}" logs failed`, + // }) + // ); + // } + // } + + // if (tracker.hasErrors()) { + // tracker.addCleanup('e2e:teardown', async () => { + // options.keepOpen = false; + // await globalTeardown(options, [{ + // name: suite, + // dir: k8se2eDir, + // suite, + // }]); + // }); + // } + // await destroyKindCluster(); +} From 7e7617d5975195c416f624838cc61abf6dbe0b25 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 13 Oct 2023 07:42:07 -0700 Subject: [PATCH 054/142] update buildDevDockerImage to take node version. --- k8se2e/package.json | 8 +-- k8se2e/testJob.json | 12 ++++ k8se2e/testReaderFile.json | 8 +++ packages/scripts/src/helpers/config.ts | 2 + packages/scripts/src/helpers/scripts.ts | 16 +++++ .../scripts/src/helpers/test-runner/index.ts | 60 +++++++++++-------- 6 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 k8se2e/testJob.json create mode 100644 k8se2e/testReaderFile.json diff --git a/k8se2e/package.json b/k8se2e/package.json index 0b261e4873f..64302c49c46 100644 --- a/k8se2e/package.json +++ b/k8se2e/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "private": true, "description": "Teraslice in Kubernetes integration test suite", - "keywords": [ // ??? + "keywords": [ "docker-compose", "elasticsearch", "teraslice" @@ -19,7 +19,7 @@ }, "license": "MIT", "author": "Terascope, LLC ", - "scripts": { // check these + "scripts": { "clean": "docker-compose down --volumes --remove-orphans --timeout=5", "logs": "./scripts/logs.sh", "logs-follow": "./scripts/logs.sh -f", @@ -32,10 +32,10 @@ "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --watch --" }, - "resolutions": { // ??? + "resolutions": { "ms": "^2.1.3" }, - "devDependencies": { // ??? + "devDependencies": { "bunyan": "^1.8.15", "elasticsearch-store": "^0.72.0", "fs-extra": "^10.1.0", diff --git a/k8se2e/testJob.json b/k8se2e/testJob.json new file mode 100644 index 00000000000..f688bdcb954 --- /dev/null +++ b/k8se2e/testJob.json @@ -0,0 +1,12 @@ +{ + "name": "Test Job", + "lifecycle": "once", + "workers": 1, + "assets": [], + "operations": [ + { + "_op": "test-reader", + "fetcher_data_file_path": "./testReaderFile.json" + } + ] +} diff --git a/k8se2e/testReaderFile.json b/k8se2e/testReaderFile.json new file mode 100644 index 00000000000..b0126017515 --- /dev/null +++ b/k8se2e/testReaderFile.json @@ -0,0 +1,8 @@ +[ + { + "foo": "bar" + }, + { + "foo": "baz" + } +] diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 889cce8bc74..e6434a496d1 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -98,6 +98,8 @@ export const DEV_DOCKER_IMAGE = process.env.DEV_DOCKER_IMAGE || undefined; */ export const SKIP_DOCKER_BUILD_IN_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_IN_E2E ?? false); +export const SKIP_DOCKER_BUILD_K8S_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_K8S_E2E ?? false); + export const SKIP_E2E_OUTPUT_LOGS = toBoolean(process.env.SKIP_E2E_OUTPUT_LOGS ?? !isCI); /** diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 4c97be2f891..1e0e3670129 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -614,3 +614,19 @@ export async function deployk8sTeraslice( subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); console.log('masterDeploy subprocess: ', subprocess); } + +export async function setAlias() { + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); + console.log('setAlias subprocess: ', subprocess1, subprocess2); +} + +export async function registerTestJob(k8se2eDir: string) { + const subprocess = await execa.command(`earl tjm register localhost ${path.join(k8se2eDir, 'testJob.json')}`); + console.log('registerTestJob subprocess: ', subprocess); +} + +export async function startTestJob(k8se2eDir: string) { + const subprocess = await execa.command(`earl tjm start ${path.join(k8se2eDir, 'testJob.json')}`); + console.log('registerTestJob subprocess: ', subprocess); +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 8307176728a..39da75774b1 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -20,7 +20,9 @@ import { runJest, dockerTag, isKindInstalled, - isKubectlInstalled + isKubectlInstalled, + registerTestJob, + startTestJob } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -359,11 +361,16 @@ async function runk8sE2ETest( try { if (SKIP_DOCKER_BUILD_K8S_E2E) { - const devImage = getDevDockerImage(); + const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; await dockerTag(devImage, k8se2eImage); await loadTerasliceImage(k8se2eImage); } else { - const devImage = await buildDevDockerImage(); + const publishOptions: PublishOptions = { + dryRun: true, + nodeVersion: options.nodeVersion, + type: PublishType.Dev + }; + const devImage = await buildDevDockerImage(publishOptions); await dockerTag(devImage, k8se2eImage); await loadTerasliceImage(k8se2eImage); } @@ -377,32 +384,35 @@ async function runk8sE2ETest( await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); - if (!tracker.hasErrors()) { - const timeLabel = `test suite "${suite}"`; - signale.time(timeLabel); - startedTest = true; + // await registerTestJob(k8se2eDir); + // await startTestJob(k8se2eDir); - const env = printAndGetEnv(suite, options); + // if (!tracker.hasErrors()) { + // const timeLabel = `test suite "${suite}"`; + // signale.time(timeLabel); + // startedTest = true; - tracker.started++; - try { - await runJest( - k8se2eDir, - getArgs(options), - env, - options.jestArgs, - options.debug - ); - tracker.ended++; - } catch (err) { - tracker.ended++; - tracker.addError(err.message); - } + // const env = printAndGetEnv(suite, options); - signale.timeEnd(timeLabel); - } + // tracker.started++; + // try { + // await runJest( + // k8se2eDir, + // getArgs(options), + // env, + // options.jestArgs, + // options.debug + // ); + // tracker.ended++; + // } catch (err) { + // tracker.ended++; + // tracker.addError(err.message); + // } - if (!startedTest) return; + // signale.timeEnd(timeLabel); + // } + + // if (!startedTest) return; // if (!options.keepOpen) { // try { From 869caf94fa8673da69cfcad1242d40e3857f090c Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 13 Oct 2023 13:13:24 -0700 Subject: [PATCH 055/142] test that elasticsearch and teraslice are running. --- packages/scripts/src/helpers/scripts.ts | 93 ++++++++++++++++++- .../scripts/src/helpers/test-runner/index.ts | 8 +- 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 1e0e3670129..ddf1871072a 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -14,6 +14,7 @@ import { TSCommands, PackageInfo } from './interfaces'; import { getRootDir } from './misc'; import signale from './signale'; import * as config from './config'; +import { setTimeout } from 'timers'; const logger = debugLogger('ts-scripts:cmd'); @@ -584,6 +585,23 @@ export async function createNamespace() { export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: string) { const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); console.log('esDeploy subprocess: ', subprocess); + + let elasticsearchRunning = false; + // TODO: add curl check to ES before continuing + while (!elasticsearchRunning) { + const ESResponse = execa.command('curl localhost:9200').stdout; + if (ESResponse) { + ESResponse.on('data', (data) => { + const jsonData = JSON.parse(data); + console.log(`response: ${JSON.stringify(jsonData)}`); + if (jsonData.tagline === 'You Know, for Search') { + console.log('jsonData.tagline === You Know, for Search') + elasticsearchRunning = true; + } + }); + } + await PromiseTimeout(3000); + } } export async function k8sSetup( @@ -613,6 +631,62 @@ export async function deployk8sTeraslice( /// Creates deployment for teraslice subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); console.log('masterDeploy subprocess: ', subprocess); + + subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); + const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); + console.log('podName: ', podName); + + // await waitForTerasliceRunning(120000); + + let terasliceRunning = false; + + while (!terasliceRunning) { + const TSMasterResponse = execa.command('curl localhost:5678').stdout; + if (TSMasterResponse) { + TSMasterResponse.on('data', (data) => { + const jsonData = JSON.parse(data); + console.log(`response: ${JSON.stringify(jsonData)}`); + if (jsonData.clustering_type === 'kubernetes') { + console.log('jsonData.clusteringType === kubernetes') + terasliceRunning = true; + } + }); + } + await PromiseTimeout(3000); + } + // function waitForTerasliceRunning(timeoutMs = 120000) : Promise { + // const endAt = Date.now() + timeoutMs; + + // const _waitForTerasliceRunning = async () : Promise => { + // if (Date.now() > endAt) { + // throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); + // } + + // let terasliceRunning = false; + // // let nodes = -1; + // try { + // const TSMasterResponse = execa.command('curl localhost:5678').stdout; + // if (TSMasterResponse) { + // TSMasterResponse.on('data', (data) => { + // const jsonData = JSON.parse(data); + // console.log(`response: ${JSON.stringify(jsonData)}`); + // if (jsonData.clustering_type === 'kubernetes') { + // console.log('jsonData.clusteringType === kubernetes'); + // terasliceRunning = true; + // } + // }); + // } + // } catch (err) { + // await PromiseTimeout(3000); + // return _waitForTerasliceRunning(); + // } + + // if (terasliceRunning) return true; + // return _waitForTerasliceRunning(); + // }; + + // return _waitForTerasliceRunning(); + // } } export async function setAlias() { @@ -621,12 +695,23 @@ export async function setAlias() { console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function registerTestJob(k8se2eDir: string) { - const subprocess = await execa.command(`earl tjm register localhost ${path.join(k8se2eDir, 'testJob.json')}`); +export async function registerTestJob() { + const subprocess = await execa.command('earl tjm register localhost testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } -export async function startTestJob(k8se2eDir: string) { - const subprocess = await execa.command(`earl tjm start ${path.join(k8se2eDir, 'testJob.json')}`); +export async function startTestJob() { + const subprocess = await execa.command('earl tjm start testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } + +export async function showState() { + const subprocess = await execa.command('kubectl -n ts-dev1 get deployments,po,svc -o wide'); + console.log('showState subprocess: ', subprocess); +} + +function PromiseTimeout(delayms: number) { + return new Promise((resolve) => { + setTimeout(resolve, delayms); + }); +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 39da75774b1..ceba05ba9bd 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -22,7 +22,8 @@ import { isKindInstalled, isKubectlInstalled, registerTestJob, - startTestJob + startTestJob, + showState } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -383,9 +384,10 @@ async function runk8sE2ETest( await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); + await showState(); - // await registerTestJob(k8se2eDir); - // await startTestJob(k8se2eDir); + await registerTestJob(); + await startTestJob(); // if (!tracker.hasErrors()) { // const timeLabel = `test suite "${suite}"`; From 4f811f447b717848d6226faae7a35c115ccd6724 Mon Sep 17 00:00:00 2001 From: sotojn Date: Fri, 13 Oct 2023 15:04:09 -0700 Subject: [PATCH 056/142] update testReaderFile & testJob for k8s e2e --- k8se2e/testJob.json | 22 ++++++++++-- k8se2e/testReaderFile.json | 72 ++++++++++++++++++++++++++++++++++---- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/k8se2e/testJob.json b/k8se2e/testJob.json index f688bdcb954..10baeb47de4 100644 --- a/k8se2e/testJob.json +++ b/k8se2e/testJob.json @@ -2,11 +2,27 @@ "name": "Test Job", "lifecycle": "once", "workers": 1, - "assets": [], + "assets": [ + "elasticsearch" + ], "operations": [ { "_op": "test-reader", - "fetcher_data_file_path": "./testReaderFile.json" + "fetcher_data_file_path": "./k8se2e/testReaderFile.json" + }, + { + "_op": "elasticsearch_bulk", + "index": "terak8s-example-data", + "type": "events", + "size": 1 + } + ], + "__metadata": { + "cli": { + "cluster": "http://localhost:5678", + "version": "0.55.1", + "job_id": "7c30c1f9-31b1-4ec9-b5c0-da4624fed030", + "updated": "2023-10-13T21:58:14.228Z" } - ] + } } diff --git a/k8se2e/testReaderFile.json b/k8se2e/testReaderFile.json index b0126017515..bada14e3ebf 100644 --- a/k8se2e/testReaderFile.json +++ b/k8se2e/testReaderFile.json @@ -1,8 +1,68 @@ [ - { - "foo": "bar" - }, - { - "foo": "baz" - } + { "name": "John Doe", "age": 44, "accountId": 43394838 }, + { "name": "Jane Smith", "age": 28, "accountId": 82736482 }, + { "name": "Michael Johnson", "age": 35, "accountId": 12874938 }, + { "name": "Emily Davis", "age": 52, "accountId": 29384729 }, + { "name": "William Brown", "age": 29, "accountId": 74839274 }, + { "name": "Sarah Wilson", "age": 41, "accountId": 49283747 }, + { "name": "David Lee", "age": 37, "accountId": 23847283 }, + { "name": "Jennifer Harris", "age": 32, "accountId": 58374635 }, + { "name": "Joseph Clark", "age": 50, "accountId": 37483748 }, + { "name": "Linda Lewis", "age": 45, "accountId": 93847564 }, + { "name": "Robert Hall", "age": 33, "accountId": 47298374 }, + { "name": "Karen Turner", "age": 49, "accountId": 73847293 }, + { "name": "Richard Garcia", "age": 39, "accountId": 94872347 }, + { "name": "Patricia Rodriguez", "age": 56, "accountId": 64827364 }, + { "name": "Charles Martinez", "age": 43, "accountId": 87348634 }, + { "name": "Nancy Jackson", "age": 30, "accountId": 74623847 }, + { "name": "Thomas Allen", "age": 47, "accountId": 38274692 }, + { "name": "Betty Anderson", "age": 36, "accountId": 92384728 }, + { "name": "Daniel Moore", "age": 53, "accountId": 37483947 }, + { "name": "Deborah White", "age": 40, "accountId": 48374839 }, + { "name": "Paul Martin", "age": 31, "accountId": 73847283 }, + { "name": "Sandra Taylor", "age": 55, "accountId": 23847236 }, + { "name": "Mark Scott", "age": 38, "accountId": 82734628 }, + { "name": "Laura King", "age": 34, "accountId": 74628374 }, + { "name": "Kevin Young", "age": 46, "accountId": 23846283 }, + { "name": "Cynthia Turner", "age": 42, "accountId": 27486374 }, + { "name": "Christopher Thomas", "age": 48, "accountId": 34872634 }, + { "name": "Susan Adams", "age": 29, "accountId": 78374628 }, + { "name": "Kenneth Walker", "age": 57, "accountId": 87263847 }, + { "name": "Ashley Baker", "age": 33, "accountId": 37462836 }, + { "name": "George White", "age": 52, "accountId": 48364782 }, + { "name": "Jessica Robinson", "age": 35, "accountId": 34628347 }, + { "name": "Edward Perez", "age": 41, "accountId": 82736482 }, + { "name": "Amanda Mitchell", "age": 44, "accountId": 73647283 }, + { "name": "Brian Turner", "age": 37, "accountId": 48327462 }, + { "name": "Maria Evans", "age": 48, "accountId": 38274636 }, + { "name": "James Davis", "age": 31, "accountId": 47283947 }, + { "name": "Karen Hall", "age": 56, "accountId": 74638274 }, + { "name": "Jason Martinez", "age": 39, "accountId": 28374683 }, + { "name": "Donna Johnson", "age": 32, "accountId": 74826374 }, + { "name": "Ronald Brown", "age": 50, "accountId": 82734682 }, + { "name": "Carol Lewis", "age": 43, "accountId": 34786374 }, + { "name": "William Taylor", "age": 38, "accountId": 73647283 }, + { "name": "Sharon Robinson", "age": 47, "accountId": 47364826 }, + { "name": "Michael Scott", "age": 30, "accountId": 82374683 }, + { "name": "Ruth Harris", "age": 45, "accountId": 48374628 }, + { "name": "Kevin Anderson", "age": 42, "accountId": 73648236 }, + { "name": "Deborah Perez", "age": 53, "accountId": 23874623 }, + { "name": "Richard Turner", "age": 34, "accountId": 37482364 }, + { "name": "Lisa Baker", "age": 49, "accountId": 48236478 }, + { "name": "David Mitchell", "age": 46, "accountId": 37468236 }, + { "name": "Mary Adams", "age": 52, "accountId": 84623847 }, + { "name": "Joseph King", "age": 41, "accountId": 23874623 }, + { "name": "Susan Harris", "age": 37, "accountId": 47364826 }, + { "name": "Christopher Moore", "age": 35, "accountId": 73648236 }, + { "name": "Patricia Davis", "age": 54, "accountId": 36478236 }, + { "name": "Matthew Garcia", "age": 44, "accountId": 74823647 }, + { "name": "Linda Turner", "age": 33, "accountId": 82364826 }, + { "name": "Charles Scott", "age": 47, "accountId": 36478236 }, + { "name": "Nancy Robinson", "age": 36, "accountId": 74826347 }, + { "name": "Daniel Evans", "age": 51, "accountId": 48374682 }, + { "name": "Betty Perez", "age": 40, "accountId": 73648236 }, + { "name": "Michael Lewis", "age": 38, "accountId": 48364782 }, + { "name": "Karen Hall", "age": 46, "accountId": 36478236 }, + { "name": "William Davis", "age": 49, "accountId": 74823647 } + ] From e3673a9e5db322beacacef8204d4aec0b3d32bd5 Mon Sep 17 00:00:00 2001 From: sotojn Date: Mon, 16 Oct 2023 08:26:57 -0700 Subject: [PATCH 057/142] add setAlias and registerElasticsearch method to k8e2e tests. Update Dockerfile --- Dockerfile | 1 + packages/scripts/src/helpers/scripts.ts | 13 ++++++++++--- packages/scripts/src/helpers/test-runner/index.ts | 6 +++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78ead80ec7a..62cc30f8c9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ COPY .yarn /app/source/.yarn COPY packages /app/source/packages COPY scripts /app/source/scripts COPY types /app/source/types +COPY k8se2e /app/source/k8se2e RUN yarn --prod=false --frozen-lockfile \ && yarn build \ diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index ddf1871072a..e87624261d5 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -690,19 +690,26 @@ export async function deployk8sTeraslice( } export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + // const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } export async function registerTestJob() { - const subprocess = await execa.command('earl tjm register localhost testJob.json'); + const subprocess = await execa.command('earl tjm register ts-k8s-e2e testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } +export async function registerElasticsearch() { + const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/elasticsearch-assets'); + console.log('registerElasticsearch asset subprocess: ', subprocess); +} + export async function startTestJob() { const subprocess = await execa.command('earl tjm start testJob.json'); - console.log('registerTestJob subprocess: ', subprocess); + console.log('Run earl tjm start testJob.json'); + console.log(subprocess.stdout); } export async function showState() { diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index ceba05ba9bd..9eb6aa55255 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -23,7 +23,9 @@ import { isKubectlInstalled, registerTestJob, startTestJob, - showState + showState, + registerElasticsearch, + setAlias } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -386,6 +388,8 @@ async function runk8sE2ETest( await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); await showState(); + await setAlias(); + await registerElasticsearch(); await registerTestJob(); await startTestJob(); From 5431ddc96e5983357c2bf6df9468e1ef0dd4bd0d Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 16 Oct 2023 08:40:04 -0700 Subject: [PATCH 058/142] Rework waitForElasticsearch and waitForTeraslice --- packages/scripts/src/helpers/scripts.ts | 125 +++++++++++++----------- 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index e87624261d5..ff4b1e3c0ba 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -14,7 +14,6 @@ import { TSCommands, PackageInfo } from './interfaces'; import { getRootDir } from './misc'; import signale from './signale'; import * as config from './config'; -import { setTimeout } from 'timers'; const logger = debugLogger('ts-scripts:cmd'); @@ -586,22 +585,44 @@ export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); console.log('esDeploy subprocess: ', subprocess); - let elasticsearchRunning = false; - // TODO: add curl check to ES before continuing - while (!elasticsearchRunning) { - const ESResponse = execa.command('curl localhost:9200').stdout; - if (ESResponse) { - ESResponse.on('data', (data) => { - const jsonData = JSON.parse(data); + const elasticsearchReady = await waitForESRunning(240000); + if (elasticsearchReady) { + signale.success('Elasticsearch is ready to go'); + } +} + +function waitForESRunning(timeoutMs = 120000): Promise { + const endAt = Date.now() + timeoutMs; + + const _waitForESRunning = async (): Promise => { + if (Date.now() > endAt) { + throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); + } + + let elasticsearchRunning = false; + try { + const ESResponse = await execa.command('curl localhost:9200'); + if (ESResponse.stdout) { + const jsonData = JSON.parse(ESResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); if (jsonData.tagline === 'You Know, for Search') { - console.log('jsonData.tagline === You Know, for Search') + console.log('jsonData.tagline === You Know, for Search'); elasticsearchRunning = true; } - }); + } + } catch (err) { + await PromiseTimeout(3000); + return _waitForESRunning(); + } + + if (elasticsearchRunning) { + return true; } await PromiseTimeout(3000); - } + return _waitForESRunning(); + }; + + return _waitForESRunning(); } export async function k8sSetup( @@ -636,66 +657,58 @@ export async function deployk8sTeraslice( const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); console.log('podName: ', podName); - // await waitForTerasliceRunning(120000); + const terasliceReady = await waitForTerasliceRunning(240000); + if (terasliceReady) { + signale.success('Teraslice is ready to go'); + } +} + +function waitForTerasliceRunning(timeoutMs = 120000): Promise { + const endAt = Date.now() + timeoutMs; - let terasliceRunning = false; + const _waitForTerasliceRunning = async (): Promise => { + if (Date.now() > endAt) { + throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); + } - while (!terasliceRunning) { - const TSMasterResponse = execa.command('curl localhost:5678').stdout; - if (TSMasterResponse) { - TSMasterResponse.on('data', (data) => { - const jsonData = JSON.parse(data); + let terasliceRunning = false; + try { + const TSMasterResponse = await execa.command('curl localhost:5678'); + if (TSMasterResponse.stdout) { + const jsonData = JSON.parse(TSMasterResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); if (jsonData.clustering_type === 'kubernetes') { - console.log('jsonData.clusteringType === kubernetes') + console.log('jsonData.clusteringType === kubernetes'); terasliceRunning = true; } - }); + } + } catch (err) { + await PromiseTimeout(3000); + return _waitForTerasliceRunning(); + } + + if (terasliceRunning) { + return true; } + await PromiseTimeout(3000); - } - // function waitForTerasliceRunning(timeoutMs = 120000) : Promise { - // const endAt = Date.now() + timeoutMs; - - // const _waitForTerasliceRunning = async () : Promise => { - // if (Date.now() > endAt) { - // throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); - // } - - // let terasliceRunning = false; - // // let nodes = -1; - // try { - // const TSMasterResponse = execa.command('curl localhost:5678').stdout; - // if (TSMasterResponse) { - // TSMasterResponse.on('data', (data) => { - // const jsonData = JSON.parse(data); - // console.log(`response: ${JSON.stringify(jsonData)}`); - // if (jsonData.clustering_type === 'kubernetes') { - // console.log('jsonData.clusteringType === kubernetes'); - // terasliceRunning = true; - // } - // }); - // } - // } catch (err) { - // await PromiseTimeout(3000); - // return _waitForTerasliceRunning(); - // } - - // if (terasliceRunning) return true; - // return _waitForTerasliceRunning(); - // }; - - // return _waitForTerasliceRunning(); - // } + return _waitForTerasliceRunning(); + }; + + return _waitForTerasliceRunning(); } export async function setAlias() { - // const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } +export async function deployESAsset() { + const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --blocking --bundle terascope/elasticsearch-assets'); + console.log('deployESAsset subprocess: ', subprocess); +} + export async function registerTestJob() { const subprocess = await execa.command('earl tjm register ts-k8s-e2e testJob.json'); console.log('registerTestJob subprocess: ', subprocess); From 41c37546ce70f3707994b97111b31aaf83a08c34 Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 16 Oct 2023 09:03:58 -0700 Subject: [PATCH 059/142] Fix merge error in setAlias function. Use shell. --- packages/scripts/src/helpers/scripts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index ff4b1e3c0ba..45a3b465da3 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -699,7 +699,7 @@ function waitForTerasliceRunning(timeoutMs = 120000): Promise { } export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true'); + const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } From 5de2808ed8514ef32c82a02154947b1827b571ac Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 08:20:55 -0700 Subject: [PATCH 060/142] Remove k8se2e directory and all its files --- k8se2e/docker-compose.yml | 32 -------------- k8se2e/elasticsearchDeployment.yaml | 45 ------------------- k8se2e/jest.config.js | 7 --- k8se2e/kindConfig.yaml | 10 ----- k8se2e/masterConfig/teraslice.yaml | 31 ------------- k8se2e/masterDeployment.yaml | 53 ---------------------- k8se2e/ns.yaml | 4 -- k8se2e/package.json | 55 ----------------------- k8se2e/priorityClass.yaml | 7 --- k8se2e/role.yaml | 9 ---- k8se2e/roleBinding.yaml | 13 ------ k8se2e/testJob.json | 28 ------------ k8se2e/testReaderFile.json | 68 ----------------------------- k8se2e/workerConfig/teraslice.yaml | 29 ------------ 14 files changed, 391 deletions(-) delete mode 100644 k8se2e/docker-compose.yml delete mode 100644 k8se2e/elasticsearchDeployment.yaml delete mode 100644 k8se2e/jest.config.js delete mode 100644 k8se2e/kindConfig.yaml delete mode 100644 k8se2e/masterConfig/teraslice.yaml delete mode 100644 k8se2e/masterDeployment.yaml delete mode 100644 k8se2e/ns.yaml delete mode 100644 k8se2e/package.json delete mode 100644 k8se2e/priorityClass.yaml delete mode 100644 k8se2e/role.yaml delete mode 100644 k8se2e/roleBinding.yaml delete mode 100644 k8se2e/testJob.json delete mode 100644 k8se2e/testReaderFile.json delete mode 100644 k8se2e/workerConfig/teraslice.yaml diff --git a/k8se2e/docker-compose.yml b/k8se2e/docker-compose.yml deleted file mode 100644 index e8cd69ed047..00000000000 --- a/k8se2e/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '2.4' -services: - teraslice-master: - image: teraslice-workspace:e2e - ports: - - '45678:45678' - scale: 1 - restart: 'no' - stop_grace_period: 30s - environment: - - TERAFOUNDATION_CONFIG=/app/config/teraslice-master.json - - NODE_OPTIONS=--max-old-space-size=256 - volumes: - - ./logs:/app/logs:delegated - - ./autoload:/app/autoload:ro - - ./.config:/app/config:ro - - ./.assets:/app/assets:delegated - teraslice-worker: - image: teraslice-workspace:e2e - scale: 2 - restart: 'no' - stop_grace_period: 30s - links: - - teraslice-master - environment: - - TERAFOUNDATION_CONFIG=/app/config/teraslice-worker.json - - NODE_OPTIONS=--max-old-space-size=256 - volumes: - - ./logs:/app/logs:delegated - - ./autoload:/app/autoload:ro - - ./.config:/app/config:ro - - ./.assets:/app/assets:delegated diff --git a/k8se2e/elasticsearchDeployment.yaml b/k8se2e/elasticsearchDeployment.yaml deleted file mode 100644 index 5ea8287b1aa..00000000000 --- a/k8se2e/elasticsearchDeployment.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: elasticsearch - labels: - app: elasticsearch - nodeType: master -spec: - replicas: 1 - selector: - matchLabels: - app: elasticsearch - nodeType: master - template: - metadata: - labels: - app: elasticsearch - nodeType: master - spec: - containers: - - name: elasticsearch - image: elasticsearch:7.9.3 - ports: - - containerPort: 9200 - env: - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - - name: discovery.type - value: single-node ---- -kind: Service -apiVersion: v1 -metadata: - name: elasticsearch - labels: - app: elasticsearch -spec: - selector: - app: elasticsearch - nodeType: master - ports: - - port: 9200 - targetPort: 9200 - nodePort: 30200 # the external port teraslice can be accessed on - type: NodePort diff --git a/k8se2e/jest.config.js b/k8se2e/jest.config.js deleted file mode 100644 index 0a4be3a992a..00000000000 --- a/k8se2e/jest.config.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -const config = require('../jest.config.base')(__dirname); - -config.collectCoverage = false; -delete config.transform; -module.exports = config; diff --git a/k8se2e/kindConfig.yaml b/k8se2e/kindConfig.yaml deleted file mode 100644 index e9d87e52e73..00000000000 --- a/k8se2e/kindConfig.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: Cluster -name: k8se2e -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 30200 - hostPort: 9200 - - containerPort: 30678 - hostPort: 5678 diff --git a/k8se2e/masterConfig/teraslice.yaml b/k8se2e/masterConfig/teraslice.yaml deleted file mode 100644 index 8f1aaddea3e..00000000000 --- a/k8se2e/masterConfig/teraslice.yaml +++ /dev/null @@ -1,31 +0,0 @@ -terafoundation: - environment: 'development' - log_level: debug - connectors: - elasticsearch: - default: - apiVersion: "5.6" - host: - - "elasticsearch:9200" - elasticsearch-next: - default: - node: - - "http://elasticsearch:9200" -teraslice: - worker_disconnect_timeout: 60000 - node_disconnect_timeout: 60000 - slicer_timeout: 60000 - shutdown_timeout: 30000 - assets_directory: '/app/assets/' - cluster_manager_type: "kubernetes" - master: true - master_hostname: "127.0.0.1" - kubernetes_image: "teraslice-workspace:k8se2e" - kubernetes_image_pull_secrets: - - "docker-tera1-secret" - kubernetes_namespace: "ts-dev1" - kubernetes_overrides_enabled: true - kubernetes_priority_class_name: 'high-priority' - name: "ts-dev1" - cpu: 1 - memory: 536870912 diff --git a/k8se2e/masterDeployment.yaml b/k8se2e/masterDeployment.yaml deleted file mode 100644 index 14433977e66..00000000000 --- a/k8se2e/masterDeployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: teraslice-master - labels: - app: teraslice - nodeType: master -spec: - replicas: 1 - selector: - matchLabels: - app: teraslice - nodeType: master - template: - metadata: - labels: - app: teraslice - nodeType: master - clusterName: ts-dev1 - spec: - containers: - - name: teraslice-master - image: teraslice-workspace:k8se2e - ports: - - containerPort: 5678 - volumeMounts: - - mountPath: /app/config # defines the directory - name: config - volumes: - - name: config - configMap: - name: teraslice-master - items: - - key: teraslice.yaml - path: teraslice.yaml # the filename that the configMap gets written to, inside the matching mountPath - imagePullSecrets: - - name: docker-tera1-secret ---- -kind: Service -apiVersion: v1 -metadata: - name: teraslice-master - labels: - app: teraslice -spec: - selector: - app: teraslice - nodeType: master - ports: - - port: 5678 - targetPort: 5678 - nodePort: 30678 # the external port teraslice can be accessed on - type: NodePort diff --git a/k8se2e/ns.yaml b/k8se2e/ns.yaml deleted file mode 100644 index 1af803c0c4f..00000000000 --- a/k8se2e/ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: ts-dev1 \ No newline at end of file diff --git a/k8se2e/package.json b/k8se2e/package.json deleted file mode 100644 index 64302c49c46..00000000000 --- a/k8se2e/package.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "k8se2e", - "displayName": "K8S E2E Tests", - "version": "0.0.1", - "private": true, - "description": "Teraslice in Kubernetes integration test suite", - "keywords": [ - "docker-compose", - "elasticsearch", - "teraslice" - ], - "homepage": "https://github.com/terascope/teraslice/tree/master/k8se2e/#readme", - "bugs": { - "url": "https://github.com/terascope/teraslice/issues" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/terascope/teraslice.git" - }, - "license": "MIT", - "author": "Terascope, LLC ", - "scripts": { - "clean": "docker-compose down --volumes --remove-orphans --timeout=5", - "logs": "./scripts/logs.sh", - "logs-follow": "./scripts/logs.sh -f", - "setup": "yarn --silent", - "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --debug --", - "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite k8se2e --", - "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite k8se2e --watch --" - }, - "resolutions": { - "ms": "^2.1.3" - }, - "devDependencies": { - "bunyan": "^1.8.15", - "elasticsearch-store": "^0.72.0", - "fs-extra": "^10.1.0", - "ms": "^2.1.3", - "nanoid": "^3.3.4", - "semver": "^7.3.8", - "signale": "^1.4.0", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14.17.0", - "yarn": ">=1.16.0" - }, - "terascope": { - "testSuite": "k8se2e" - } -} diff --git a/k8se2e/priorityClass.yaml b/k8se2e/priorityClass.yaml deleted file mode 100644 index 0747825c87d..00000000000 --- a/k8se2e/priorityClass.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: high-priority -value: 1000000 -globalDefault: false -description: "This priority class is for Teraslice pods." diff --git a/k8se2e/role.yaml b/k8se2e/role.yaml deleted file mode 100644 index 725cec875a5..00000000000 --- a/k8se2e/role.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: teraslice-all-ts-dev1 - namespace: ts-dev1 -rules: - - apiGroups: ["*"] - resources: ["*"] - verbs: ["*"] \ No newline at end of file diff --git a/k8se2e/roleBinding.yaml b/k8se2e/roleBinding.yaml deleted file mode 100644 index b46683aeb41..00000000000 --- a/k8se2e/roleBinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: teraslice-all-ts-dev1 - namespace: ts-dev1 -subjects: - - kind: ServiceAccount - name: default - namespace: ts-dev1 -roleRef: - kind: Role - name: teraslice-all-ts-dev1 - apiGroup: "rbac.authorization.k8s.io" \ No newline at end of file diff --git a/k8se2e/testJob.json b/k8se2e/testJob.json deleted file mode 100644 index 10baeb47de4..00000000000 --- a/k8se2e/testJob.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "Test Job", - "lifecycle": "once", - "workers": 1, - "assets": [ - "elasticsearch" - ], - "operations": [ - { - "_op": "test-reader", - "fetcher_data_file_path": "./k8se2e/testReaderFile.json" - }, - { - "_op": "elasticsearch_bulk", - "index": "terak8s-example-data", - "type": "events", - "size": 1 - } - ], - "__metadata": { - "cli": { - "cluster": "http://localhost:5678", - "version": "0.55.1", - "job_id": "7c30c1f9-31b1-4ec9-b5c0-da4624fed030", - "updated": "2023-10-13T21:58:14.228Z" - } - } -} diff --git a/k8se2e/testReaderFile.json b/k8se2e/testReaderFile.json deleted file mode 100644 index bada14e3ebf..00000000000 --- a/k8se2e/testReaderFile.json +++ /dev/null @@ -1,68 +0,0 @@ -[ - { "name": "John Doe", "age": 44, "accountId": 43394838 }, - { "name": "Jane Smith", "age": 28, "accountId": 82736482 }, - { "name": "Michael Johnson", "age": 35, "accountId": 12874938 }, - { "name": "Emily Davis", "age": 52, "accountId": 29384729 }, - { "name": "William Brown", "age": 29, "accountId": 74839274 }, - { "name": "Sarah Wilson", "age": 41, "accountId": 49283747 }, - { "name": "David Lee", "age": 37, "accountId": 23847283 }, - { "name": "Jennifer Harris", "age": 32, "accountId": 58374635 }, - { "name": "Joseph Clark", "age": 50, "accountId": 37483748 }, - { "name": "Linda Lewis", "age": 45, "accountId": 93847564 }, - { "name": "Robert Hall", "age": 33, "accountId": 47298374 }, - { "name": "Karen Turner", "age": 49, "accountId": 73847293 }, - { "name": "Richard Garcia", "age": 39, "accountId": 94872347 }, - { "name": "Patricia Rodriguez", "age": 56, "accountId": 64827364 }, - { "name": "Charles Martinez", "age": 43, "accountId": 87348634 }, - { "name": "Nancy Jackson", "age": 30, "accountId": 74623847 }, - { "name": "Thomas Allen", "age": 47, "accountId": 38274692 }, - { "name": "Betty Anderson", "age": 36, "accountId": 92384728 }, - { "name": "Daniel Moore", "age": 53, "accountId": 37483947 }, - { "name": "Deborah White", "age": 40, "accountId": 48374839 }, - { "name": "Paul Martin", "age": 31, "accountId": 73847283 }, - { "name": "Sandra Taylor", "age": 55, "accountId": 23847236 }, - { "name": "Mark Scott", "age": 38, "accountId": 82734628 }, - { "name": "Laura King", "age": 34, "accountId": 74628374 }, - { "name": "Kevin Young", "age": 46, "accountId": 23846283 }, - { "name": "Cynthia Turner", "age": 42, "accountId": 27486374 }, - { "name": "Christopher Thomas", "age": 48, "accountId": 34872634 }, - { "name": "Susan Adams", "age": 29, "accountId": 78374628 }, - { "name": "Kenneth Walker", "age": 57, "accountId": 87263847 }, - { "name": "Ashley Baker", "age": 33, "accountId": 37462836 }, - { "name": "George White", "age": 52, "accountId": 48364782 }, - { "name": "Jessica Robinson", "age": 35, "accountId": 34628347 }, - { "name": "Edward Perez", "age": 41, "accountId": 82736482 }, - { "name": "Amanda Mitchell", "age": 44, "accountId": 73647283 }, - { "name": "Brian Turner", "age": 37, "accountId": 48327462 }, - { "name": "Maria Evans", "age": 48, "accountId": 38274636 }, - { "name": "James Davis", "age": 31, "accountId": 47283947 }, - { "name": "Karen Hall", "age": 56, "accountId": 74638274 }, - { "name": "Jason Martinez", "age": 39, "accountId": 28374683 }, - { "name": "Donna Johnson", "age": 32, "accountId": 74826374 }, - { "name": "Ronald Brown", "age": 50, "accountId": 82734682 }, - { "name": "Carol Lewis", "age": 43, "accountId": 34786374 }, - { "name": "William Taylor", "age": 38, "accountId": 73647283 }, - { "name": "Sharon Robinson", "age": 47, "accountId": 47364826 }, - { "name": "Michael Scott", "age": 30, "accountId": 82374683 }, - { "name": "Ruth Harris", "age": 45, "accountId": 48374628 }, - { "name": "Kevin Anderson", "age": 42, "accountId": 73648236 }, - { "name": "Deborah Perez", "age": 53, "accountId": 23874623 }, - { "name": "Richard Turner", "age": 34, "accountId": 37482364 }, - { "name": "Lisa Baker", "age": 49, "accountId": 48236478 }, - { "name": "David Mitchell", "age": 46, "accountId": 37468236 }, - { "name": "Mary Adams", "age": 52, "accountId": 84623847 }, - { "name": "Joseph King", "age": 41, "accountId": 23874623 }, - { "name": "Susan Harris", "age": 37, "accountId": 47364826 }, - { "name": "Christopher Moore", "age": 35, "accountId": 73648236 }, - { "name": "Patricia Davis", "age": 54, "accountId": 36478236 }, - { "name": "Matthew Garcia", "age": 44, "accountId": 74823647 }, - { "name": "Linda Turner", "age": 33, "accountId": 82364826 }, - { "name": "Charles Scott", "age": 47, "accountId": 36478236 }, - { "name": "Nancy Robinson", "age": 36, "accountId": 74826347 }, - { "name": "Daniel Evans", "age": 51, "accountId": 48374682 }, - { "name": "Betty Perez", "age": 40, "accountId": 73648236 }, - { "name": "Michael Lewis", "age": 38, "accountId": 48364782 }, - { "name": "Karen Hall", "age": 46, "accountId": 36478236 }, - { "name": "William Davis", "age": 49, "accountId": 74823647 } - -] diff --git a/k8se2e/workerConfig/teraslice.yaml b/k8se2e/workerConfig/teraslice.yaml deleted file mode 100644 index 6615cc43757..00000000000 --- a/k8se2e/workerConfig/teraslice.yaml +++ /dev/null @@ -1,29 +0,0 @@ -terafoundation: - environment: 'development' - log_level: debug - connectors: - elasticsearch: - default: - apiVersion: "5.6" - host: - - "elasticsearch:9200" - elasticsearch-next: - default: - node: - - "http://elasticsearch:9200" -teraslice: - worker_disconnect_timeout: 60000 - node_disconnect_timeout: 60000 - slicer_timeout: 60000 - shutdown_timeout: 30000 - assets_directory: '/app/assets/' - cluster_manager_type: "kubernetes" - master: false - master_hostname: "teraslice-master" - kubernetes_image: "teraslice-workspace:k8se2e" - kubernetes_namespace: "ts-dev1" - kubernetes_overrides_enabled: true - kubernetes_priority_class_name: 'high-priority' - name: "ts-dev1" - cpu: 1 - memory: 536870912 From eb92ecc0e7b7d27616352103468a16d31b9c4a3c Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 08:21:39 -0700 Subject: [PATCH 061/142] Add k8se2e to top level package.json's workspaces --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 6fa1e5ba81e..c80facbbf97 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ }, "workspaces": [ "packages/*", - "e2e" + "e2e", + "k8se2e" ], "scripts": { "prebuild": "./packages/xlucene-parser/scripts/generate-engine.js", From c76a1808637a43d835df017f582f5abd3eed9b4c Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 08:23:32 -0700 Subject: [PATCH 062/142] Add k8s directory and kind files to e2e --- e2e/k8s/Dockerfile | 43 ++++++++++++++++++++++ e2e/k8s/elasticsearchDeployment.yaml | 45 +++++++++++++++++++++++ e2e/k8s/kindConfig.yaml | 10 ++++++ e2e/k8s/masterConfig/teraslice.yaml | 31 ++++++++++++++++ e2e/k8s/masterDeployment.yaml | 53 ++++++++++++++++++++++++++++ e2e/k8s/ns.yaml | 4 +++ e2e/k8s/priorityClass.yaml | 7 ++++ e2e/k8s/role.yaml | 9 +++++ e2e/k8s/roleBinding.yaml | 13 +++++++ e2e/k8s/workerConfig/teraslice.yaml | 29 +++++++++++++++ 10 files changed, 244 insertions(+) create mode 100644 e2e/k8s/Dockerfile create mode 100644 e2e/k8s/elasticsearchDeployment.yaml create mode 100644 e2e/k8s/kindConfig.yaml create mode 100644 e2e/k8s/masterConfig/teraslice.yaml create mode 100644 e2e/k8s/masterDeployment.yaml create mode 100644 e2e/k8s/ns.yaml create mode 100644 e2e/k8s/priorityClass.yaml create mode 100644 e2e/k8s/role.yaml create mode 100644 e2e/k8s/roleBinding.yaml create mode 100644 e2e/k8s/workerConfig/teraslice.yaml diff --git a/e2e/k8s/Dockerfile b/e2e/k8s/Dockerfile new file mode 100644 index 00000000000..9b5789fe6cc --- /dev/null +++ b/e2e/k8s/Dockerfile @@ -0,0 +1,43 @@ +# NODE_VERSION is set by default in the config.ts, the following value will only +# be used if you build images by default with docker build +ARG NODE_VERSION=14.21.3 +FROM terascope/node-base:${NODE_VERSION} + +ENV NODE_ENV production + +ENV YARN_SETUP_ARGS "--prod=false --silent --frozen-lockfile" + +COPY package.json yarn.lock tsconfig.json .yarnrc /app/source/ +COPY .yarnclean.ci /app/source/.yarnclean +COPY .yarn /app/source/.yarn +COPY packages /app/source/packages +COPY scripts /app/source/scripts +COPY types /app/source/types +COPY k8se2e /app/source/k8se2e + +RUN yarn --prod=false --frozen-lockfile \ + && yarn build \ + && yarn \ + --prod=true \ + --silent \ + --frozen-lockfile \ + --skip-integrity-check \ + --ignore-scripts \ + && yarn cache clean + + +COPY service.js /app/source/ + +# verify node-rdkafka is installed right +RUN node -e "require('node-rdkafka')" + +# verify teraslice is installed right +RUN node -e "require('teraslice')" + +EXPOSE 5678 + +# set up the volumes +VOLUME /app/config /app/logs /app/assets +ENV TERAFOUNDATION_CONFIG /app/config/teraslice.yaml + +CMD ["node", "service.js"] diff --git a/e2e/k8s/elasticsearchDeployment.yaml b/e2e/k8s/elasticsearchDeployment.yaml new file mode 100644 index 00000000000..5ea8287b1aa --- /dev/null +++ b/e2e/k8s/elasticsearchDeployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + nodeType: master + template: + metadata: + labels: + app: elasticsearch + nodeType: master + spec: + containers: + - name: elasticsearch + image: elasticsearch:7.9.3 + ports: + - containerPort: 9200 + env: + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: single-node +--- +kind: Service +apiVersion: v1 +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + selector: + app: elasticsearch + nodeType: master + ports: + - port: 9200 + targetPort: 9200 + nodePort: 30200 # the external port teraslice can be accessed on + type: NodePort diff --git a/e2e/k8s/kindConfig.yaml b/e2e/k8s/kindConfig.yaml new file mode 100644 index 00000000000..e9d87e52e73 --- /dev/null +++ b/e2e/k8s/kindConfig.yaml @@ -0,0 +1,10 @@ +kind: Cluster +name: k8se2e +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30200 + hostPort: 9200 + - containerPort: 30678 + hostPort: 5678 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml new file mode 100644 index 00000000000..8f1aaddea3e --- /dev/null +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -0,0 +1,31 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: true + master_hostname: "127.0.0.1" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image_pull_secrets: + - "docker-tera1-secret" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml new file mode 100644 index 00000000000..14433977e66 --- /dev/null +++ b/e2e/k8s/masterDeployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: teraslice-master + labels: + app: teraslice + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: teraslice + nodeType: master + template: + metadata: + labels: + app: teraslice + nodeType: master + clusterName: ts-dev1 + spec: + containers: + - name: teraslice-master + image: teraslice-workspace:k8se2e + ports: + - containerPort: 5678 + volumeMounts: + - mountPath: /app/config # defines the directory + name: config + volumes: + - name: config + configMap: + name: teraslice-master + items: + - key: teraslice.yaml + path: teraslice.yaml # the filename that the configMap gets written to, inside the matching mountPath + imagePullSecrets: + - name: docker-tera1-secret +--- +kind: Service +apiVersion: v1 +metadata: + name: teraslice-master + labels: + app: teraslice +spec: + selector: + app: teraslice + nodeType: master + ports: + - port: 5678 + targetPort: 5678 + nodePort: 30678 # the external port teraslice can be accessed on + type: NodePort diff --git a/e2e/k8s/ns.yaml b/e2e/k8s/ns.yaml new file mode 100644 index 00000000000..1af803c0c4f --- /dev/null +++ b/e2e/k8s/ns.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: ts-dev1 \ No newline at end of file diff --git a/e2e/k8s/priorityClass.yaml b/e2e/k8s/priorityClass.yaml new file mode 100644 index 00000000000..0747825c87d --- /dev/null +++ b/e2e/k8s/priorityClass.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000000 +globalDefault: false +description: "This priority class is for Teraslice pods." diff --git a/e2e/k8s/role.yaml b/e2e/k8s/role.yaml new file mode 100644 index 00000000000..725cec875a5 --- /dev/null +++ b/e2e/k8s/role.yaml @@ -0,0 +1,9 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] \ No newline at end of file diff --git a/e2e/k8s/roleBinding.yaml b/e2e/k8s/roleBinding.yaml new file mode 100644 index 00000000000..b46683aeb41 --- /dev/null +++ b/e2e/k8s/roleBinding.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-ts-dev1 + namespace: ts-dev1 +subjects: + - kind: ServiceAccount + name: default + namespace: ts-dev1 +roleRef: + kind: Role + name: teraslice-all-ts-dev1 + apiGroup: "rbac.authorization.k8s.io" \ No newline at end of file diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml new file mode 100644 index 00000000000..6615cc43757 --- /dev/null +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -0,0 +1,29 @@ +terafoundation: + environment: 'development' + log_level: debug + connectors: + elasticsearch: + default: + apiVersion: "5.6" + host: + - "elasticsearch:9200" + elasticsearch-next: + default: + node: + - "http://elasticsearch:9200" +teraslice: + worker_disconnect_timeout: 60000 + node_disconnect_timeout: 60000 + slicer_timeout: 60000 + shutdown_timeout: 30000 + assets_directory: '/app/assets/' + cluster_manager_type: "kubernetes" + master: false + master_hostname: "teraslice-master" + kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_namespace: "ts-dev1" + kubernetes_overrides_enabled: true + kubernetes_priority_class_name: 'high-priority' + name: "ts-dev1" + cpu: 1 + memory: 536870912 From 382bb68ec2b57eee979715b86fa50cbeab4f00a6 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:19:25 -0700 Subject: [PATCH 063/142] revert changes to root dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 62cc30f8c9a..78ead80ec7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY .yarn /app/source/.yarn COPY packages /app/source/packages COPY scripts /app/source/scripts COPY types /app/source/types -COPY k8se2e /app/source/k8se2e RUN yarn --prod=false --frozen-lockfile \ && yarn build \ From db452545d26d4b849fecb817e07209d57e3c2abb Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:20:23 -0700 Subject: [PATCH 064/142] modify k8s yaml file ports --- e2e/k8s/Dockerfile | 2 +- e2e/k8s/kindConfig.yaml | 8 ++++---- e2e/k8s/masterConfig/teraslice.yaml | 4 ++-- e2e/k8s/masterDeployment.yaml | 8 ++++---- e2e/k8s/workerConfig/teraslice.yaml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/e2e/k8s/Dockerfile b/e2e/k8s/Dockerfile index 9b5789fe6cc..d61a1a66e8a 100644 --- a/e2e/k8s/Dockerfile +++ b/e2e/k8s/Dockerfile @@ -13,7 +13,7 @@ COPY .yarn /app/source/.yarn COPY packages /app/source/packages COPY scripts /app/source/scripts COPY types /app/source/types -COPY k8se2e /app/source/k8se2e +COPY e2e/k8s /app/source/e2e/k8s RUN yarn --prod=false --frozen-lockfile \ && yarn build \ diff --git a/e2e/k8s/kindConfig.yaml b/e2e/k8s/kindConfig.yaml index e9d87e52e73..9a344500de0 100644 --- a/e2e/k8s/kindConfig.yaml +++ b/e2e/k8s/kindConfig.yaml @@ -4,7 +4,7 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - - containerPort: 30200 - hostPort: 9200 - - containerPort: 30678 - hostPort: 5678 + - containerPort: 9200 + hostPort: 49200 + - containerPort: 45678 + hostPort: 45678 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 8f1aaddea3e..2a2a972caa4 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -6,11 +6,11 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:9200" + - "elasticsearch:49200" elasticsearch-next: default: node: - - "http://elasticsearch:9200" + - "http://elasticsearch:49200" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 14433977e66..6a3827dfaec 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -22,7 +22,7 @@ spec: - name: teraslice-master image: teraslice-workspace:k8se2e ports: - - containerPort: 5678 + - containerPort: 45678 volumeMounts: - mountPath: /app/config # defines the directory name: config @@ -47,7 +47,7 @@ spec: app: teraslice nodeType: master ports: - - port: 5678 - targetPort: 5678 - nodePort: 30678 # the external port teraslice can be accessed on + - port: 45678 + targetPort: 45678 + nodePort: 45678 # the external port teraslice can be accessed on type: NodePort diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 6615cc43757..4c0a872bc55 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -6,11 +6,11 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:9200" + - "elasticsearch:49200" elasticsearch-next: default: node: - - "http://elasticsearch:9200" + - "http://elasticsearch:49200" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 From dc2f19ccbf7a0e3286c6aed995ef639deb76fc71 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:21:20 -0700 Subject: [PATCH 065/142] update port in masterDeployment yaml file --- e2e/k8s/masterDeployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 6a3827dfaec..4b5eb01c947 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -49,5 +49,5 @@ spec: ports: - port: 45678 targetPort: 45678 - nodePort: 45678 # the external port teraslice can be accessed on + nodePort: 30678 # the external port teraslice can be accessed on type: NodePort From d063cca4f88aac65fd34a2ef5a1205b9cba218cc Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:22:41 -0700 Subject: [PATCH 066/142] Add checks for k8s test environment --- e2e/test/global.setup.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index bcd66773d55..bf9ddf3e237 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -1,6 +1,7 @@ 'use strict'; const { pDelay } = require('@terascope/utils'); +const { getE2eK8sDir } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); const globalTeardown = require('./global.teardown'); @@ -9,13 +10,17 @@ const signale = require('./signale'); const setupTerasliceConfig = require('./setup-config'); const downloadAssets = require('./download-assets'); const { CONFIG_PATH, ASSETS_PATH } = require('./config'); +const { createKindCluster, destroyKindCluster } = require('../../packages/scripts/src/helpers/scripts.ts');// FIXME: is this right? module.exports = async () => { const teraslice = new TerasliceHarness(); + await teraslice.init();// create TS and ES or OS clients - await teraslice.init(); - - await globalTeardown(teraslice.client); + if (process.env.TEST_PLATFORM === 'kubernetes') { + await destroyKindCluster(); + } else { + await globalTeardown(teraslice.client); // docker compose down and ES or OS teardown + } await teraslice.resetLogs(); process.stdout.write('\n'); @@ -35,7 +40,12 @@ module.exports = async () => { await Promise.all([setupTerasliceConfig(), downloadAssets()]); - await dockerUp(); + if (process.env.TEST_PLATFORM === 'kubernetes') { + const e2eK8sDir = getE2eK8sDir(); + await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); + } else { + await dockerUp(); // create TS master and workers from docker-compose file + } await teraslice.waitForTeraslice(); await pDelay(2000); await teraslice.resetState(); From a0a6bd11767f7cdf6ca72f95a94bf2f8fedea341 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:25:36 -0700 Subject: [PATCH 067/142] Add deployKafka, and kindLoadServiceImage functions. Add registerStandardAssets function --- packages/scripts/src/helpers/scripts.ts | 67 ++++++++++++++++++++----- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 45a3b465da3..bee455073ae 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -14,6 +14,7 @@ import { TSCommands, PackageInfo } from './interfaces'; import { getRootDir } from './misc'; import signale from './signale'; import * as config from './config'; +import { getE2eK8sDir } from '../helpers/packages'; const logger = debugLogger('ts-scripts:cmd'); @@ -165,6 +166,12 @@ export async function runJest( signale.debug(`executing: jest ${args.join(' ')}`); } + console.log('######## cwd: ', cwd); + console.log('######## argsMap: ', argsMap); + console.log('######## args: ', args); + console.log('######## env: ', env); + console.log('######## extraArgs: ', extraArgs); + await fork({ cmd: 'jest', cwd, @@ -534,10 +541,10 @@ export async function yarnPublish( } export async function createKindCluster( - k8se2eDir: string, + e2eK8sDir: string, kindConfigFileName: string ): Promise { - const configPath = path.join(k8se2eDir, kindConfigFileName); + const configPath = path.join(e2eK8sDir, kindConfigFileName); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); signale.log(subprocess.stderr); // const test = await execa.command('echo hello'); @@ -576,13 +583,29 @@ export async function loadTerasliceImage(terasliceImage: string): Promise console.log('load teraslice image subprocess: ', subprocess); } +export async function kindLoadServiceImage(serviceName: string): Promise { + console.log('@@@@@ kindLoadServiceImage'); + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + if (serviceName === 'elasticsearch') { + await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); + } else if (serviceName === 'kafka') { + await deployKafka(e2eK8sDir, 'elasticsearchDeployment.yaml'); + } else { + signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); + } +} + export async function createNamespace() { const subprocess = await execa.command('kubectl create namespace ts-dev1'); console.log('namespace subprocess: ', subprocess); } -export async function deployElasticSearch(k8se2eDir: string, elasticsearchYaml: string) { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, elasticsearchYaml)}`); +export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { + console.log('@@@@@ deployElasticsearch'); + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); console.log('esDeploy subprocess: ', subprocess); const elasticsearchReady = await waitForESRunning(240000); @@ -625,32 +648,43 @@ function waitForESRunning(timeoutMs = 120000): Promise { return _waitForESRunning(); } +function deployKafka(e2eK8sDir: string, arg1: string) { + throw new Error('Function not implemented.'); + // const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); + // console.log('esDeploy subprocess: ', subprocess); + + // const elasticsearchReady = await waitForESRunning(240000); + // if (elasticsearchReady) { + // signale.success('Elasticsearch is ready to go'); + // } +} + export async function k8sSetup( - k8se2eDir: string, + e2eK8sDir: string, roleYaml: string, roleBindingYaml: string, priorityClassYaml: string ): Promise { - const subprocess1 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleYaml)}`); - const subprocess2 = await execa.command(`kubectl create -f ${path.join(k8se2eDir, roleBindingYaml)}`); - const subprocess3 = await execa.command(`kubectl apply -f ${path.join(k8se2eDir, priorityClassYaml)}`); + const subprocess1 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleYaml)}`); + const subprocess2 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleBindingYaml)}`); + const subprocess3 = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, priorityClassYaml)}`); console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); } export async function deployk8sTeraslice( - k8se2eDir: string, + e2eK8sDir: string, masterDeploymentYaml: string ) { /// Creates configmap for terasclice-master - let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(k8se2eDir, 'masterConfig', 'teraslice.yaml')}`); + let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); console.log('masterConfig subprocess: ', subprocess); /// Creates configmap for teraslice-worker - subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(k8se2eDir, 'workerConfig', 'teraslice.yaml')}`); + subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); console.log('workerConfig subprocess: ', subprocess); /// Creates deployment for teraslice - subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(k8se2eDir, masterDeploymentYaml)}`); + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); console.log('masterDeploy subprocess: ', subprocess); subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); @@ -714,11 +748,16 @@ export async function registerTestJob() { console.log('registerTestJob subprocess: ', subprocess); } -export async function registerElasticsearch() { +export async function registerElasticsearchAssets() { const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/elasticsearch-assets'); console.log('registerElasticsearch asset subprocess: ', subprocess); } +export async function registerStandardAssets() { + const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/standard-assets'); + console.log('registerStandardAssets subprocess: ', subprocess); +} + export async function startTestJob() { const subprocess = await execa.command('earl tjm start testJob.json'); console.log('Run earl tjm start testJob.json'); @@ -730,8 +769,10 @@ export async function showState() { console.log('showState subprocess: ', subprocess); } +// TODO: utils/src/promises/.ts pDelay already does this function PromiseTimeout(delayms: number) { return new Promise((resolve) => { setTimeout(resolve, delayms); }); } + From 678f1d58799092348f8ae1a9d0682c4b4e938a4f Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:26:41 -0700 Subject: [PATCH 068/142] remove unneeded K8S E2E variable --- packages/scripts/src/helpers/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index e6434a496d1..9d8b755b39c 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -98,8 +98,6 @@ export const DEV_DOCKER_IMAGE = process.env.DEV_DOCKER_IMAGE || undefined; */ export const SKIP_DOCKER_BUILD_IN_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_IN_E2E ?? false); -export const SKIP_DOCKER_BUILD_K8S_E2E = toBoolean(process.env.SKIP_DOCKER_BUILD_K8S_E2E ?? false); - export const SKIP_E2E_OUTPUT_LOGS = toBoolean(process.env.SKIP_E2E_OUTPUT_LOGS ?? !isCI); /** @@ -160,3 +158,5 @@ export const SEARCH_TEST_HOST = testHost; // https://github.com/terascope/base-docker-image // This overrides the value in the Dockerfile export const NODE_VERSION = process.env.NODE_VERSION || '18.16.0'; + +export const { TEST_PLATFORM = 'native' } = process.env; From ce1e719e0d9bcc9cd8f957ff776515b79d9407f2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:27:20 -0700 Subject: [PATCH 069/142] rename function that gets e2e directory --- packages/scripts/src/helpers/packages.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/scripts/src/helpers/packages.ts b/packages/scripts/src/helpers/packages.ts index 75ae9b384de..b1bca1e2621 100644 --- a/packages/scripts/src/helpers/packages.ts +++ b/packages/scripts/src/helpers/packages.ts @@ -15,7 +15,7 @@ import * as i from './interfaces'; let _packages: i.PackageInfo[] = []; let _e2eDir: string|undefined; -let _k8se2eDir: string|undefined; +let _e2e_k8s_dir: string|undefined; export function getE2EDir(): string|undefined { if (_e2eDir) return _e2eDir; @@ -28,12 +28,12 @@ export function getE2EDir(): string|undefined { return undefined; } -export function getK8SE2EDir(): string|undefined { - if (_k8se2eDir) return _k8se2eDir; +export function getE2eK8sDir(): string|undefined { + if (_e2e_k8s_dir) return _e2e_k8s_dir; - if (fs.existsSync(path.join(misc.getRootDir(), 'k8se2e'))) { - _k8se2eDir = path.join(misc.getRootDir(), 'k8se2e'); - return _k8se2eDir; + if (fs.existsSync(path.join(misc.getRootDir(), 'e2e/k8s'))) { + _e2e_k8s_dir = path.join(misc.getRootDir(), 'e2e/k8s'); + return _e2e_k8s_dir; } return undefined; From ca9727e91ef32ac37c73facba2274b92b58873a5 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:51:26 -0700 Subject: [PATCH 070/142] Check test platform to call service in proper env --- .../src/helpers/test-runner/services.ts | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 5e2f67f96f2..77d5980f460 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -10,7 +10,8 @@ import { DockerRunOptions, getContainerInfo, dockerStop, - dockerPull + dockerPull, + kindLoadServiceImage } from '../scripts'; import { TestOptions } from './interfaces'; import { Service } from '../interfaces'; @@ -135,6 +136,8 @@ const services: Readonly>> = { }; export async function pullServices(suite: string, options: TestOptions): Promise { + console.log('@@@ in pull services'); + const launchServices = getServicesForSuite(suite); try { @@ -189,6 +192,7 @@ export async function pullServices(suite: string, options: TestOptions): Promise } export async function ensureServices(suite: string, options: TestOptions): Promise<() => void> { + console.log('@@@ in ensure services'); const launchServices = getServicesForSuite(suite); const promises: Promise<(() => void)>[] = []; @@ -242,7 +246,7 @@ export async function ensureServices(suite: string, options: TestOptions): Promi } export async function ensureKafka(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Kafka); await checkKafka(options, startTime); @@ -250,7 +254,7 @@ export async function ensureKafka(options: TestOptions): Promise<() => void> { } export async function ensureMinio(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Minio); await checkMinio(options, startTime); @@ -258,7 +262,7 @@ export async function ensureMinio(options: TestOptions): Promise<() => void> { } export async function ensureElasticsearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Elasticsearch); await checkElasticsearch(options, startTime); @@ -266,7 +270,7 @@ export async function ensureElasticsearch(options: TestOptions): Promise<() => v } export async function ensureRestrainedElasticsearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.RestrainedElasticsearch); await checkRestrainedElasticsearch(options, startTime); @@ -274,7 +278,7 @@ export async function ensureRestrainedElasticsearch(options: TestOptions): Promi } export async function ensureRestrainedOpensearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.RestrainedOpensearch); await checkRestrainedOpensearch(options, startTime); @@ -282,7 +286,7 @@ export async function ensureRestrainedOpensearch(options: TestOptions): Promise< } export async function ensureOpensearch(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.Opensearch); await checkOpensearch(options, startTime); @@ -290,7 +294,7 @@ export async function ensureOpensearch(options: TestOptions): Promise<() => void } export async function ensureRabbitMQ(options: TestOptions): Promise<() => void> { - let fn = () => {}; + let fn = () => { }; const startTime = Date.now(); fn = await startService(options, Service.RabbitMQ); await checkRabbitMQ(options, startTime); @@ -680,6 +684,8 @@ async function checkKafka(options: TestOptions, startTime: number) { } async function startService(options: TestOptions, service: Service): Promise<() => void> { + console.log('in startService: ', service); + let serviceName = service; if (serviceName === 'restrained_elasticsearch') { @@ -693,13 +699,21 @@ async function startService(options: TestOptions, service: Service): Promise<() const version = options[`${serviceName}Version`] as string; if (options.useExistingServices) { signale.warn(`expecting ${service}@${version} to be running (this can be dangerous)...`); - return () => {}; + return () => { }; } signale.pending(`starting ${service}@${version} service...`); await stopService(service); + if (process.env.TEST_PLATFORM === 'kubernetes') { + await stopService(service); // FIXME: does this use docker + console.log(`@@@@@@@ loading ${service} via kind`); + // load via kind + kindLoadServiceImage(service); + return () => { }; + } + console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], version, @@ -707,6 +721,7 @@ async function startService(options: TestOptions, service: Service): Promise<() options.debug || options.trace ); + console.log('@@@@@ fn(): ', fn.toString()); return () => { try { fn(); @@ -718,4 +733,5 @@ async function startService(options: TestOptions, service: Service): Promise<() ); } }; + } From 5b2cd0f26a7e5559197a285b6abb650c1d0d1034 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:55:09 -0700 Subject: [PATCH 071/142] check test platform before starting docker or kind --- packages/scripts/src/cmds/test.ts | 2 +- .../scripts/src/helpers/test-runner/index.ts | 290 +++++++++--------- .../src/helpers/test-runner/interfaces.ts | 2 +- 3 files changed, 149 insertions(+), 145 deletions(-) diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index 9f3c24c7f75..435932ae56b 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -26,7 +26,7 @@ type Options = { 'node-version': string; 'use-existing-services': boolean; packages?: PackageInfo[]; - 'ignore-mount': boolean + 'ignore-mount': boolean; }; const jestArgs = getExtraArgs(); diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 9eb6aa55255..47c342b3d2d 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -11,21 +11,12 @@ import { PackageInfo } from '../interfaces'; import { TestOptions } from './interfaces'; import { createKindCluster, - destroyKindCluster, - loadTerasliceImage, - createNamespace, - deployElasticSearch, - k8sSetup, - deployk8sTeraslice, runJest, dockerTag, isKindInstalled, isKubectlInstalled, - registerTestJob, - startTestJob, - showState, - registerElasticsearch, - setAlias + createNamespace, + k8sSetup, } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -34,16 +25,12 @@ import { } from './utils'; import signale from '../signale'; import { - getE2EDir, readPackageInfo, listPackages, getK8SE2EDir + getE2EDir, readPackageInfo, listPackages, getE2eK8sDir } from '../packages'; import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; import { TestTracker } from './tracker'; -import { - MAX_PROJECTS_PER_BATCH, - SKIP_DOCKER_BUILD_IN_E2E, - SKIP_DOCKER_BUILD_K8S_E2E -} from '../config'; +import { MAX_PROJECTS_PER_BATCH, SKIP_DOCKER_BUILD_IN_E2E } from '../config'; const logger = debugLogger('ts-scripts:cmd:test'); @@ -85,11 +72,6 @@ async function _runTests( return; } - if (options.suite?.includes('k8se2e')) { - await runk8sE2ETest(options, tracker); - return; - } - const filtered = filterBySuite(pkgInfos, options); if (!filtered.length) { signale.warn('No tests found.'); @@ -216,6 +198,7 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { + console.log('options: ', options); tracker.expected++; const suite = 'e2e'; @@ -226,13 +209,34 @@ async function runE2ETest( throw new Error('Missing e2e test directory'); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const kindInstalled = await isKindInstalled(); + if (!kindInstalled) { + signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); + process.exit(1); + } + + const kubectlInstalled = await isKubectlInstalled(); + if (!kubectlInstalled) { + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + process.exit(1); + } + // TODO: pass kind config file in as a variable + await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); + await createNamespace(); + await k8sSetup(e2eK8sDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); + } + const rootInfo = getRootInfo(); - // const e2eImage = `${rootInfo.name}:e2e-nodev${options.nodeVersion}`; const e2eImage = `${rootInfo.name}:e2e`; if (isCI) { // pull the services first in CI - await pullServices(suite, options); + await pullServices(suite, options); // FIXME: if in k8s run different function } try { @@ -255,7 +259,7 @@ async function runE2ETest( try { tracker.addCleanup( 'e2e:services', - await ensureServices(suite, options) + await ensureServices(suite, options) // FIXME: if in k8s run different function ); } catch (err) { tracker.addError(err); @@ -326,121 +330,121 @@ function printAndGetEnv(suite: string, options: TestOptions) { return env; } -async function runk8sE2ETest( - options: TestOptions, tracker: TestTracker -): Promise { - console.log('options: ', options); - tracker.expected++; - - const k8se2eDir = getK8SE2EDir(); - if (!k8se2eDir) { - throw new Error('Missing k8se2e test directory'); - } - - const kindInstalled = await isKindInstalled(); - if (!kindInstalled) { - signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); - process.exit(1); - } - - const kubectlInstalled = await isKubectlInstalled(); - if (!kubectlInstalled) { - signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); - process.exit(1); - } - // TODO: pass kind config file in as a variable - await createKindCluster(k8se2eDir, 'kindConfig.yaml'); - - const suite = 'k8se2e'; - let startedTest = false; - - const rootInfo = getRootInfo(); - const k8se2eImage = `${rootInfo.name}:k8se2e`; - - // if (isCI) { - // // pull the services first in CI - // await pullServices(suite, options); - // } - - try { - if (SKIP_DOCKER_BUILD_K8S_E2E) { - const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; - await dockerTag(devImage, k8se2eImage); - await loadTerasliceImage(k8se2eImage); - } else { - const publishOptions: PublishOptions = { - dryRun: true, - nodeVersion: options.nodeVersion, - type: PublishType.Dev - }; - const devImage = await buildDevDockerImage(publishOptions); - await dockerTag(devImage, k8se2eImage); - await loadTerasliceImage(k8se2eImage); - } - } catch (err) { - tracker.addError(err); - } - - // TODO: add tracker - await createNamespace(); - await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); - await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); - await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); - await showState(); - - await setAlias(); - await registerElasticsearch(); - await registerTestJob(); - await startTestJob(); - - // if (!tracker.hasErrors()) { - // const timeLabel = `test suite "${suite}"`; - // signale.time(timeLabel); - // startedTest = true; - - // const env = printAndGetEnv(suite, options); - - // tracker.started++; - // try { - // await runJest( - // k8se2eDir, - // getArgs(options), - // env, - // options.jestArgs, - // options.debug - // ); - // tracker.ended++; - // } catch (err) { - // tracker.ended++; - // tracker.addError(err.message); - // } - - // signale.timeEnd(timeLabel); - // } - - // if (!startedTest) return; - - // if (!options.keepOpen) { - // try { - // await logE2E(k8se2eDir, tracker.hasErrors()); - // } catch (err) { - // signale.error( - // new TSError(err, { - // reason: `Writing the "${suite}" logs failed`, - // }) - // ); - // } - // } - - // if (tracker.hasErrors()) { - // tracker.addCleanup('e2e:teardown', async () => { - // options.keepOpen = false; - // await globalTeardown(options, [{ - // name: suite, - // dir: k8se2eDir, - // suite, - // }]); - // }); - // } - // await destroyKindCluster(); -} +// async function runk8sE2ETest( +// options: TestOptions, tracker: TestTracker +// ): Promise { +// console.log('options: ', options); +// tracker.expected++; + +// const k8se2eDir = getK8SE2EDir(); +// if (!k8se2eDir) { +// throw new Error('Missing k8se2e test directory'); +// } + +// const kindInstalled = await isKindInstalled(); +// if (!kindInstalled) { +// signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); +// process.exit(1); +// } + +// const kubectlInstalled = await isKubectlInstalled(); +// if (!kubectlInstalled) { +// signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); +// process.exit(1); +// } +// // TODO: pass kind config file in as a variable +// await createKindCluster(k8se2eDir, 'kindConfig.yaml'); + +// const suite = 'k8se2e'; +// let startedTest = false; + +// const rootInfo = getRootInfo(); +// const k8se2eImage = `${rootInfo.name}:k8se2e`; + +// // if (isCI) { +// // // pull the services first in CI +// // await pullServices(suite, options); +// // } + +// try { +// if (SKIP_DOCKER_BUILD_IN_E2E) { +// const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; +// await dockerTag(devImage, k8se2eImage); +// await loadTerasliceImage(k8se2eImage); +// } else { +// const publishOptions: PublishOptions = { +// dryRun: true, +// nodeVersion: options.nodeVersion, +// type: PublishType.Dev +// }; +// const devImage = await buildDevDockerImage(publishOptions); +// await dockerTag(devImage, k8se2eImage); +// await loadTerasliceImage(k8se2eImage); +// } +// } catch (err) { +// tracker.addError(err); +// } + +// TODO: add tracker +// await createNamespace(); +// await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); +// await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); +// await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); +// await showState(); + +// await setAlias(); +// await registerElasticsearchAssets(); +// await registerStandardAssets(); +// await registerTestJob(); +// await startTestJob(); + +// if (!tracker.hasErrors()) { +// const timeLabel = `test suite "${suite}"`; +// signale.time(timeLabel); +// startedTest = true; + +// const env = printAndGetEnv(suite, options); + +// tracker.started++; +// try { +// await runJest( +// k8se2eDir, +// getArgs(options), +// env, +// options.jestArgs, +// options.debug +// ); +// tracker.ended++; +// } catch (err) { +// tracker.ended++; +// tracker.addError(err.message); +// } + +// signale.timeEnd(timeLabel); +// } + +// if (!startedTest) return; + +// if (!options.keepOpen) { +// try { +// await logE2E(k8se2eDir, tracker.hasErrors()); +// } catch (err) { +// signale.error( +// new TSError(err, { +// reason: `Writing the "${suite}" logs failed`, +// }) +// ); +// } +// } + +// if (tracker.hasErrors()) { +// tracker.addCleanup('e2e:teardown', async () => { +// options.keepOpen = false; +// await globalTeardown(options, [{ +// name: suite, +// dir: k8se2eDir, +// suite, +// }]); +// }); +// } +// } diff --git a/packages/scripts/src/helpers/test-runner/interfaces.ts b/packages/scripts/src/helpers/test-runner/interfaces.ts index 5ce208e7a3e..fdb40b8a7af 100644 --- a/packages/scripts/src/helpers/test-runner/interfaces.ts +++ b/packages/scripts/src/helpers/test-runner/interfaces.ts @@ -19,7 +19,7 @@ export type TestOptions = { opensearchVersion: string; nodeVersion: string; jestArgs?: string[]; - ignoreMount: boolean + ignoreMount: boolean; }; export type GroupedPackages = { From 55f3b1c3f2926db2b78faa0a771c20ba0f660223 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 17 Oct 2023 13:55:33 -0700 Subject: [PATCH 072/142] create script to run k8s e2e tests --- e2e/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/package.json b/e2e/package.json index 2f6277098c9..0b7182e9f99 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,6 +25,7 @@ "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", + "test:k8s": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", From 209c458ba61129adacdc769d6019ba8dce7e6fa7 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:08:09 -0700 Subject: [PATCH 073/142] Add kafka to teraslice.yaml files --- e2e/k8s/masterConfig/teraslice.yaml | 9 +++++++-- e2e/k8s/workerConfig/teraslice.yaml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 2a2a972caa4..22458f4958c 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -6,11 +6,16 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:49200" + - "elasticsearch:9200" elasticsearch-next: default: node: - - "http://elasticsearch:49200" + - "http://elasticsearch:9200" + kafka: + default: + brokers: + - "kafka:9092" + teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 4c0a872bc55..647d1c05870 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -6,11 +6,16 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:49200" + - "elasticsearch:9200" elasticsearch-next: default: node: - - "http://elasticsearch:49200" + - "http://elasticsearch:9200" + kafka: + default: + brokers: + - "kafka:9092" + teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 From 370008ca89816f18d5922adffcdcdaf1d4f484c4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:08:31 -0700 Subject: [PATCH 074/142] Add kafkaDeployment.yaml file --- e2e/k8s/kafkaDeployment.yaml | 45 ++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 e2e/k8s/kafkaDeployment.yaml diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml new file mode 100644 index 00000000000..37197548860 --- /dev/null +++ b/e2e/k8s/kafkaDeployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka + labels: + app: kafka + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: kafka + nodeType: master + template: + metadata: + labels: + app: kafka + nodeType: master + spec: + containers: + - name: kafka + image: kafka:3.3 + ports: + - containerPort: 9092 + env: + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + - name: discovery.type + value: single-node +--- +kind: Service +apiVersion: v1 +metadata: + name: kafka + labels: + app: kafka +spec: + selector: + app: kafka + nodeType: master + ports: + - port: 9092 + targetPort: 9092 + nodePort: 30092 # the external port teraslice can be accessed on + type: NodePort From 68966076d6a99fef6bbffadbd2492d3159f22905 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:09:09 -0700 Subject: [PATCH 075/142] finally get ports forwarded correctly --- e2e/k8s/kindConfig.yaml | 6 ++++-- e2e/k8s/masterDeployment.yaml | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/e2e/k8s/kindConfig.yaml b/e2e/k8s/kindConfig.yaml index 9a344500de0..54fd983a1f4 100644 --- a/e2e/k8s/kindConfig.yaml +++ b/e2e/k8s/kindConfig.yaml @@ -4,7 +4,9 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane extraPortMappings: - - containerPort: 9200 + - containerPort: 30200 hostPort: 49200 - - containerPort: 45678 + - containerPort: 30678 hostPort: 45678 + - containerPort: 30092 + hostPort: 49092 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 4b5eb01c947..14433977e66 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -22,7 +22,7 @@ spec: - name: teraslice-master image: teraslice-workspace:k8se2e ports: - - containerPort: 45678 + - containerPort: 5678 volumeMounts: - mountPath: /app/config # defines the directory name: config @@ -47,7 +47,7 @@ spec: app: teraslice nodeType: master ports: - - port: 45678 - targetPort: 45678 + - port: 5678 + targetPort: 5678 nodePort: 30678 # the external port teraslice can be accessed on type: NodePort From ca7186919cd0b800deed07434316fc16b88b4c61 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:09:30 -0700 Subject: [PATCH 076/142] remove unneeeded dockerfile --- e2e/k8s/Dockerfile | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 e2e/k8s/Dockerfile diff --git a/e2e/k8s/Dockerfile b/e2e/k8s/Dockerfile deleted file mode 100644 index d61a1a66e8a..00000000000 --- a/e2e/k8s/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# NODE_VERSION is set by default in the config.ts, the following value will only -# be used if you build images by default with docker build -ARG NODE_VERSION=14.21.3 -FROM terascope/node-base:${NODE_VERSION} - -ENV NODE_ENV production - -ENV YARN_SETUP_ARGS "--prod=false --silent --frozen-lockfile" - -COPY package.json yarn.lock tsconfig.json .yarnrc /app/source/ -COPY .yarnclean.ci /app/source/.yarnclean -COPY .yarn /app/source/.yarn -COPY packages /app/source/packages -COPY scripts /app/source/scripts -COPY types /app/source/types -COPY e2e/k8s /app/source/e2e/k8s - -RUN yarn --prod=false --frozen-lockfile \ - && yarn build \ - && yarn \ - --prod=true \ - --silent \ - --frozen-lockfile \ - --skip-integrity-check \ - --ignore-scripts \ - && yarn cache clean - - -COPY service.js /app/source/ - -# verify node-rdkafka is installed right -RUN node -e "require('node-rdkafka')" - -# verify teraslice is installed right -RUN node -e "require('teraslice')" - -EXPOSE 5678 - -# set up the volumes -VOLUME /app/config /app/logs /app/assets -ENV TERAFOUNDATION_CONFIG /app/config/teraslice.yaml - -CMD ["node", "service.js"] From 0b5f67d5b02baf1c3d359bc7d094bbe402c1fbe0 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:12:24 -0700 Subject: [PATCH 077/142] wip: update global.setup to be k8s aware --- e2e/test/global.setup.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index bf9ddf3e237..c61c698fb75 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -1,7 +1,7 @@ 'use strict'; const { pDelay } = require('@terascope/utils'); -const { getE2eK8sDir } = require('@terascope/scripts'); +const { getE2eK8sDir, deployK8sTeraslice, destroyKindCluster } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); const globalTeardown = require('./global.teardown'); @@ -10,7 +10,6 @@ const signale = require('./signale'); const setupTerasliceConfig = require('./setup-config'); const downloadAssets = require('./download-assets'); const { CONFIG_PATH, ASSETS_PATH } = require('./config'); -const { createKindCluster, destroyKindCluster } = require('../../packages/scripts/src/helpers/scripts.ts');// FIXME: is this right? module.exports = async () => { const teraslice = new TerasliceHarness(); @@ -42,7 +41,7 @@ module.exports = async () => { if (process.env.TEST_PLATFORM === 'kubernetes') { const e2eK8sDir = getE2eK8sDir(); - await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); + await deployK8sTeraslice(e2eK8sDir, 'kindConfig.yaml'); } else { await dockerUp(); // create TS master and workers from docker-compose file } From 4012f10c27de8fee0b965a28d161923918480afe Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 07:14:18 -0700 Subject: [PATCH 078/142] update ES port, rename variables --- packages/scripts/src/helpers/scripts.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index bee455073ae..bfccade00bd 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -592,7 +592,7 @@ export async function kindLoadServiceImage(serviceName: string): Promise { if (serviceName === 'elasticsearch') { await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); } else if (serviceName === 'kafka') { - await deployKafka(e2eK8sDir, 'elasticsearchDeployment.yaml'); + await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml'); } else { signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); } @@ -606,7 +606,7 @@ export async function createNamespace() { export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { console.log('@@@@@ deployElasticsearch'); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - console.log('esDeploy subprocess: ', subprocess); + console.log('deployElasticserach subprocess: ', subprocess); const elasticsearchReady = await waitForESRunning(240000); if (elasticsearchReady) { @@ -624,7 +624,7 @@ function waitForESRunning(timeoutMs = 120000): Promise { let elasticsearchRunning = false; try { - const ESResponse = await execa.command('curl localhost:9200'); + const ESResponse = await execa.command('curl http://localhost:49200'); if (ESResponse.stdout) { const jsonData = JSON.parse(ESResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); @@ -648,10 +648,9 @@ function waitForESRunning(timeoutMs = 120000): Promise { return _waitForESRunning(); } -function deployKafka(e2eK8sDir: string, arg1: string) { - throw new Error('Function not implemented.'); - // const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - // console.log('esDeploy subprocess: ', subprocess); +export async function deployKafka(e2eK8sDir: string, kafkaYaml: string) { + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaYaml)}`); + console.log('deployKafka subprocess: ', subprocess); // const elasticsearchReady = await waitForESRunning(240000); // if (elasticsearchReady) { @@ -671,7 +670,7 @@ export async function k8sSetup( console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); } -export async function deployk8sTeraslice( +export async function deployK8sTeraslice( e2eK8sDir: string, masterDeploymentYaml: string ) { From 02ff69ea3636f84383ca343c58ac8ff3ac32dd71 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Oct 2023 13:40:19 -0700 Subject: [PATCH 079/142] Update images for kafka and master deployments --- e2e/k8s/kafkaDeployment.yaml | 8 ++------ e2e/k8s/masterDeployment.yaml | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index 37197548860..a1ac36cccf0 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -19,14 +19,10 @@ spec: spec: containers: - name: kafka - image: kafka:3.3 + image: sotojn6/kafka:3.3 ports: - containerPort: 9092 - env: - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - - name: discovery.type - value: single-node + --- kind: Service apiVersion: v1 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 14433977e66..20b31f7c188 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: teraslice-master - image: teraslice-workspace:k8se2e + image: teraslice-workspace:e2e ports: - containerPort: 5678 volumeMounts: From 1d63144b8ebb2941e05d5ed3c0d3369699b5597c Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:45:45 -0700 Subject: [PATCH 080/142] fix kubernetes image tag --- e2e/k8s/masterConfig/teraslice.yaml | 3 +-- e2e/k8s/workerConfig/teraslice.yaml | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 22458f4958c..924d4bc6dae 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -15,7 +15,6 @@ terafoundation: default: brokers: - "kafka:9092" - teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 @@ -25,7 +24,7 @@ teraslice: cluster_manager_type: "kubernetes" master: true master_hostname: "127.0.0.1" - kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image: "teraslice-workspace:e2e" kubernetes_image_pull_secrets: - "docker-tera1-secret" kubernetes_namespace: "ts-dev1" diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 647d1c05870..af04736d632 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -11,11 +11,10 @@ terafoundation: default: node: - "http://elasticsearch:9200" - kafka: + kafka: default: brokers: - "kafka:9092" - teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 @@ -25,7 +24,7 @@ teraslice: cluster_manager_type: "kubernetes" master: false master_hostname: "teraslice-master" - kubernetes_image: "teraslice-workspace:k8se2e" + kubernetes_image: "teraslice-workspace:e2e" kubernetes_namespace: "ts-dev1" kubernetes_overrides_enabled: true kubernetes_priority_class_name: 'high-priority' From 276cec8b28721a295e9c612731d7112a1d2afef5 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:46:21 -0700 Subject: [PATCH 081/142] add env variables and change image used --- e2e/k8s/kafkaDeployment.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index a1ac36cccf0..d2f7626ccb7 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -19,10 +19,14 @@ spec: spec: containers: - name: kafka - image: sotojn6/kafka:3.3 + image: terascope/kafka-zookeeper:v1.1.0 + env: + - name: KAFKA_ADVERTISED_LISTENERS + value: PLAINTEXT://kafka:9092 + - name: host.name + value: kafka ports: - containerPort: 9092 - --- kind: Service apiVersion: v1 From 6acf34b65efc11e424714e2b01dd438c3d5b68f4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:46:37 -0700 Subject: [PATCH 082/142] change cluster name to k8se2e --- e2e/k8s/masterDeployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index 20b31f7c188..b5a521560de 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: teraslice nodeType: master - clusterName: ts-dev1 + clusterName: k8se2e spec: containers: - name: teraslice-master From 48dbf71002753331dbd913b3a5091fb394c1ff0b Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:49:50 -0700 Subject: [PATCH 083/142] changes to test setup just to get things running --- e2e/test/global.setup.js | 24 +++++++++++++++++------- e2e/test/global.teardown.js | 13 +++++++++++-- e2e/test/teraslice-harness.js | 28 +++++++++++++++++++++------- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index c61c698fb75..84fafa6d6b6 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -1,7 +1,9 @@ 'use strict'; const { pDelay } = require('@terascope/utils'); -const { getE2eK8sDir, deployK8sTeraslice, destroyKindCluster } = require('@terascope/scripts'); +const { + getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets +} = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); const globalTeardown = require('./global.teardown'); @@ -15,10 +17,8 @@ module.exports = async () => { const teraslice = new TerasliceHarness(); await teraslice.init();// create TS and ES or OS clients - if (process.env.TEST_PLATFORM === 'kubernetes') { - await destroyKindCluster(); - } else { - await globalTeardown(teraslice.client); // docker compose down and ES or OS teardown + if (process.env.TEST_PLATFORM === 'native') { + await globalTeardown(teraslice.client); // docker compose down and ES teardown FIXME for k8s } await teraslice.resetLogs(); @@ -37,18 +37,28 @@ module.exports = async () => { fse.ensureDir(CONFIG_PATH), ]); + // FIXME: config diff between k8s and native await Promise.all([setupTerasliceConfig(), downloadAssets()]); + // await pDelay(10000); + if (process.env.TEST_PLATFORM === 'kubernetes') { const e2eK8sDir = getE2eK8sDir(); - await deployK8sTeraslice(e2eK8sDir, 'kindConfig.yaml'); + await deployK8sTeraslice(e2eK8sDir, 'masterDeployment.yaml'); } else { - await dockerUp(); // create TS master and workers from docker-compose file + await dockerUp(); } + await teraslice.waitForTeraslice(); await pDelay(2000); await teraslice.resetState(); + if (process.env.TEST_PLATFORM === 'kubernetes') { + await setAlias(); + await deployElasticsearchAssets(); + await deployStandardAssets(); + } + try { await teraslice.generateTestData(); } catch (err) { diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index e59a84a9462..806d293d99c 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -1,6 +1,7 @@ 'use strict'; const { ElasticsearchTestHelpers } = require('elasticsearch-store'); +const { k8sTearDown } = require('@terascope/scripts'); const fse = require('fs-extra'); const { KEEP_OPEN, CONFIG_PATH, ASSETS_PATH } = require('./config'); const { tearDown, TEST_INDEX_PREFIX } = require('./docker-helpers'); @@ -14,16 +15,24 @@ async function getClient(client) { } async function globalTeardown(testClient) { - const client = await getClient(testClient); + console.log('@@@@@@@@ testClient: ', testClient); + console.log('@@@@@@@@ KEEP_OPEN?: ', KEEP_OPEN); if (KEEP_OPEN) { return; } + console.log('@@@@@@@@ past KEEP_OPEN'); + + const client = await getClient(testClient); const errors = []; try { - await tearDown(); + if (process.env.TEST_PATTERN === 'kubernetes') { + await k8sTearDown(); + } else { + await tearDown(); + } } catch (err) { errors.push(err); } diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 36a96061aaf..37610617b0f 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -9,6 +9,7 @@ const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store' const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); const fse = require('fs-extra'); +const { resetTeraslice } = require('@terascope/scripts'); const { TEST_HOST, HOST_IP, SPEC_INDEX_PREFIX, DEFAULT_NODES, newId, DEFAULT_WORKERS, GENERATE_ONLY, @@ -23,7 +24,7 @@ const generateOnly = GENERATE_ONLY ? parseInt(GENERATE_ONLY, 10) : null; module.exports = class TerasliceHarness { async init() { - const { client } = await createClient({ node: TEST_HOST }); + const { client } = await createClient({ node: TEST_HOST });// create ES or OS db this.client = client; this.teraslice = new TerasliceClient({ host: `http://${HOST_IP}:45678`, @@ -113,11 +114,16 @@ module.exports = class TerasliceHarness { ); })(), (async () => { - const count = Object.keys(state).length; - if (count !== DEFAULT_NODES) { - signale.warn(`resetting cluster state of ${count} nodes`); - await scaleWorkers(); - await this.forWorkers(); + if (process.env.TEST_PLATFORM === 'kubernetes') { + // FIXME: scale workers in k8s???????? + } else { + console.log('@@@@@@@ else'); + const count = Object.keys(state).length; + if (count !== DEFAULT_NODES) { + signale.warn(`resetting cluster state of ${count} nodes`); + await scaleWorkers(); + await this.forWorkers(); + } } })() ]); @@ -350,6 +356,9 @@ module.exports = class TerasliceHarness { return _waitForClusterState(); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + if (nodes === 0) return nodes; + } if (nodes >= DEFAULT_NODES) return nodes; return _waitForClusterState(); }; @@ -390,7 +399,12 @@ module.exports = class TerasliceHarness { signale.pending('Waiting for Teraslice...'); const nodes = await this.waitForClusterState(); - signale.success(`Teraslice is ready to go with ${nodes} nodes`, getElapsed(startTime)); + + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.success('Teraslice is ready to go', getElapsed(startTime)); + } else { + signale.success(`Teraslice is ready to go with ${nodes} nodes`, getElapsed(startTime)); + } } async postJob(jobSpec) { From 8193f929288c2f7c370ca3275563bf15a001be28 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:52:27 -0700 Subject: [PATCH 084/142] new functions: kindStopService, takeDownTeraslice --- packages/scripts/src/helpers/scripts.ts | 163 ++++++++++++++++++------ 1 file changed, 122 insertions(+), 41 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index bfccade00bd..2242850157c 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -547,8 +547,6 @@ export async function createKindCluster( const configPath = path.join(e2eK8sDir, kindConfigFileName); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); signale.log(subprocess.stderr); - // const test = await execa.command('echo hello'); - // console.log('test: ', test); } export async function destroyKindCluster(): Promise { @@ -583,6 +581,25 @@ export async function loadTerasliceImage(terasliceImage: string): Promise console.log('load teraslice image subprocess: ', subprocess); } +export async function kindStopService(serviceName: string): Promise { + console.log('@@@@@ kindStopService'); + // const e2eK8sDir = getE2eK8sDir(); + // if (!e2eK8sDir) { + // throw new Error('Missing k8s e2e test directory'); + // } + // if (serviceName === 'elasticsearch') { + // const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'elasticsearchDeployment.yaml')}`); + // console.log('stopElasticsearch subprocess: ', subprocess); + // } else if (serviceName === 'kafka') { + // let subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaDeployment.yaml')}`); + // console.log('stopKafkaDeployment subprocess: ', subprocess); + // // subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaService.yaml')}`); + // // console.log('stopKafkaService subprocess: ', subprocess); + // } else { + // signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); + // } +} + export async function kindLoadServiceImage(serviceName: string): Promise { console.log('@@@@@ kindLoadServiceImage'); const e2eK8sDir = getE2eK8sDir(); @@ -598,15 +615,15 @@ export async function kindLoadServiceImage(serviceName: string): Promise { } } -export async function createNamespace() { - const subprocess = await execa.command('kubectl create namespace ts-dev1'); +export async function createNamespace(e2eK8sDir: string, namespaceYaml: string) { + const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); console.log('namespace subprocess: ', subprocess); } export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { console.log('@@@@@ deployElasticsearch'); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - console.log('deployElasticserach subprocess: ', subprocess); + console.log('deployElasticsearch subprocess: ', subprocess); const elasticsearchReady = await waitForESRunning(240000); if (elasticsearchReady) { @@ -648,16 +665,56 @@ function waitForESRunning(timeoutMs = 120000): Promise { return _waitForESRunning(); } -export async function deployKafka(e2eK8sDir: string, kafkaYaml: string) { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaYaml)}`); +export async function deployKafka( + e2eK8sDir: string, + kafkaDeploymentYaml: string, +) { + let subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); console.log('deployKafka subprocess: ', subprocess); - // const elasticsearchReady = await waitForESRunning(240000); - // if (elasticsearchReady) { - // signale.success('Elasticsearch is ready to go'); + // subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaServiceYaml)}`); + // console.log('deployKafkaService subprocess: ', subprocess); + + // const kafkaReady = await waitForKafkaRunning(240000); + // if (kafkaReady) { + signale.success('Kafka *might* be ready to go'); // } } +function waitForKafkaRunning(timeoutMs = 120000): Promise { + const endAt = Date.now() + timeoutMs; + + const _waitForKafkaRunning = async (): Promise => { + if (Date.now() > endAt) { + throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); + } + + let elasticsearchRunning = false; + try { + const kafkaResponse = await execa.command('kubectl -n ts-dev1 get po '); + if (kafkaResponse.stdout) { + const jsonData = JSON.parse(kafkaResponse.stdout); + console.log(`response: ${JSON.stringify(jsonData)}`); + if (jsonData.tagline === 'You Know, for Search') { + console.log('jsonData.tagline === You Know, for Search'); + elasticsearchRunning = true; + } + } + } catch (err) { + await PromiseTimeout(3000); + return _waitForKafkaRunning(); + } + + if (elasticsearchRunning) { + return true; + } + await PromiseTimeout(3000); + return _waitForKafkaRunning(); + }; + + return _waitForKafkaRunning(); +} + export async function k8sSetup( e2eK8sDir: string, roleYaml: string, @@ -674,25 +731,29 @@ export async function deployK8sTeraslice( e2eK8sDir: string, masterDeploymentYaml: string ) { - /// Creates configmap for terasclice-master - let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); - console.log('masterConfig subprocess: ', subprocess); + try { + /// Creates configmap for terasclice-master + let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); + console.log('masterConfig subprocess: ', subprocess); - /// Creates configmap for teraslice-worker - subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); - console.log('workerConfig subprocess: ', subprocess); + /// Creates configmap for teraslice-worker + subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); + console.log('workerConfig subprocess: ', subprocess); - /// Creates deployment for teraslice - subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); - console.log('masterDeploy subprocess: ', subprocess); + /// Creates deployment for teraslice + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); + console.log('masterDeploy subprocess: ', subprocess); - subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); - const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); - console.log('podName: ', podName); + subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); + const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); + console.log('podName: ', podName); - const terasliceReady = await waitForTerasliceRunning(240000); - if (terasliceReady) { - signale.success('Teraslice is ready to go'); + const terasliceReady = await waitForTerasliceRunning(240000); + if (terasliceReady) { + signale.success('Teraslice is ready to go'); + } + } catch (error) { + console.log('@@@@@@@@ deployK8sTeraslice error: ', error); } } @@ -706,7 +767,7 @@ function waitForTerasliceRunning(timeoutMs = 120000): Promise { let terasliceRunning = false; try { - const TSMasterResponse = await execa.command('curl localhost:5678'); + const TSMasterResponse = await execa.command('curl localhost:45678'); if (TSMasterResponse.stdout) { const jsonData = JSON.parse(TSMasterResponse.stdout); console.log(`response: ${JSON.stringify(jsonData)}`); @@ -732,29 +793,29 @@ function waitForTerasliceRunning(timeoutMs = 120000): Promise { } export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove ts-k8s-e2e 2> /dev/null || true', { shell: true }); - const subprocess2 = await execa.command('earl aliases add ts-k8s-e2e http://localhost:5678'); + const subprocess1 = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); + const subprocess2 = await execa.command('earl aliases add k8se2e http://localhost:45678'); console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function deployESAsset() { - const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --blocking --bundle terascope/elasticsearch-assets'); - console.log('deployESAsset subprocess: ', subprocess); -} - export async function registerTestJob() { - const subprocess = await execa.command('earl tjm register ts-k8s-e2e testJob.json'); + const subprocess = await execa.command('earl tjm register k8se2e testJob.json'); console.log('registerTestJob subprocess: ', subprocess); } -export async function registerElasticsearchAssets() { - const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/elasticsearch-assets'); - console.log('registerElasticsearch asset subprocess: ', subprocess); +export async function deployElasticsearchAssets() { + const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/elasticsearch-assets'); // FIXME: specify version + console.log('deployElasticsearchAssets subprocess: ', subprocess); +} + +export async function deployStandardAssets() { + const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/standard-assets'); // FIXME: specify version + console.log('deployStandardAssets subprocess: ', subprocess); } -export async function registerStandardAssets() { - const subprocess = await execa.command('earl assets deploy ts-k8s-e2e --bundle terascope/standard-assets'); - console.log('registerStandardAssets subprocess: ', subprocess); +export async function deployKafkaAssets() { + const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/kafka-assets'); // FIXME: specify version + console.log('deployKafkaAssets subprocess: ', subprocess); } export async function startTestJob() { @@ -768,10 +829,30 @@ export async function showState() { console.log('showState subprocess: ', subprocess); } -// TODO: utils/src/promises/.ts pDelay already does this +// FIXME: utils/src/promises/.ts pDelay already does this function PromiseTimeout(delayms: number) { return new Promise((resolve) => { setTimeout(resolve, delayms); }); } +export async function takeDownTeraslice() { + const subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); + console.log('resetTeraslice subprocess: ', subprocess); +} + +export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { + await takeDownTeraslice(); + await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); +} + +export async function k8sTearDown() { + await takeDownTeraslice(); // FIXME: or reset + /// Creates configmap for teraslice-worker + let subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); + console.log('workerConfig subprocess: ', subprocess); + + /// Creates deployment for teraslice + subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-worker || echo "* it is okay..."'); + console.log('masterDeploy subprocess: ', subprocess); +} From d94b60cbc1e52215a6cd6ae398872d80e20ac845 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:53:23 -0700 Subject: [PATCH 085/142] temp change to get k8s tests running --- .../scripts/src/helpers/test-runner/index.ts | 21 ++++++++++++++----- .../src/helpers/test-runner/services.ts | 12 ++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 47c342b3d2d..1e24901a8df 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -17,6 +17,8 @@ import { isKubectlInstalled, createNamespace, k8sSetup, + loadTerasliceImage, + destroyKindCluster, } from '../scripts'; import { getArgs, filterBySuite, globalTeardown, @@ -225,18 +227,19 @@ async function runE2ETest( signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); process.exit(1); } - // TODO: pass kind config file in as a variable + // TODO: pass kind config files in as variables + // FIXME: error handling await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); - await createNamespace(); + await createNamespace(e2eK8sDir, 'ns.yaml'); await k8sSetup(e2eK8sDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); } const rootInfo = getRootInfo(); const e2eImage = `${rootInfo.name}:e2e`; - if (isCI) { + if (isCI && process.env.TEST_PLATFORM === 'native') { // pull the services first in CI - await pullServices(suite, options); // FIXME: if in k8s run different function + await pullServices(suite, options); } try { @@ -256,10 +259,14 @@ async function runE2ETest( tracker.addError(err); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + await loadTerasliceImage(e2eImage); // FIXME: move to global.setup? + } + try { tracker.addCleanup( 'e2e:services', - await ensureServices(suite, options) // FIXME: if in k8s run different function + await ensureServices(suite, options) ); } catch (err) { tracker.addError(err); @@ -314,6 +321,10 @@ async function runE2ETest( }]); }); } + + // if (process.env.TEST_PLATFORM === 'kubernetes') { + // await destroyKindCluster(); + // } } function printAndGetEnv(suite: string, options: TestOptions) { diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 77d5980f460..80dacf3cda9 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -11,7 +11,8 @@ import { getContainerInfo, dockerStop, dockerPull, - kindLoadServiceImage + kindLoadServiceImage, + kindStopService } from '../scripts'; import { TestOptions } from './interfaces'; import { Service } from '../interfaces'; @@ -704,15 +705,16 @@ async function startService(options: TestOptions, service: Service): Promise<() signale.pending(`starting ${service}@${version} service...`); - await stopService(service); if (process.env.TEST_PLATFORM === 'kubernetes') { - await stopService(service); // FIXME: does this use docker - console.log(`@@@@@@@ loading ${service} via kind`); + await kindStopService(service); // load via kind - kindLoadServiceImage(service); + console.log(`@@@@@@@ loading ${service} via kind`); + await kindLoadServiceImage(service); return () => { }; } + await stopService(service); + console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], From 60fcc3df8bc769d772d803034a4e523580a992a9 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Oct 2023 15:59:27 -0700 Subject: [PATCH 086/142] deploy kafka assets --- e2e/test/global.setup.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 84fafa6d6b6..110463c3eb3 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -2,7 +2,7 @@ const { pDelay } = require('@terascope/utils'); const { - getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets + getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets, deployKafkaAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -57,6 +57,7 @@ module.exports = async () => { await setAlias(); await deployElasticsearchAssets(); await deployStandardAssets(); + await deployKafkaAssets(); } try { From c632b3dadcea9d1264bdee9f67a2435e34dde452 Mon Sep 17 00:00:00 2001 From: sotojn Date: Fri, 20 Oct 2023 10:20:46 -0700 Subject: [PATCH 087/142] update kafka connections for k8s --- e2e/k8s/kafkaDeployment.yaml | 17 +++++++++-------- e2e/k8s/masterConfig/teraslice.yaml | 2 +- e2e/k8s/workerConfig/teraslice.yaml | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index d2f7626ccb7..1aaf60374e1 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -21,12 +21,12 @@ spec: - name: kafka image: terascope/kafka-zookeeper:v1.1.0 env: - - name: KAFKA_ADVERTISED_LISTENERS - value: PLAINTEXT://kafka:9092 - - name: host.name + - name: ADVERTISED_HOST value: kafka + - name: ADVERTISED_PORT + value: "9093" ports: - - containerPort: 9092 + - containerPort: 9093 --- kind: Service apiVersion: v1 @@ -35,11 +35,12 @@ metadata: labels: app: kafka spec: + type: NodePort selector: app: kafka nodeType: master ports: - - port: 9092 - targetPort: 9092 - nodePort: 30092 # the external port teraslice can be accessed on - type: NodePort + - port: 9093 + name: kafka + targetPort: 9093 + nodePort: 30092 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 924d4bc6dae..5868d2c46a6 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9092" + - "kafka:9093" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index af04736d632..1e887b17795 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9092" + - "kafka:9093" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 From 0c72f9d3f38d27be5d491e49f109a9dee28dc735 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Oct 2023 16:58:30 -0700 Subject: [PATCH 088/142] Add ignore pattern for testing in k8s --- e2e/jest.config.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/jest.config.js b/e2e/jest.config.js index 0a4be3a992a..f211efc9d58 100644 --- a/e2e/jest.config.js +++ b/e2e/jest.config.js @@ -2,6 +2,8 @@ const config = require('../jest.config.base')(__dirname); +// FIXME update arrays to run tests specific to platform +config.testPathIgnorePatterns = process.env.TEST_PLATFORM === 'kubernetes' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : []; config.collectCoverage = false; delete config.transform; module.exports = config; From af23eead6c5805f6e59cc86da39a59d2cd324c62 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Oct 2023 16:59:41 -0700 Subject: [PATCH 089/142] Add version variables for kafka and elasticsearch --- e2e/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/package.json b/e2e/package.json index 0b7182e9f99..b5291bb2862 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,7 +25,7 @@ "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", From 5a3b680a35ea9bffc125f7b22e9150cee8198564 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Oct 2023 17:00:18 -0700 Subject: [PATCH 090/142] Refactoring and cleanup. --- e2e/test/global.setup.js | 15 ++++++++------- e2e/test/global.teardown.js | 16 +++++++--------- e2e/test/teraslice-harness.js | 11 ++++++----- packages/scripts/src/helpers/scripts.ts | 21 ++++++--------------- 4 files changed, 27 insertions(+), 36 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 110463c3eb3..b5549f72443 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -2,7 +2,12 @@ const { pDelay } = require('@terascope/utils'); const { - getE2eK8sDir, deployK8sTeraslice, setAlias, deployElasticsearchAssets, deployStandardAssets, deployKafkaAssets + getE2eK8sDir, + deployK8sTeraslice, + setAlias, + deployElasticsearchAssets, + deployStandardAssets, + deployKafkaAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -15,7 +20,7 @@ const { CONFIG_PATH, ASSETS_PATH } = require('./config'); module.exports = async () => { const teraslice = new TerasliceHarness(); - await teraslice.init();// create TS and ES or OS clients + await teraslice.init(); if (process.env.TEST_PLATFORM === 'native') { await globalTeardown(teraslice.client); // docker compose down and ES teardown FIXME for k8s @@ -37,15 +42,11 @@ module.exports = async () => { fse.ensureDir(CONFIG_PATH), ]); - // FIXME: config diff between k8s and native - await Promise.all([setupTerasliceConfig(), downloadAssets()]); - - // await pDelay(10000); - if (process.env.TEST_PLATFORM === 'kubernetes') { const e2eK8sDir = getE2eK8sDir(); await deployK8sTeraslice(e2eK8sDir, 'masterDeployment.yaml'); } else { + await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); } diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index 806d293d99c..17b2663a41e 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -1,10 +1,12 @@ 'use strict'; const { ElasticsearchTestHelpers } = require('elasticsearch-store'); -const { k8sTearDown } = require('@terascope/scripts'); +const { tearDownTerasliceK8s } = require('@terascope/scripts'); const fse = require('fs-extra'); -const { KEEP_OPEN, CONFIG_PATH, ASSETS_PATH } = require('./config'); -const { tearDown, TEST_INDEX_PREFIX } = require('./docker-helpers'); +const { + KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, TEST_INDEX_PREFIX +} = require('./config'); +const { tearDown } = require('./docker-helpers'); const signale = require('./signale'); const { cleanupIndex, makeClient } = ElasticsearchTestHelpers; @@ -15,13 +17,9 @@ async function getClient(client) { } async function globalTeardown(testClient) { - console.log('@@@@@@@@ testClient: ', testClient); - console.log('@@@@@@@@ KEEP_OPEN?: ', KEEP_OPEN); - if (KEEP_OPEN) { return; } - console.log('@@@@@@@@ past KEEP_OPEN'); const client = await getClient(testClient); @@ -29,14 +27,14 @@ async function globalTeardown(testClient) { try { if (process.env.TEST_PATTERN === 'kubernetes') { - await k8sTearDown(); + await tearDownTerasliceK8s(); } else { await tearDown(); } } catch (err) { errors.push(err); } - + console.log('@@@@@ global.teardown, TEST_INDEX_PREFIX: ', TEST_INDEX_PREFIX); await cleanupIndex(client, `${TEST_INDEX_PREFIX}*`); if (fse.existsSync(CONFIG_PATH)) { diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 37610617b0f..87eae115c17 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -9,7 +9,6 @@ const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store' const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); const fse = require('fs-extra'); -const { resetTeraslice } = require('@terascope/scripts'); const { TEST_HOST, HOST_IP, SPEC_INDEX_PREFIX, DEFAULT_NODES, newId, DEFAULT_WORKERS, GENERATE_ONLY, @@ -114,16 +113,18 @@ module.exports = class TerasliceHarness { ); })(), (async () => { - if (process.env.TEST_PLATFORM === 'kubernetes') { - // FIXME: scale workers in k8s???????? - } else { - console.log('@@@@@@@ else'); + if (process.env.TEST_PLATFORM === 'native') { const count = Object.keys(state).length; if (count !== DEFAULT_NODES) { signale.warn(`resetting cluster state of ${count} nodes`); await scaleWorkers(); await this.forWorkers(); } + } else { + // Do nothing + // TODO: If tests are ever implemented to scale nodes in Kind, + // a scaleWorkers implementation will need to be created that works with Kind. + // As of 10/2023 Kind doesn't let you scale nodes w/o restarting the cluster. } })() ]); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 2242850157c..030ff876018 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -166,12 +166,6 @@ export async function runJest( signale.debug(`executing: jest ${args.join(' ')}`); } - console.log('######## cwd: ', cwd); - console.log('######## argsMap: ', argsMap); - console.log('######## args: ', args); - console.log('######## env: ', env); - console.log('######## extraArgs: ', extraArgs); - await fork({ cmd: 'jest', cwd, @@ -836,20 +830,17 @@ function PromiseTimeout(delayms: number) { }); } -export async function takeDownTeraslice() { - const subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); - console.log('resetTeraslice subprocess: ', subprocess); -} - export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { - await takeDownTeraslice(); + await tearDownTerasliceK8s(); await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); } -export async function k8sTearDown() { - await takeDownTeraslice(); // FIXME: or reset +export async function tearDownTerasliceK8s() { + let subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); + console.log('resetTeraslice subprocess: ', subprocess); + /// Creates configmap for teraslice-worker - let subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); + subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); console.log('workerConfig subprocess: ', subprocess); /// Creates deployment for teraslice From e9ca68dbfad5d687fe82a6d01906f4d76871ea24 Mon Sep 17 00:00:00 2001 From: sotojn Date: Tue, 24 Oct 2023 07:26:38 -0700 Subject: [PATCH 091/142] update docker compose yaml to use cp-kafka and cp-zookeeper images --- docker-compose.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fcbcecdd76b..5c57163ffd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,16 +73,32 @@ services: cap_add: - IPC_LOCK kafka: - image: terascope/kafka-zookeeper:v1.1.0 - platform: linux/amd64 # platform specified to force amd64 on arm MacOS docker + image: confluentinc/cp-kafka:7.2.0 # Has kafka 3.2.0 + ports: + - "9094:9094" + restart: unless-stopped + depends_on: + - zookeeper + networks: + - cluster + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://0.0.0.0:9094 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://localhost:9094 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + zookeeper: + image: confluentinc/cp-zookeeper:7.2.0 ports: - "2181:2181" - - "9092:9092" restart: unless-stopped networks: - cluster - volumes: - - kafka-data:/kafka + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 minio: image: minio/minio:RELEASE.2023-09-30T07-02-29Z ports: From b18d8d4c83a3a74fa721c1e5cf7d298994b68fa4 Mon Sep 17 00:00:00 2001 From: sotojn Date: Tue, 24 Oct 2023 09:05:29 -0700 Subject: [PATCH 092/142] update k8s tests to use cp-kafka and cp-zkeeper --- e2e/k8s/kafkaDeployment.yaml | 42 +++++++++++++--------- e2e/k8s/masterConfig/teraslice.yaml | 2 +- e2e/k8s/workerConfig/teraslice.yaml | 2 +- e2e/k8s/zookeeperDeployment.yaml | 46 +++++++++++++++++++++++++ packages/scripts/src/helpers/scripts.ts | 4 ++- 5 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 e2e/k8s/zookeeperDeployment.yaml diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index 1aaf60374e1..90220bf0d6d 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -1,46 +1,54 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: kafka + name: cpkafka labels: - app: kafka + app: cpkafka nodeType: master spec: replicas: 1 selector: matchLabels: - app: kafka + app: cpkafka nodeType: master template: metadata: labels: - app: kafka + app: cpkafka nodeType: master spec: containers: - - name: kafka - image: terascope/kafka-zookeeper:v1.1.0 + - name: cpkafka + image: confluentinc/cp-kafka:7.2.0 env: - - name: ADVERTISED_HOST - value: kafka - - name: ADVERTISED_PORT - value: "9093" + - name: KAFKA_BROKER_ID + value: "1" + - name: KAFKA_ZOOKEEPER_CONNECT + value: zookeeper:2181 + - name: KAFKA_ADVERTISED_LISTENERS + value: INTERNAL://cpkafka:9092 + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: INTERNAL:PLAINTEXT + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: INTERNAL + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" ports: - - containerPort: 9093 + - containerPort: 9092 --- kind: Service apiVersion: v1 metadata: - name: kafka + name: cpkafka labels: - app: kafka + app: cpkafka spec: type: NodePort selector: - app: kafka + app: cpkafka nodeType: master ports: - - port: 9093 - name: kafka - targetPort: 9093 + - port: 9092 + name: cpkafka + targetPort: 9092 nodePort: 30092 diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 5868d2c46a6..457d537d514 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9093" + - "cpkafka:9092" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 1e887b17795..3b1870c3d27 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -14,7 +14,7 @@ terafoundation: kafka: default: brokers: - - "kafka:9093" + - "cpkafka:9092" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/zookeeperDeployment.yaml b/e2e/k8s/zookeeperDeployment.yaml new file mode 100644 index 00000000000..d8069babb38 --- /dev/null +++ b/e2e/k8s/zookeeperDeployment.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper + labels: + app: zookeeper + nodeType: master +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + nodeType: master + template: + metadata: + labels: + app: zookeeper + nodeType: master + spec: + containers: + - name: zookeeper + image: confluentinc/cp-zookeeper:7.2.0 + env: + - name: ZOOKEEPER_CLIENT_PORT + value: "2181" + - name: ZOOKEEPER_TICK_TIME + value: "2000" + ports: + - containerPort: 2181 +--- +kind: Service +apiVersion: v1 +metadata: + name: zookeeper + labels: + app: zookeeper +spec: + type: NodePort + selector: + app: zookeeper + nodeType: master + ports: + - port: 2181 + name: zookeeper + targetPort: 2181 + nodePort: 32181 \ No newline at end of file diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 030ff876018..d5011277f36 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -603,7 +603,7 @@ export async function kindLoadServiceImage(serviceName: string): Promise { if (serviceName === 'elasticsearch') { await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); } else if (serviceName === 'kafka') { - await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml'); + await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml', 'zookeeperDeployment.yaml'); } else { signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); } @@ -662,7 +662,9 @@ function waitForESRunning(timeoutMs = 120000): Promise { export async function deployKafka( e2eK8sDir: string, kafkaDeploymentYaml: string, + zookeeperDeploymentYaml: string ) { + await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, zookeeperDeploymentYaml)}`); let subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); console.log('deployKafka subprocess: ', subprocess); From cd96a200b534cecbbac08a682942c74d73a92eff Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 23 Oct 2023 16:25:51 -0700 Subject: [PATCH 093/142] Refactoring, add error handling, add comments. --- e2e/test/global.setup.js | 14 +- e2e/test/global.teardown.js | 1 - e2e/test/teraslice-harness.js | 2 + packages/scripts/src/helpers/scripts.ts | 315 +++++++----------- .../scripts/src/helpers/test-runner/index.ts | 167 ++-------- .../src/helpers/test-runner/services.ts | 13 +- 6 files changed, 161 insertions(+), 351 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index b5549f72443..0f3f315264f 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -5,9 +5,7 @@ const { getE2eK8sDir, deployK8sTeraslice, setAlias, - deployElasticsearchAssets, - deployStandardAssets, - deployKafkaAssets + deployAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -22,9 +20,7 @@ module.exports = async () => { const teraslice = new TerasliceHarness(); await teraslice.init(); - if (process.env.TEST_PLATFORM === 'native') { - await globalTeardown(teraslice.client); // docker compose down and ES teardown FIXME for k8s - } + await globalTeardown(teraslice.client); await teraslice.resetLogs(); process.stdout.write('\n'); @@ -56,9 +52,9 @@ module.exports = async () => { if (process.env.TEST_PLATFORM === 'kubernetes') { await setAlias(); - await deployElasticsearchAssets(); - await deployStandardAssets(); - await deployKafkaAssets(); + await deployAssets('elasticsearch'); + await deployAssets('standard'); + await deployAssets('kafka'); } try { diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index 17b2663a41e..e0605afb025 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -34,7 +34,6 @@ async function globalTeardown(testClient) { } catch (err) { errors.push(err); } - console.log('@@@@@ global.teardown, TEST_INDEX_PREFIX: ', TEST_INDEX_PREFIX); await cleanupIndex(client, `${TEST_INDEX_PREFIX}*`); if (fse.existsSync(CONFIG_PATH)) { diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 87eae115c17..bde5676b4f5 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -358,6 +358,8 @@ module.exports = class TerasliceHarness { } if (process.env.TEST_PLATFORM === 'kubernetes') { + // A get request to 'cluster/state' will return an empty object in kubernetes. + // Therefore nodes will be 0. if (nodes === 0) return nodes; } if (nodes >= DEFAULT_NODES) return nodes; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index d5011277f36..0ccf6f1ea92 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -534,19 +534,20 @@ export async function yarnPublish( }); } -export async function createKindCluster( - e2eK8sDir: string, - kindConfigFileName: string -): Promise { - const configPath = path.join(e2eK8sDir, kindConfigFileName); +export async function createKindCluster(): Promise { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const configPath = path.join(e2eK8sDir, 'kindConfig.yaml'); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); - signale.log(subprocess.stderr); + signale.info(subprocess.stderr); } export async function destroyKindCluster(): Promise { // TODO: pass cluster name in as variable const subprocess = await execa.command('kind delete cluster --name k8se2e'); - signale.log(subprocess.stderr); + signale.info(subprocess.stderr); } export async function isKindInstalled(): Promise { @@ -572,30 +573,68 @@ export async function isKubectlInstalled(): Promise { export async function loadTerasliceImage(terasliceImage: string): Promise { const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); - console.log('load teraslice image subprocess: ', subprocess); + // console.log('load teraslice image subprocess: ', subprocess); + signale.info(subprocess.stderr); + + // FIXME: is it necessary to wait? + // const imageLoaded = await waitForTerasliceImageLoaded(); + // if (imageLoaded) { + // signale.success('Teraslice image successfully loaded.'); + // } } +// function waitForTerasliceImageLoaded(timeoutMs = 120000): Promise { +// const endAt = Date.now() + timeoutMs; + +// const _waitForTerasliceImageLoaded = async (): Promise => { +// if (Date.now() > endAt) { +// throw new Error(`Failure to load teraslice image after ${timeoutMs}ms`); +// } + +// let terasliceLoaded = false; +// try { +// // docker exec into kind-control-plane +// // crictl images -o json +// // find repo tag "docker.io/library/teraslice-workspace:e2e" +// const kubectlResponse = await execa.command(''); +// const kindFinished = kubectlResponse.stdout; +// console.log('kubectl response: ', kubectlResponse); +// if (kindFinished === '"true"') { +// terasliceLoaded = true; +// } +// } catch (err) { +// await pDelay(3000); +// return _waitForTerasliceImageLoaded(); +// } + +// if (terasliceLoaded) { +// return true; +// } +// await pDelay(3000); +// return _waitForTerasliceImageLoaded(); +// }; + +// return _waitForTerasliceImageLoaded(); +// } + export async function kindStopService(serviceName: string): Promise { - console.log('@@@@@ kindStopService'); - // const e2eK8sDir = getE2eK8sDir(); - // if (!e2eK8sDir) { - // throw new Error('Missing k8s e2e test directory'); - // } - // if (serviceName === 'elasticsearch') { - // const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'elasticsearchDeployment.yaml')}`); - // console.log('stopElasticsearch subprocess: ', subprocess); - // } else if (serviceName === 'kafka') { - // let subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaDeployment.yaml')}`); - // console.log('stopKafkaDeployment subprocess: ', subprocess); - // // subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, 'kafkaService.yaml')}`); - // // console.log('stopKafkaService subprocess: ', subprocess); - // } else { - // signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); - // } + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + try { + // Any new service's yaml file must be named 'Deployment.yaml' + const yamlFile = `${serviceName}Deployment.yaml`; + const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + signale.info(subprocess.stdout); + // console.log('stopElasticsearch subprocess: ', subprocess); + } catch (err) { + signale.warn(`The service ${serviceName} could not be deleted because it does not exist.`); + } } +// FIXME: refactor export async function kindLoadServiceImage(serviceName: string): Promise { - console.log('@@@@@ kindLoadServiceImage'); const e2eK8sDir = getE2eK8sDir(); if (!e2eK8sDir) { throw new Error('Missing k8s e2e test directory'); @@ -609,54 +648,20 @@ export async function kindLoadServiceImage(serviceName: string): Promise { } } -export async function createNamespace(e2eK8sDir: string, namespaceYaml: string) { - const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); - console.log('namespace subprocess: ', subprocess); +export async function createNamespace() { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'ns.yaml')}`); + signale.info(subprocess.stdout); + // console.log('namespace subprocess: ', subprocess); } export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { - console.log('@@@@@ deployElasticsearch'); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - console.log('deployElasticsearch subprocess: ', subprocess); - - const elasticsearchReady = await waitForESRunning(240000); - if (elasticsearchReady) { - signale.success('Elasticsearch is ready to go'); - } -} - -function waitForESRunning(timeoutMs = 120000): Promise { - const endAt = Date.now() + timeoutMs; - - const _waitForESRunning = async (): Promise => { - if (Date.now() > endAt) { - throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); - } - - let elasticsearchRunning = false; - try { - const ESResponse = await execa.command('curl http://localhost:49200'); - if (ESResponse.stdout) { - const jsonData = JSON.parse(ESResponse.stdout); - console.log(`response: ${JSON.stringify(jsonData)}`); - if (jsonData.tagline === 'You Know, for Search') { - console.log('jsonData.tagline === You Know, for Search'); - elasticsearchRunning = true; - } - } - } catch (err) { - await PromiseTimeout(3000); - return _waitForESRunning(); - } - - if (elasticsearchRunning) { - return true; - } - await PromiseTimeout(3000); - return _waitForESRunning(); - }; - - return _waitForESRunning(); + signale.info(subprocess.stdout); + // console.log('deployElasticsearch subprocess: ', subprocess); } export async function deployKafka( @@ -664,17 +669,11 @@ export async function deployKafka( kafkaDeploymentYaml: string, zookeeperDeploymentYaml: string ) { - await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, zookeeperDeploymentYaml)}`); - let subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); - console.log('deployKafka subprocess: ', subprocess); - - // subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaServiceYaml)}`); - // console.log('deployKafkaService subprocess: ', subprocess); + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); + signale.info(subprocess.stdout); + // console.log('deployKafka subprocess: ', subprocess); - // const kafkaReady = await waitForKafkaRunning(240000); - // if (kafkaReady) { - signale.success('Kafka *might* be ready to go'); - // } + await waitForKafkaRunning(240000); } function waitForKafkaRunning(timeoutMs = 120000): Promise { @@ -682,45 +681,44 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { const _waitForKafkaRunning = async (): Promise => { if (Date.now() > endAt) { - throw new Error(`Failure to communicate with elasticsearch after ${timeoutMs}ms`); + throw new Error(`Failure to communicate with kafka after ${timeoutMs}ms`); } - let elasticsearchRunning = false; + let kafkaRunning = false; try { - const kafkaResponse = await execa.command('kubectl -n ts-dev1 get po '); - if (kafkaResponse.stdout) { - const jsonData = JSON.parse(kafkaResponse.stdout); - console.log(`response: ${JSON.stringify(jsonData)}`); - if (jsonData.tagline === 'You Know, for Search') { - console.log('jsonData.tagline === You Know, for Search'); - elasticsearchRunning = true; - } + const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=kafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + const kafkaReady = kubectlResponse.stdout; + // console.log('kafka response: ', kafkaReady); + if (kafkaReady === '"true"') { + kafkaRunning = true; } } catch (err) { - await PromiseTimeout(3000); + await pDelay(3000); return _waitForKafkaRunning(); } - if (elasticsearchRunning) { + if (kafkaRunning) { return true; } - await PromiseTimeout(3000); + await pDelay(3000); return _waitForKafkaRunning(); }; return _waitForKafkaRunning(); } -export async function k8sSetup( - e2eK8sDir: string, - roleYaml: string, - roleBindingYaml: string, - priorityClassYaml: string -): Promise { - const subprocess1 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleYaml)}`); - const subprocess2 = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, roleBindingYaml)}`); - const subprocess3 = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, priorityClassYaml)}`); - console.log('role, binding, priority, subprocesses: ', subprocess1, subprocess2, subprocess3); +export async function k8sSetup(): Promise { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + + let subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'role.yaml')}`); + signale.info(subprocess.stdout); + subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'roleBinding.yaml')}`); + signale.info(subprocess.stdout); + subprocess = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, 'priorityClass.yaml')}`); + signale.info(subprocess.stdout); } export async function deployK8sTeraslice( @@ -730,108 +728,45 @@ export async function deployK8sTeraslice( try { /// Creates configmap for terasclice-master let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); - console.log('masterConfig subprocess: ', subprocess); + // console.log('masterConfig subprocess: ', subprocess); + signale.info(subprocess.stdout); /// Creates configmap for teraslice-worker subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); - console.log('workerConfig subprocess: ', subprocess); + // console.log('workerConfig subprocess: ', subprocess); + signale.info(subprocess.stdout); /// Creates deployment for teraslice subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); - console.log('masterDeploy subprocess: ', subprocess); - - subprocess = await execa.command('kubectl get pods -n ts-dev1 --output name'); - const podName = subprocess.stdout.split('\n').filter((podString) => podString.includes('teraslice-master')); - console.log('podName: ', podName); - - const terasliceReady = await waitForTerasliceRunning(240000); - if (terasliceReady) { - signale.success('Teraslice is ready to go'); - } + // console.log('masterDeploy subprocess: ', subprocess); + signale.info(subprocess.stdout); } catch (error) { - console.log('@@@@@@@@ deployK8sTeraslice error: ', error); + signale.error('Error in deployK8sTeraslice function: ', error); } } -function waitForTerasliceRunning(timeoutMs = 120000): Promise { - const endAt = Date.now() + timeoutMs; - - const _waitForTerasliceRunning = async (): Promise => { - if (Date.now() > endAt) { - throw new Error(`Failure to communicate with the Teraslice Master after ${timeoutMs}ms`); - } - - let terasliceRunning = false; - try { - const TSMasterResponse = await execa.command('curl localhost:45678'); - if (TSMasterResponse.stdout) { - const jsonData = JSON.parse(TSMasterResponse.stdout); - console.log(`response: ${JSON.stringify(jsonData)}`); - if (jsonData.clustering_type === 'kubernetes') { - console.log('jsonData.clusteringType === kubernetes'); - terasliceRunning = true; - } - } - } catch (err) { - await PromiseTimeout(3000); - return _waitForTerasliceRunning(); - } - - if (terasliceRunning) { - return true; - } - - await PromiseTimeout(3000); - return _waitForTerasliceRunning(); - }; - - return _waitForTerasliceRunning(); -} - export async function setAlias() { - const subprocess1 = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); - const subprocess2 = await execa.command('earl aliases add k8se2e http://localhost:45678'); - console.log('setAlias subprocess: ', subprocess1, subprocess2); -} - -export async function registerTestJob() { - const subprocess = await execa.command('earl tjm register k8se2e testJob.json'); - console.log('registerTestJob subprocess: ', subprocess); -} - -export async function deployElasticsearchAssets() { - const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/elasticsearch-assets'); // FIXME: specify version - console.log('deployElasticsearchAssets subprocess: ', subprocess); -} - -export async function deployStandardAssets() { - const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/standard-assets'); // FIXME: specify version - console.log('deployStandardAssets subprocess: ', subprocess); + let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); + signale.info(subprocess.stdout); + subprocess = await execa.command('earl aliases add k8se2e http://localhost:45678'); + signale.info(subprocess.stdout); + // console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function deployKafkaAssets() { - const subprocess = await execa.command('earl assets deploy k8se2e --bundle --blocking terascope/kafka-assets'); // FIXME: specify version - console.log('deployKafkaAssets subprocess: ', subprocess); -} - -export async function startTestJob() { - const subprocess = await execa.command('earl tjm start testJob.json'); - console.log('Run earl tjm start testJob.json'); - console.log(subprocess.stdout); +export async function deployAssets(assetName: string) { + const subprocess = await execa.command(`earl assets deploy k8se2e --bundle --blocking terascope/${assetName}-assets`); + signale.info(subprocess.stdout); + // console.log('deployKafkaAssets subprocess: ', subprocess); } +// FIXME: delete before merging - for testing export async function showState() { const subprocess = await execa.command('kubectl -n ts-dev1 get deployments,po,svc -o wide'); - console.log('showState subprocess: ', subprocess); -} - -// FIXME: utils/src/promises/.ts pDelay already does this -function PromiseTimeout(delayms: number) { - return new Promise((resolve) => { - setTimeout(resolve, delayms); - }); + signale.info(subprocess.stdout); + // console.log('showState subprocess: ', subprocess); } +// FIXME: delete before merging if not needed in refactor export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { await tearDownTerasliceK8s(); await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); @@ -839,13 +774,13 @@ export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: st export async function tearDownTerasliceK8s() { let subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); - console.log('resetTeraslice subprocess: ', subprocess); + signale.info(subprocess.stdout); - /// Creates configmap for teraslice-worker subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); - console.log('workerConfig subprocess: ', subprocess); + signale.info(subprocess.stdout); + // console.log('workerConfig subprocess: ', subprocess); - /// Creates deployment for teraslice subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-worker || echo "* it is okay..."'); - console.log('masterDeploy subprocess: ', subprocess); + signale.info(subprocess.stdout); + // console.log('masterDeploy subprocess: ', subprocess); } diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 1e24901a8df..cd7227830fd 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -27,7 +27,7 @@ import { } from './utils'; import signale from '../signale'; import { - getE2EDir, readPackageInfo, listPackages, getE2eK8sDir + getE2EDir, readPackageInfo, listPackages } from '../packages'; import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; @@ -200,7 +200,7 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { - console.log('options: ', options); + // console.log('options: ', options); tracker.expected++; const suite = 'e2e'; @@ -212,26 +212,25 @@ async function runE2ETest( } if (process.env.TEST_PLATFORM === 'kubernetes') { - const e2eK8sDir = getE2eK8sDir(); - if (!e2eK8sDir) { - throw new Error('Missing k8s e2e test directory'); - } - const kindInstalled = await isKindInstalled(); - if (!kindInstalled) { - signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); - process.exit(1); - } + try { + const kindInstalled = await isKindInstalled(); + if (!kindInstalled) { + signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); + process.exit(1); + } - const kubectlInstalled = await isKubectlInstalled(); - if (!kubectlInstalled) { - signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); - process.exit(1); + const kubectlInstalled = await isKubectlInstalled(); + if (!kubectlInstalled) { + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + process.exit(1); + } + + await createKindCluster(); + await createNamespace(); + await k8sSetup(); + } catch (err) { + tracker.addError(err); } - // TODO: pass kind config files in as variables - // FIXME: error handling - await createKindCluster(e2eK8sDir, 'kindConfig.yaml'); - await createNamespace(e2eK8sDir, 'ns.yaml'); - await k8sSetup(e2eK8sDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); } const rootInfo = getRootInfo(); @@ -260,13 +259,18 @@ async function runE2ETest( } if (process.env.TEST_PLATFORM === 'kubernetes') { - await loadTerasliceImage(e2eImage); // FIXME: move to global.setup? + try { + await loadTerasliceImage(e2eImage); + } catch (err) { + tracker.addError(err); + } } try { + const svcFn = await ensureServices(suite, options); tracker.addCleanup( 'e2e:services', - await ensureServices(suite, options) + svcFn ); } catch (err) { tracker.addError(err); @@ -340,122 +344,3 @@ function printAndGetEnv(suite: string, options: TestOptions) { } return env; } - -// async function runk8sE2ETest( -// options: TestOptions, tracker: TestTracker -// ): Promise { -// console.log('options: ', options); -// tracker.expected++; - -// const k8se2eDir = getK8SE2EDir(); -// if (!k8se2eDir) { -// throw new Error('Missing k8se2e test directory'); -// } - -// const kindInstalled = await isKindInstalled(); -// if (!kindInstalled) { -// signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); -// process.exit(1); -// } - -// const kubectlInstalled = await isKubectlInstalled(); -// if (!kubectlInstalled) { -// signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); -// process.exit(1); -// } -// // TODO: pass kind config file in as a variable -// await createKindCluster(k8se2eDir, 'kindConfig.yaml'); - -// const suite = 'k8se2e'; -// let startedTest = false; - -// const rootInfo = getRootInfo(); -// const k8se2eImage = `${rootInfo.name}:k8se2e`; - -// // if (isCI) { -// // // pull the services first in CI -// // await pullServices(suite, options); -// // } - -// try { -// if (SKIP_DOCKER_BUILD_IN_E2E) { -// const devImage = `${getDevDockerImage()}-nodev${options.nodeVersion}`; -// await dockerTag(devImage, k8se2eImage); -// await loadTerasliceImage(k8se2eImage); -// } else { -// const publishOptions: PublishOptions = { -// dryRun: true, -// nodeVersion: options.nodeVersion, -// type: PublishType.Dev -// }; -// const devImage = await buildDevDockerImage(publishOptions); -// await dockerTag(devImage, k8se2eImage); -// await loadTerasliceImage(k8se2eImage); -// } -// } catch (err) { -// tracker.addError(err); -// } - -// TODO: add tracker -// await createNamespace(); -// await deployElasticSearch(k8se2eDir, 'elasticsearchDeployment.yaml'); -// await k8sSetup(k8se2eDir, 'role.yaml', 'roleBinding.yaml', 'priorityClass.yaml'); -// await deployk8sTeraslice(k8se2eDir, 'masterDeployment.yaml'); -// await showState(); - -// await setAlias(); -// await registerElasticsearchAssets(); -// await registerStandardAssets(); -// await registerTestJob(); -// await startTestJob(); - -// if (!tracker.hasErrors()) { -// const timeLabel = `test suite "${suite}"`; -// signale.time(timeLabel); -// startedTest = true; - -// const env = printAndGetEnv(suite, options); - -// tracker.started++; -// try { -// await runJest( -// k8se2eDir, -// getArgs(options), -// env, -// options.jestArgs, -// options.debug -// ); -// tracker.ended++; -// } catch (err) { -// tracker.ended++; -// tracker.addError(err.message); -// } - -// signale.timeEnd(timeLabel); -// } - -// if (!startedTest) return; - -// if (!options.keepOpen) { -// try { -// await logE2E(k8se2eDir, tracker.hasErrors()); -// } catch (err) { -// signale.error( -// new TSError(err, { -// reason: `Writing the "${suite}" logs failed`, -// }) -// ); -// } -// } - -// if (tracker.hasErrors()) { -// tracker.addCleanup('e2e:teardown', async () => { -// options.keepOpen = false; -// await globalTeardown(options, [{ -// name: suite, -// dir: k8se2eDir, -// suite, -// }]); -// }); -// } -// } diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 80dacf3cda9..c5df90ac310 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -137,8 +137,6 @@ const services: Readonly>> = { }; export async function pullServices(suite: string, options: TestOptions): Promise { - console.log('@@@ in pull services'); - const launchServices = getServicesForSuite(suite); try { @@ -193,7 +191,6 @@ export async function pullServices(suite: string, options: TestOptions): Promise } export async function ensureServices(suite: string, options: TestOptions): Promise<() => void> { - console.log('@@@ in ensure services'); const launchServices = getServicesForSuite(suite); const promises: Promise<(() => void)>[] = []; @@ -685,8 +682,6 @@ async function checkKafka(options: TestOptions, startTime: number) { } async function startService(options: TestOptions, service: Service): Promise<() => void> { - console.log('in startService: ', service); - let serviceName = service; if (serviceName === 'restrained_elasticsearch') { @@ -707,15 +702,15 @@ async function startService(options: TestOptions, service: Service): Promise<() if (process.env.TEST_PLATFORM === 'kubernetes') { await kindStopService(service); - // load via kind - console.log(`@@@@@@@ loading ${service} via kind`); + + // console.log(`@@@@@@@ loading ${service} via kind`); await kindLoadServiceImage(service); return () => { }; } await stopService(service); - console.log(`@@@@@@@ loading ${service} via docker`); + // console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], version, @@ -723,7 +718,6 @@ async function startService(options: TestOptions, service: Service): Promise<() options.debug || options.trace ); - console.log('@@@@@ fn(): ', fn.toString()); return () => { try { fn(); @@ -735,5 +729,4 @@ async function startService(options: TestOptions, service: Service): Promise<() ); } }; - } From fb37658d73e161aaf9f0668934ec7825a0d6822a Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 24 Oct 2023 11:39:00 -0700 Subject: [PATCH 094/142] Refactoring, typo fixes, remove logs. --- e2e/package.json | 1 + e2e/test/global.setup.js | 14 ++- e2e/test/global.teardown.js | 2 +- e2e/test/teraslice-harness.js | 2 +- packages/scripts/src/helpers/scripts.ts | 100 +++++++++--------- .../src/helpers/test-runner/services.ts | 3 - 6 files changed, 64 insertions(+), 58 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index b5291bb2862..2f9fec1e192 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -26,6 +26,7 @@ "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", + "test:k8sFast": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 0f3f315264f..14eea818529 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -51,10 +51,16 @@ module.exports = async () => { await teraslice.resetState(); if (process.env.TEST_PLATFORM === 'kubernetes') { - await setAlias(); - await deployAssets('elasticsearch'); - await deployAssets('standard'); - await deployAssets('kafka'); + try { + await setAlias(); + await deployAssets('elasticsearch'); + await deployAssets('standard'); + await deployAssets('kafka'); + } catch (err) { + signale.error('Setup failed'); + signale.error(err); + process.exit(1); + } } try { diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index e0605afb025..bd3465da691 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -26,7 +26,7 @@ async function globalTeardown(testClient) { const errors = []; try { - if (process.env.TEST_PATTERN === 'kubernetes') { + if (process.env.TEST_PLATFORM === 'kubernetes') { await tearDownTerasliceK8s(); } else { await tearDown(); diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index bde5676b4f5..b9902765476 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -124,7 +124,7 @@ module.exports = class TerasliceHarness { // Do nothing // TODO: If tests are ever implemented to scale nodes in Kind, // a scaleWorkers implementation will need to be created that works with Kind. - // As of 10/2023 Kind doesn't let you scale nodes w/o restarting the cluster. + // As of Oct 2023 Kind doesn't let you scale nodes w/o restarting the cluster. } })() ]); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 0ccf6f1ea92..adfa4ac1deb 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -545,7 +545,6 @@ export async function createKindCluster(): Promise { } export async function destroyKindCluster(): Promise { - // TODO: pass cluster name in as variable const subprocess = await execa.command('kind delete cluster --name k8se2e'); signale.info(subprocess.stderr); } @@ -556,7 +555,6 @@ export async function isKindInstalled(): Promise { // console.log('kind subprocess: ', subprocess); return !!subprocess.stdout; } catch (err) { - // console.log(err); return false; } } @@ -594,7 +592,7 @@ export async function loadTerasliceImage(terasliceImage: string): Promise // let terasliceLoaded = false; // try { // // docker exec into kind-control-plane -// // crictl images -o json +// // docker exec -it k8se2e crictl images -o json // // find repo tag "docker.io/library/teraslice-workspace:e2e" // const kubectlResponse = await execa.command(''); // const kindFinished = kubectlResponse.stdout; @@ -622,6 +620,7 @@ export async function kindStopService(serviceName: string): Promise { if (!e2eK8sDir) { throw new Error('Missing k8s e2e test directory'); } + try { // Any new service's yaml file must be named 'Deployment.yaml' const yamlFile = `${serviceName}Deployment.yaml`; @@ -633,47 +632,24 @@ export async function kindStopService(serviceName: string): Promise { } } -// FIXME: refactor export async function kindLoadServiceImage(serviceName: string): Promise { const e2eK8sDir = getE2eK8sDir(); if (!e2eK8sDir) { throw new Error('Missing k8s e2e test directory'); } - if (serviceName === 'elasticsearch') { - await deployElasticsearch(e2eK8sDir, 'elasticsearchDeployment.yaml'); - } else if (serviceName === 'kafka') { - await deployKafka(e2eK8sDir, 'kafkaDeployment.yaml', 'zookeeperDeployment.yaml'); - } else { - signale.error(`The service ${serviceName} is not avalable in the kubernetes test platform.`); - } -} -export async function createNamespace() { - const e2eK8sDir = getE2eK8sDir(); - if (!e2eK8sDir) { - throw new Error('Missing k8s e2e test directory'); + try { + // Any new service's yaml file must be named 'Deployment.yaml' + const yamlFile = `${serviceName}Deployment.yaml`; + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + signale.info(subprocess.stdout); + } catch (err) { + signale.error(`The service ${serviceName} could not be loaded.`); } - const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'ns.yaml')}`); - signale.info(subprocess.stdout); - // console.log('namespace subprocess: ', subprocess); -} -export async function deployElasticsearch(e2eK8sDir: string, elasticsearchYaml: string) { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, elasticsearchYaml)}`); - signale.info(subprocess.stdout); - // console.log('deployElasticsearch subprocess: ', subprocess); -} - -export async function deployKafka( - e2eK8sDir: string, - kafkaDeploymentYaml: string, - zookeeperDeploymentYaml: string -) { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); - signale.info(subprocess.stdout); - // console.log('deployKafka subprocess: ', subprocess); - - await waitForKafkaRunning(240000); + if (serviceName === 'kafka') { + await waitForKafkaRunning(240000); + } } function waitForKafkaRunning(timeoutMs = 120000): Promise { @@ -686,7 +662,7 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { let kafkaRunning = false; try { - const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=kafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); const kafkaReady = kubectlResponse.stdout; // console.log('kafka response: ', kafkaReady); if (kafkaReady === '"true"') { @@ -707,6 +683,16 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { return _waitForKafkaRunning(); } +export async function createNamespace() { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'ns.yaml')}`); + signale.info(subprocess.stdout); + // console.log('namespace subprocess: ', subprocess); +} + export async function k8sSetup(): Promise { const e2eK8sDir = getE2eK8sDir(); if (!e2eK8sDir) { @@ -740,8 +726,10 @@ export async function deployK8sTeraslice( subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); // console.log('masterDeploy subprocess: ', subprocess); signale.info(subprocess.stdout); - } catch (error) { - signale.error('Error in deployK8sTeraslice function: ', error); + } catch (err) { + signale.error('Error deploying Teraslice'); + signale.error(err); + process.exit(1); } } @@ -773,14 +761,28 @@ export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: st } export async function tearDownTerasliceK8s() { - let subprocess = await execa.command('kubectl delete --namespace ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); - signale.info(subprocess.stdout); - - subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-master || echo "* it is okay..."'); - signale.info(subprocess.stdout); - // console.log('workerConfig subprocess: ', subprocess); - - subprocess = await execa.command('kubectl delete --namespace ts-dev1 configmap teraslice-worker || echo "* it is okay..."'); - signale.info(subprocess.stdout); - // console.log('masterDeploy subprocess: ', subprocess); + try { + const subprocess = await execa.command('kubectl delete -n ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); + signale.info(subprocess.stdout); + // console.log('delete where app=teraslice subprocess: ', subprocess); + } catch (err) { + signale.error('Error in k8s teraslice teardown: ', err); + } + try { + const subprocess = await execa.command('kubectl delete -n ts-dev1 configmap teraslice-master'); + signale.info(subprocess.stdout); + // console.log('delete configmap master subprocess: ', subprocess); + } catch (err) { + console.log(err) + signale.warn(err); + } + try { + const subprocess = await execa.command('kubectl delete -n ts-dev1 configmap teraslice-worker'); + signale.info(subprocess.stdout); + // console.log('delete configmap worker subprocess: ', subprocess); + } catch (err) { + // Do nothing + console.log(err) + signale.warn(err); + } } diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index c5df90ac310..66630e48800 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -702,15 +702,12 @@ async function startService(options: TestOptions, service: Service): Promise<() if (process.env.TEST_PLATFORM === 'kubernetes') { await kindStopService(service); - - // console.log(`@@@@@@@ loading ${service} via kind`); await kindLoadServiceImage(service); return () => { }; } await stopService(service); - // console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], version, From 392ed6dfaab7380008227ddc6ebb9cb40edbbbb7 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 24 Oct 2023 15:08:00 -0700 Subject: [PATCH 095/142] Refactor kindLoadServiceImage create deleteWorkerDeploymentsAndPods --- e2e/test/teraslice-harness.js | 9 +++++ packages/scripts/src/helpers/scripts.ts | 49 +++++++++++++++++-------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index b9902765476..d095e24b988 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,6 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); +const { deleteWorkerDeploymentsAndPods, showState } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -90,6 +91,14 @@ module.exports = class TerasliceHarness { await Promise.all([ pDelay(800), cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`), + (async () => { + if (process.env.TEST_PLATFORM === 'kubernetes') { + await showState(); + await deleteWorkerDeploymentsAndPods(); + } else { + // Do nothing + } + })(), (async () => { const cleanupExIds = []; Object.values(state).forEach((node) => { diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index adfa4ac1deb..06b592944c8 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -633,21 +633,27 @@ export async function kindStopService(serviceName: string): Promise { } export async function kindLoadServiceImage(serviceName: string): Promise { - const e2eK8sDir = getE2eK8sDir(); - if (!e2eK8sDir) { - throw new Error('Missing k8s e2e test directory'); - } + // Any new service's yaml file must be named 'Deployment.yaml' + const fileName = `${serviceName}Deployment.yaml`; - try { - // Any new service's yaml file must be named 'Deployment.yaml' - const yamlFile = `${serviceName}Deployment.yaml`; - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); - signale.info(subprocess.stdout); - } catch (err) { - signale.error(`The service ${serviceName} could not be loaded.`); + async function createFromYaml(yamlFile: string) { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + + try { + const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + signale.info(subprocess.stdout); + } catch (err) { + signale.error(`The service ${serviceName} could not be loaded.`); + } } + await createFromYaml(fileName); + if (serviceName === 'kafka') { + await createFromYaml('zookeeperDeployment.yaml'); await waitForKafkaRunning(240000); } } @@ -747,6 +753,19 @@ export async function deployAssets(assetName: string) { // console.log('deployKafkaAssets subprocess: ', subprocess); } +export async function deleteWorkerDeploymentsAndPods() { + try { + let subprocess = await execa.command('kubectl -n ts-dev1 delete deployments -l app.kubernetes.io/component=worker'); + signale.info(subprocess.stdout); + console.log('deleteWorkerDeployments subprocess: ', subprocess); + subprocess = await execa.command('kubectl -n ts-dev1 delete pods -l app.kubernetes.io/component=execution_controller'); + signale.info(subprocess.stdout); + console.log('deleteWorkerDeployments subprocess: ', subprocess); + } catch (err) { + signale.error('Error deleting worker deployments: ', err); + } +} + // FIXME: delete before merging - for testing export async function showState() { const subprocess = await execa.command('kubectl -n ts-dev1 get deployments,po,svc -o wide'); @@ -773,8 +792,8 @@ export async function tearDownTerasliceK8s() { signale.info(subprocess.stdout); // console.log('delete configmap master subprocess: ', subprocess); } catch (err) { - console.log(err) - signale.warn(err); + console.log(err); + signale.info('info: ', err); } try { const subprocess = await execa.command('kubectl delete -n ts-dev1 configmap teraslice-worker'); @@ -782,7 +801,7 @@ export async function tearDownTerasliceK8s() { // console.log('delete configmap worker subprocess: ', subprocess); } catch (err) { // Do nothing - console.log(err) - signale.warn(err); + console.log(err); + signale.warn('warn: ', err); } } From 555304265da9ae0d19149ffbb559222806cb9e34 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 26 Oct 2023 13:04:04 -0700 Subject: [PATCH 096/142] Add testPlatform to options and config. Refactor --- e2e/test/config.js | 6 +- e2e/test/global.setup.js | 25 +--- e2e/test/global.teardown.js | 9 +- e2e/test/teraslice-harness.js | 108 ++++++++++-------- packages/scripts/src/cmds/test.ts | 10 +- packages/scripts/src/helpers/scripts.ts | 80 +++++++------ .../scripts/src/helpers/test-runner/index.ts | 12 +- .../src/helpers/test-runner/interfaces.ts | 1 + .../src/helpers/test-runner/services.ts | 2 +- packages/scripts/test/service-spec.ts | 3 +- packages/scripts/test/test-runner-spec.ts | 3 +- 11 files changed, 140 insertions(+), 119 deletions(-) diff --git a/e2e/test/config.js b/e2e/test/config.js index b97ffbb415d..9d4040b1560 100644 --- a/e2e/test/config.js +++ b/e2e/test/config.js @@ -34,7 +34,8 @@ const { KAFKA_BROKER = 'locahost:9092', HOST_IP = '127.0.0.1', GENERATE_ONLY, - TEST_OPENSEARCH = false + TEST_OPENSEARCH = false, + TEST_PLATFORM = 'native' } = process.env; const TEST_HOST = TEST_OPENSEARCH ? OPENSEARCH_HOST : ELASTICSEARCH_HOST; @@ -76,5 +77,6 @@ module.exports = { OPENSEARCH_VERSION, GENERATE_ONLY, newId, - TEST_HOST + TEST_HOST, + TEST_PLATFORM }; diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 14eea818529..3eccd1251c3 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -2,10 +2,8 @@ const { pDelay } = require('@terascope/utils'); const { - getE2eK8sDir, deployK8sTeraslice, - setAlias, - deployAssets + showState } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -14,7 +12,7 @@ const { dockerUp } = require('./docker-helpers'); const signale = require('./signale'); const setupTerasliceConfig = require('./setup-config'); const downloadAssets = require('./download-assets'); -const { CONFIG_PATH, ASSETS_PATH } = require('./config'); +const { CONFIG_PATH, ASSETS_PATH, TEST_PLATFORM } = require('./config'); module.exports = async () => { const teraslice = new TerasliceHarness(); @@ -38,9 +36,9 @@ module.exports = async () => { fse.ensureDir(CONFIG_PATH), ]); - if (process.env.TEST_PLATFORM === 'kubernetes') { - const e2eK8sDir = getE2eK8sDir(); - await deployK8sTeraslice(e2eK8sDir, 'masterDeployment.yaml'); + if (TEST_PLATFORM === 'kubernetes') { + await deployK8sTeraslice(); // here + await showState(); } else { await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); @@ -50,19 +48,6 @@ module.exports = async () => { await pDelay(2000); await teraslice.resetState(); - if (process.env.TEST_PLATFORM === 'kubernetes') { - try { - await setAlias(); - await deployAssets('elasticsearch'); - await deployAssets('standard'); - await deployAssets('kafka'); - } catch (err) { - signale.error('Setup failed'); - signale.error(err); - process.exit(1); - } - } - try { await teraslice.generateTestData(); } catch (err) { diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index bd3465da691..38dfa656e31 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -1,10 +1,10 @@ 'use strict'; const { ElasticsearchTestHelpers } = require('elasticsearch-store'); -const { tearDownTerasliceK8s } = require('@terascope/scripts'); +const { deleteTerasliceNamespace } = require('@terascope/scripts'); const fse = require('fs-extra'); const { - KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, TEST_INDEX_PREFIX + KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, TEST_INDEX_PREFIX, TEST_PLATFORM } = require('./config'); const { tearDown } = require('./docker-helpers'); const signale = require('./signale'); @@ -26,8 +26,9 @@ async function globalTeardown(testClient) { const errors = []; try { - if (process.env.TEST_PLATFORM === 'kubernetes') { - await tearDownTerasliceK8s(); + if (TEST_PLATFORM === 'kubernetes' && !process.env.KEEP_OPEN) { + await deleteTerasliceNamespace(); + await cleanupIndex(client, 'ts-dev1_*'); } else { await tearDown(); } diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index d095e24b988..45f2aa0fe53 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { deleteWorkerDeploymentsAndPods, showState } = require('@terascope/scripts'); +const { showState, deployK8sTeraslice, setAliasAndBaseAssets } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -13,7 +13,7 @@ const fse = require('fs-extra'); const { TEST_HOST, HOST_IP, SPEC_INDEX_PREFIX, DEFAULT_NODES, newId, DEFAULT_WORKERS, GENERATE_ONLY, - EXAMPLE_INDEX_SIZES, EXAMPLE_INDEX_PREFIX + EXAMPLE_INDEX_SIZES, EXAMPLE_INDEX_PREFIX, TEST_PLATFORM } = require('./config'); const { scaleWorkers, getElapsed } = require('./docker-helpers'); const signale = require('./signale'); @@ -88,55 +88,71 @@ module.exports = class TerasliceHarness { const startTime = Date.now(); const state = await this.teraslice.cluster.state(); - await Promise.all([ - pDelay(800), - cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`), - (async () => { - if (process.env.TEST_PLATFORM === 'kubernetes') { - await showState(); - await deleteWorkerDeploymentsAndPods(); - } else { - // Do nothing - } - })(), - (async () => { - const cleanupExIds = []; - Object.values(state).forEach((node) => { - const { assignment, ex_id: exId } = node; - - const isWorker = ['execution_controller', 'worker'].includes(assignment); - if (isWorker) { - cleanupExIds.push(exId); + if (TEST_PLATFORM === 'kubernetes') { + await Promise.all([ + (async () => { + try { + console.log('@@@@ before state reset'); + await showState(); + await cleanupIndex(this.client, 'ts-dev1_*'); + await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); + } catch (err) { + signale.error('Failure to clean indices', err); + throw err; } - }); - - await Promise.all( - uniq(cleanupExIds).map(async (exId) => { - signale.warn(`resetting ex ${exId}`); - try { - await this.teraslice.executions.wrap(exId).stop({ blocking: true }); - } catch (err) { - // ignore error; + })(), + (async () => { + try { + await deployK8sTeraslice(); + await this.waitForTeraslice(); + await setAliasAndBaseAssets(); + console.log('@@@@ after state reset'); + await showState(); + } catch (err) { + signale.error('Failure to reset teraslice state', err); + throw err; + } + })(), + // TODO: If tests are ever implemented to scale nodes in Kind, + // a scaleWorkers implementation will need to be created that works with Kind. + // As of Oct 2023 Kind doesn't let you scale nodes w/o restarting the cluster. + ]); + } else { + await Promise.all([ + pDelay(800), + cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`), + (async () => { + const cleanupExIds = []; + Object.values(state).forEach((node) => { + const { assignment, ex_id: exId } = node; + + const isWorker = ['execution_controller', 'worker'].includes(assignment); + if (isWorker) { + cleanupExIds.push(exId); } - }) - ); - })(), - (async () => { - if (process.env.TEST_PLATFORM === 'native') { + }); + + await Promise.all( + uniq(cleanupExIds).map(async (exId) => { + signale.warn(`resetting ex ${exId}`); + try { + await this.teraslice.executions.wrap(exId).stop({ blocking: true }); + } catch (err) { + // ignore error; + } + }) + ); + })(), + (async () => { const count = Object.keys(state).length; if (count !== DEFAULT_NODES) { signale.warn(`resetting cluster state of ${count} nodes`); await scaleWorkers(); await this.forWorkers(); } - } else { - // Do nothing - // TODO: If tests are ever implemented to scale nodes in Kind, - // a scaleWorkers implementation will need to be created that works with Kind. - // As of Oct 2023 Kind doesn't let you scale nodes w/o restarting the cluster. - } - })() - ]); + })() + ]); + } const elapsed = Date.now() - startTime; if (elapsed > 1000) { @@ -366,7 +382,7 @@ module.exports = class TerasliceHarness { return _waitForClusterState(); } - if (process.env.TEST_PLATFORM === 'kubernetes') { + if (TEST_PLATFORM === 'kubernetes') { // A get request to 'cluster/state' will return an empty object in kubernetes. // Therefore nodes will be 0. if (nodes === 0) return nodes; @@ -392,7 +408,7 @@ module.exports = class TerasliceHarness { return count; } } catch (err) { - // it probably okay + // it probably okay } await pDelay(50); @@ -412,7 +428,7 @@ module.exports = class TerasliceHarness { const nodes = await this.waitForClusterState(); - if (process.env.TEST_PLATFORM === 'kubernetes') { + if (TEST_PLATFORM === 'kubernetes') { signale.success('Teraslice is ready to go', getElapsed(startTime)); } else { signale.success(`Teraslice is ready to go with ${nodes} nodes`, getElapsed(startTime)); diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index 435932ae56b..3d3ad47fe5b 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -27,6 +27,7 @@ type Options = { 'use-existing-services': boolean; packages?: PackageInfo[]; 'ignore-mount': boolean; + 'test-platform': string; }; const jestArgs = getExtraArgs(); @@ -130,6 +131,11 @@ const cmd: CommandModule = { type: 'boolean', default: false, }) + .option('test-platform', { + description: 'Clustering platform for e2e tests', + type: 'string', + default: config.TEST_PLATFORM, + }) .positional('packages', { description: 'Runs the tests for one or more package and/or an asset, if none specified it will run all of the tests', coerce(arg) { @@ -159,6 +165,7 @@ const cmd: CommandModule = { const nodeVersion = hoistJestArg(argv, 'node-version', 'string'); const forceSuite = hoistJestArg(argv, 'force-suite', 'string'); const ignoreMount = hoistJestArg(argv, 'ignore-mount', 'boolean'); + const testPlatform = hoistJestArg(argv, 'test-platform', 'string'); if (debug && watch) { throw new Error('--debug and --watch conflict, please set one or the other'); @@ -183,7 +190,8 @@ const cmd: CommandModule = { all: !argv.packages || !argv.packages.length, reportCoverage, jestArgs, - ignoreMount + ignoreMount, + testPlatform }); }, }; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 06b592944c8..cc9006dd667 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -624,11 +624,11 @@ export async function kindStopService(serviceName: string): Promise { try { // Any new service's yaml file must be named 'Deployment.yaml' const yamlFile = `${serviceName}Deployment.yaml`; - const subprocess = await execa.command(`kubectl delete -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + const subprocess = await execa.command(`kubectl delete -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); signale.info(subprocess.stdout); // console.log('stopElasticsearch subprocess: ', subprocess); } catch (err) { - signale.warn(`The service ${serviceName} could not be deleted because it does not exist.`); + signale.info(`The service ${serviceName} could not be deleted because it does not exist.`); } } @@ -643,7 +643,7 @@ export async function kindLoadServiceImage(serviceName: string): Promise { } try { - const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + const subprocess = await execa.command(`kubectl create -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); signale.info(subprocess.stdout); } catch (err) { signale.error(`The service ${serviceName} could not be loaded.`); @@ -668,7 +668,8 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { let kafkaRunning = false; try { - const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + const kubectlResponse = await execa.command('kubectl -n services-dev1 get pods -l app=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + // const kubectlResponse = await execa.command('kubectl -n services-dev1 get pods -l app.kubernetes.io/name=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); const kafkaReady = kubectlResponse.stdout; // console.log('kafka response: ', kafkaReady); if (kafkaReady === '"true"') { @@ -689,12 +690,12 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { return _waitForKafkaRunning(); } -export async function createNamespace() { +export async function createNamespace(namespaceYaml: string) { const e2eK8sDir = getE2eK8sDir(); if (!e2eK8sDir) { throw new Error('Missing k8s e2e test directory'); } - const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'ns.yaml')}`); + const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); signale.info(subprocess.stdout); // console.log('namespace subprocess: ', subprocess); } @@ -713,10 +714,15 @@ export async function k8sSetup(): Promise { signale.info(subprocess.stdout); } -export async function deployK8sTeraslice( - e2eK8sDir: string, - masterDeploymentYaml: string -) { +export async function deployK8sTeraslice() { + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); + } + + await deleteTerasliceNamespace(); + await createNamespace('ts-ns.yaml'); + await k8sSetup(); try { /// Creates configmap for terasclice-master let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); @@ -729,7 +735,7 @@ export async function deployK8sTeraslice( signale.info(subprocess.stdout); /// Creates deployment for teraslice - subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, masterDeploymentYaml)}`); + subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, 'masterDeployment.yaml')}`); // console.log('masterDeploy subprocess: ', subprocess); signale.info(subprocess.stdout); } catch (err) { @@ -739,7 +745,14 @@ export async function deployK8sTeraslice( } } -export async function setAlias() { +export async function setAliasAndBaseAssets() { + await setAlias(); + await deployAssets('elasticsearch'); + await deployAssets('standard'); + await deployAssets('kafka'); +} + +async function setAlias() { let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); signale.info(subprocess.stdout); subprocess = await execa.command('earl aliases add k8se2e http://localhost:45678'); @@ -747,7 +760,7 @@ export async function setAlias() { // console.log('setAlias subprocess: ', subprocess1, subprocess2); } -export async function deployAssets(assetName: string) { +async function deployAssets(assetName: string) { const subprocess = await execa.command(`earl assets deploy k8se2e --bundle --blocking terascope/${assetName}-assets`); signale.info(subprocess.stdout); // console.log('deployKafkaAssets subprocess: ', subprocess); @@ -768,40 +781,35 @@ export async function deleteWorkerDeploymentsAndPods() { // FIXME: delete before merging - for testing export async function showState() { - const subprocess = await execa.command('kubectl -n ts-dev1 get deployments,po,svc -o wide'); + const subprocess = await execa.command('kubectl get deployments,po,svc -o wide --all-namespaces'); signale.info(subprocess.stdout); - // console.log('showState subprocess: ', subprocess); + console.log('showState subprocess: ', subprocess); + await showESIndices(); + await showAssets(); } -// FIXME: delete before merging if not needed in refactor -export async function resetTeraslice(e2eK8sDir: string, masterDeploymentYaml: string) { - await tearDownTerasliceK8s(); - await deployK8sTeraslice(e2eK8sDir, masterDeploymentYaml); +export async function showESIndices() { + const subprocess = await execa.command('curl localhost:49200/_cat/indices'); + signale.info(subprocess.stdout); + console.log('showESIndices subprocess: ', subprocess); } -export async function tearDownTerasliceK8s() { +export async function showAssets() { try { - const subprocess = await execa.command('kubectl delete -n ts-dev1 deployments,jobs,services,pods -l app=teraslice --grace-period=1'); + const subprocess = await execa.command('curl localhost:45678/v1/assets'); signale.info(subprocess.stdout); - // console.log('delete where app=teraslice subprocess: ', subprocess); - } catch (err) { - signale.error('Error in k8s teraslice teardown: ', err); - } - try { - const subprocess = await execa.command('kubectl delete -n ts-dev1 configmap teraslice-master'); - signale.info(subprocess.stdout); - // console.log('delete configmap master subprocess: ', subprocess); + + console.log('showAssets subprocess: ', subprocess); } catch (err) { - console.log(err); - signale.info('info: ', err); + signale.info(err); } +} + +export async function deleteTerasliceNamespace() { try { - const subprocess = await execa.command('kubectl delete -n ts-dev1 configmap teraslice-worker'); + const subprocess = await execa.command('kubectl delete namespace ts-dev1'); signale.info(subprocess.stdout); - // console.log('delete configmap worker subprocess: ', subprocess); } catch (err) { - // Do nothing - console.log(err); - signale.warn('warn: ', err); + signale.info('Teraslice namespace cannot be deleted because it does not exist'); } } diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index cd7227830fd..9d7898a13ac 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -200,7 +200,6 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { - // console.log('options: ', options); tracker.expected++; const suite = 'e2e'; @@ -211,7 +210,7 @@ async function runE2ETest( throw new Error('Missing e2e test directory'); } - if (process.env.TEST_PLATFORM === 'kubernetes') { + if (options.testPlatform === 'kubernetes') { try { const kindInstalled = await isKindInstalled(); if (!kindInstalled) { @@ -226,8 +225,7 @@ async function runE2ETest( } await createKindCluster(); - await createNamespace(); - await k8sSetup(); + await createNamespace('services-ns.yaml'); } catch (err) { tracker.addError(err); } @@ -236,7 +234,7 @@ async function runE2ETest( const rootInfo = getRootInfo(); const e2eImage = `${rootInfo.name}:e2e`; - if (isCI && process.env.TEST_PLATFORM === 'native') { + if (isCI && options.testPlatform === 'native') { // pull the services first in CI await pullServices(suite, options); } @@ -258,7 +256,7 @@ async function runE2ETest( tracker.addError(err); } - if (process.env.TEST_PLATFORM === 'kubernetes') { + if (options.testPlatform === 'kubernetes') { try { await loadTerasliceImage(e2eImage); } catch (err) { @@ -326,7 +324,7 @@ async function runE2ETest( }); } - // if (process.env.TEST_PLATFORM === 'kubernetes') { + // if (poptions.testPlatform === 'kubernetes') { // await destroyKindCluster(); // } } diff --git a/packages/scripts/src/helpers/test-runner/interfaces.ts b/packages/scripts/src/helpers/test-runner/interfaces.ts index fdb40b8a7af..e213a74a749 100644 --- a/packages/scripts/src/helpers/test-runner/interfaces.ts +++ b/packages/scripts/src/helpers/test-runner/interfaces.ts @@ -20,6 +20,7 @@ export type TestOptions = { nodeVersion: string; jestArgs?: string[]; ignoreMount: boolean; + testPlatform: string; }; export type GroupedPackages = { diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 66630e48800..fbbd216952f 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -700,7 +700,7 @@ async function startService(options: TestOptions, service: Service): Promise<() signale.pending(`starting ${service}@${version} service...`); - if (process.env.TEST_PLATFORM === 'kubernetes') { + if (options.testPlatform === 'kubernetes') { await kindStopService(service); await kindLoadServiceImage(service); return () => { }; diff --git a/packages/scripts/test/service-spec.ts b/packages/scripts/test/service-spec.ts index a0587bfedba..447039cefbf 100644 --- a/packages/scripts/test/service-spec.ts +++ b/packages/scripts/test/service-spec.ts @@ -19,7 +19,8 @@ describe('services', () => { rabbitmqVersion: 'very-bad-version', opensearchVersion: 'very-bad-version', nodeVersion: 'very-bad-version', - ignoreMount: false + ignoreMount: false, + testPlatform: 'native' }; describe('pullServices', () => { diff --git a/packages/scripts/test/test-runner-spec.ts b/packages/scripts/test/test-runner-spec.ts index 3c8259af576..8840f2e41de 100644 --- a/packages/scripts/test/test-runner-spec.ts +++ b/packages/scripts/test/test-runner-spec.ts @@ -25,7 +25,8 @@ describe('Test Runner Helpers', () => { rabbitmqVersion: '', opensearchVersion: '', nodeVersion: '', - ignoreMount: true + ignoreMount: true, + testPlatform: 'native' }; function makeTestOptions(input: Partial): TestOptions { From 573cdac0e631947b461ba1bc281bbed9d2861d3d Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 26 Oct 2023 14:55:43 -0700 Subject: [PATCH 097/142] Update yaml files with proper labels and hostnames --- e2e/k8s/elasticsearchDeployment.yaml | 18 +++++++++--------- e2e/k8s/kafkaDeployment.yaml | 20 ++++++++++---------- e2e/k8s/masterConfig/teraslice.yaml | 6 +++--- e2e/k8s/masterDeployment.yaml | 20 ++++++++++---------- e2e/k8s/services-ns.yaml | 4 ++++ e2e/k8s/{ns.yaml => ts-ns.yaml} | 0 e2e/k8s/workerConfig/teraslice.yaml | 6 +++--- e2e/k8s/zookeeperDeployment.yaml | 20 ++++++++++---------- 8 files changed, 49 insertions(+), 45 deletions(-) create mode 100644 e2e/k8s/services-ns.yaml rename e2e/k8s/{ns.yaml => ts-ns.yaml} (100%) diff --git a/e2e/k8s/elasticsearchDeployment.yaml b/e2e/k8s/elasticsearchDeployment.yaml index 5ea8287b1aa..0ba2dc9969d 100644 --- a/e2e/k8s/elasticsearchDeployment.yaml +++ b/e2e/k8s/elasticsearchDeployment.yaml @@ -3,19 +3,19 @@ kind: Deployment metadata: name: elasticsearch labels: - app: elasticsearch - nodeType: master + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master spec: replicas: 1 selector: matchLabels: - app: elasticsearch - nodeType: master + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master template: metadata: labels: - app: elasticsearch - nodeType: master + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master spec: containers: - name: elasticsearch @@ -33,11 +33,11 @@ apiVersion: v1 metadata: name: elasticsearch labels: - app: elasticsearch + app.kubernetes.io/name: elasticsearch spec: selector: - app: elasticsearch - nodeType: master + app.kubernetes.io/name: elasticsearch + app.kubernetes.io/component: master ports: - port: 9200 targetPort: 9200 diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index 90220bf0d6d..73521fcc029 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -3,19 +3,19 @@ kind: Deployment metadata: name: cpkafka labels: - app: cpkafka - nodeType: master + app.kubernetes.io/name: cpkafka + app.kubernetes.io/component: master spec: replicas: 1 selector: matchLabels: - app: cpkafka - nodeType: master + app.kubernetes.io/name: cpkafka + app.kubernetes.io/component: master template: metadata: labels: - app: cpkafka - nodeType: master + app.kubernetes.io/name: cpkafka + app.kubernetes.io/component: master spec: containers: - name: cpkafka @@ -26,7 +26,7 @@ spec: - name: KAFKA_ZOOKEEPER_CONNECT value: zookeeper:2181 - name: KAFKA_ADVERTISED_LISTENERS - value: INTERNAL://cpkafka:9092 + value: INTERNAL://cpkafka.services-dev1:9092 - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP value: INTERNAL:PLAINTEXT - name: KAFKA_INTER_BROKER_LISTENER_NAME @@ -41,12 +41,12 @@ apiVersion: v1 metadata: name: cpkafka labels: - app: cpkafka + app.kubernetes.io/name: cpkafka spec: type: NodePort selector: - app: cpkafka - nodeType: master + app.kubernetes.io/name: cpkafka + app.kubernetes.io/component: master ports: - port: 9092 name: cpkafka diff --git a/e2e/k8s/masterConfig/teraslice.yaml b/e2e/k8s/masterConfig/teraslice.yaml index 457d537d514..2fe3eac4add 100644 --- a/e2e/k8s/masterConfig/teraslice.yaml +++ b/e2e/k8s/masterConfig/teraslice.yaml @@ -6,15 +6,15 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:9200" + - "elasticsearch.services-dev1:9200" elasticsearch-next: default: node: - - "http://elasticsearch:9200" + - "http://elasticsearch.services-dev1:9200" kafka: default: brokers: - - "cpkafka:9092" + - "cpkafka.services-dev1:9092" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/masterDeployment.yaml b/e2e/k8s/masterDeployment.yaml index b5a521560de..e4576906427 100644 --- a/e2e/k8s/masterDeployment.yaml +++ b/e2e/k8s/masterDeployment.yaml @@ -3,20 +3,20 @@ kind: Deployment metadata: name: teraslice-master labels: - app: teraslice - nodeType: master + app.kubernetes.io/name: teraslice + app.kubernetes.io/component: master spec: replicas: 1 selector: matchLabels: - app: teraslice - nodeType: master + app.kubernetes.io/name: teraslice + app.kubernetes.io/component: master template: metadata: labels: - app: teraslice - nodeType: master - clusterName: k8se2e + app.kubernetes.io/name: teraslice + app.kubernetes.io/component: master + app.kubernetes.io/instance: k8se2e spec: containers: - name: teraslice-master @@ -41,11 +41,11 @@ apiVersion: v1 metadata: name: teraslice-master labels: - app: teraslice + app.kubernetes.io/name: teraslice spec: selector: - app: teraslice - nodeType: master + app.kubernetes.io/name: teraslice + app.kubernetes.io/component: master ports: - port: 5678 targetPort: 5678 diff --git a/e2e/k8s/services-ns.yaml b/e2e/k8s/services-ns.yaml new file mode 100644 index 00000000000..cd979b11d2b --- /dev/null +++ b/e2e/k8s/services-ns.yaml @@ -0,0 +1,4 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: services-dev1 diff --git a/e2e/k8s/ns.yaml b/e2e/k8s/ts-ns.yaml similarity index 100% rename from e2e/k8s/ns.yaml rename to e2e/k8s/ts-ns.yaml diff --git a/e2e/k8s/workerConfig/teraslice.yaml b/e2e/k8s/workerConfig/teraslice.yaml index 3b1870c3d27..26a51e52b3a 100644 --- a/e2e/k8s/workerConfig/teraslice.yaml +++ b/e2e/k8s/workerConfig/teraslice.yaml @@ -6,15 +6,15 @@ terafoundation: default: apiVersion: "5.6" host: - - "elasticsearch:9200" + - "elasticsearch.services-dev1:9200" elasticsearch-next: default: node: - - "http://elasticsearch:9200" + - "http://elasticsearch.services-dev1:9200" kafka: default: brokers: - - "cpkafka:9092" + - "cpkafka.services-dev1:9092" teraslice: worker_disconnect_timeout: 60000 node_disconnect_timeout: 60000 diff --git a/e2e/k8s/zookeeperDeployment.yaml b/e2e/k8s/zookeeperDeployment.yaml index d8069babb38..cbdfb47b72f 100644 --- a/e2e/k8s/zookeeperDeployment.yaml +++ b/e2e/k8s/zookeeperDeployment.yaml @@ -3,19 +3,19 @@ kind: Deployment metadata: name: zookeeper labels: - app: zookeeper - nodeType: master + app.kubernetes.io/name: zookeeper + app.kubernetes.io/component: master spec: replicas: 1 selector: matchLabels: - app: zookeeper - nodeType: master + app.kubernetes.io/name: zookeeper + app.kubernetes.io/component: master template: metadata: labels: - app: zookeeper - nodeType: master + app.kubernetes.io/name: zookeeper + app.kubernetes.io/component: master spec: containers: - name: zookeeper @@ -33,14 +33,14 @@ apiVersion: v1 metadata: name: zookeeper labels: - app: zookeeper + app.kubernetes.io/name: zookeeper spec: type: NodePort selector: - app: zookeeper - nodeType: master + app.kubernetes.io/name: zookeeper + app.kubernetes.io/component: master ports: - port: 2181 name: zookeeper targetPort: 2181 - nodePort: 32181 \ No newline at end of file + nodePort: 32181 From 7c5ba7dbc3840f9de09bb37b4b395844355db2bb Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 26 Oct 2023 14:57:20 -0700 Subject: [PATCH 098/142] Fix label used to test kafka --- packages/scripts/src/helpers/scripts.ts | 36 ++++++++++++------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index cc9006dd667..590d790f749 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -632,7 +632,19 @@ export async function kindStopService(serviceName: string): Promise { } } -export async function kindLoadServiceImage(serviceName: string): Promise { +// FIXME: implement this function +// export async function kindLoadServiceImage(serviceName: string): Promise { +// try { +// const serviceImage = ; +// const subprocess = await execa.command(`kind load docker-image ${serviceImage} --name k8se2e`); +// // console.log('load service image subprocess: ', subprocess); +// signale.info(subprocess.stderr); +// } catch (err) { +// signale.info(`The service ${serviceName} could not be loaded. It may not be present locally`); +// } +// } + +export async function kindStartService(serviceName: string): Promise { // Any new service's yaml file must be named 'Deployment.yaml' const fileName = `${serviceName}Deployment.yaml`; @@ -646,7 +658,7 @@ export async function kindLoadServiceImage(serviceName: string): Promise { const subprocess = await execa.command(`kubectl create -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); signale.info(subprocess.stdout); } catch (err) { - signale.error(`The service ${serviceName} could not be loaded.`); + signale.error(`The service ${serviceName} could not be started.`); } } @@ -668,8 +680,7 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { let kafkaRunning = false; try { - const kubectlResponse = await execa.command('kubectl -n services-dev1 get pods -l app=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); - // const kubectlResponse = await execa.command('kubectl -n services-dev1 get pods -l app.kubernetes.io/name=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + const kubectlResponse = await execa.command('kubectl -n services-dev1 get pods -l app.kubernetes.io/name=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); const kafkaReady = kubectlResponse.stdout; // console.log('kafka response: ', kafkaReady); if (kafkaReady === '"true"') { @@ -766,22 +777,9 @@ async function deployAssets(assetName: string) { // console.log('deployKafkaAssets subprocess: ', subprocess); } -export async function deleteWorkerDeploymentsAndPods() { - try { - let subprocess = await execa.command('kubectl -n ts-dev1 delete deployments -l app.kubernetes.io/component=worker'); - signale.info(subprocess.stdout); - console.log('deleteWorkerDeployments subprocess: ', subprocess); - subprocess = await execa.command('kubectl -n ts-dev1 delete pods -l app.kubernetes.io/component=execution_controller'); - signale.info(subprocess.stdout); - console.log('deleteWorkerDeployments subprocess: ', subprocess); - } catch (err) { - signale.error('Error deleting worker deployments: ', err); - } -} - -// FIXME: delete before merging - for testing +// FIXME: delete before merging? - for testing export async function showState() { - const subprocess = await execa.command('kubectl get deployments,po,svc -o wide --all-namespaces'); + const subprocess = await execa.command('kubectl get deployments,po,svc -o wide --all-namespaces --show-labels'); signale.info(subprocess.stdout); console.log('showState subprocess: ', subprocess); await showESIndices(); From 9ed219e9cf8d4de8d375be096ed87488794a6022 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 26 Oct 2023 14:58:21 -0700 Subject: [PATCH 099/142] Refactor so teraslice only starts once --- e2e/test/global.setup.js | 6 ++-- e2e/test/teraslice-harness.js | 53 ++++++++++++++++------------------- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 3eccd1251c3..014ff5bd020 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -37,14 +37,14 @@ module.exports = async () => { ]); if (TEST_PLATFORM === 'kubernetes') { - await deployK8sTeraslice(); // here - await showState(); + // await deployK8sTeraslice(); // here + // await showState(); } else { await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); + await teraslice.waitForTeraslice(); } - await teraslice.waitForTeraslice(); await pDelay(2000); await teraslice.resetState(); diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 45f2aa0fe53..b6bbf204d72 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -86,38 +86,33 @@ module.exports = class TerasliceHarness { async resetState() { const startTime = Date.now(); - const state = await this.teraslice.cluster.state(); if (TEST_PLATFORM === 'kubernetes') { - await Promise.all([ - (async () => { - try { - console.log('@@@@ before state reset'); - await showState(); - await cleanupIndex(this.client, 'ts-dev1_*'); - await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); - } catch (err) { - signale.error('Failure to clean indices', err); - throw err; - } - })(), - (async () => { - try { - await deployK8sTeraslice(); - await this.waitForTeraslice(); - await setAliasAndBaseAssets(); - console.log('@@@@ after state reset'); - await showState(); - } catch (err) { - signale.error('Failure to reset teraslice state', err); - throw err; - } - })(), - // TODO: If tests are ever implemented to scale nodes in Kind, - // a scaleWorkers implementation will need to be created that works with Kind. - // As of Oct 2023 Kind doesn't let you scale nodes w/o restarting the cluster. - ]); + try { + console.log('@@@@ before state reset'); + await showState(); + await cleanupIndex(this.client, 'ts-dev1_*'); + await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); + } catch (err) { + signale.error('Failure to clean indices', err); + throw err; + } + try { + await deployK8sTeraslice(); + await this.waitForTeraslice(); + await setAliasAndBaseAssets(); + console.log('@@@@ after state reset'); + await showState(); + } catch (err) { + signale.error('Failure to reset teraslice state', err); + throw err; + } + // TODO: If tests are ever implemented to scale nodes in Kind, + // a scaleWorkers implementation will need to be created that works with Kind. + // As of Oct 2023 Kind doesn't let you scale nodes w/o restarting the cluster. } else { + const state = await this.teraslice.cluster.state(); + await Promise.all([ pDelay(800), cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`), From 8de6358300841844b0d099d591bb8c60cb3c06de Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 26 Oct 2023 14:59:09 -0700 Subject: [PATCH 100/142] rename function that starts services --- packages/scripts/src/helpers/test-runner/services.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index fbbd216952f..ed41ae72997 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -11,7 +11,8 @@ import { getContainerInfo, dockerStop, dockerPull, - kindLoadServiceImage, + // kindLoadServiceImage, + kindStartService, kindStopService } from '../scripts'; import { TestOptions } from './interfaces'; @@ -702,7 +703,8 @@ async function startService(options: TestOptions, service: Service): Promise<() if (options.testPlatform === 'kubernetes') { await kindStopService(service); - await kindLoadServiceImage(service); + // await kindLoadServiceImage(service); + await kindStartService(service); return () => { }; } From 4bcec6c5dd8ebe6b8f1a0d42a8e5103375262ee8 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 07:13:23 -0700 Subject: [PATCH 101/142] implement kindLoadServiceImage function --- packages/scripts/src/helpers/scripts.ts | 32 ++++++++++--------- .../src/helpers/test-runner/services.ts | 4 +-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 590d790f749..587d09f6282 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -632,17 +632,18 @@ export async function kindStopService(serviceName: string): Promise { } } -// FIXME: implement this function -// export async function kindLoadServiceImage(serviceName: string): Promise { -// try { -// const serviceImage = ; -// const subprocess = await execa.command(`kind load docker-image ${serviceImage} --name k8se2e`); -// // console.log('load service image subprocess: ', subprocess); -// signale.info(subprocess.stderr); -// } catch (err) { -// signale.info(`The service ${serviceName} could not be loaded. It may not be present locally`); -// } -// } +// FIXME: implement this function with cp-kafka support +export async function kindLoadServiceImage( + serviceName: string, serviceImage: string, version: string +): Promise { + try { + const subprocess = await execa.command(`kind load docker-image ${serviceImage}:${version} --name k8se2e`); + // console.log('load service image subprocess: ', subprocess); + signale.info(subprocess.stderr); + } catch (err) { + signale.info(`The service ${serviceName} could not be loaded. It may not be present locally`); + } +} export async function kindStartService(serviceName: string): Promise { // Any new service's yaml file must be named 'Deployment.yaml' @@ -779,9 +780,9 @@ async function deployAssets(assetName: string) { // FIXME: delete before merging? - for testing export async function showState() { - const subprocess = await execa.command('kubectl get deployments,po,svc -o wide --all-namespaces --show-labels'); + const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels'); signale.info(subprocess.stdout); - console.log('showState subprocess: ', subprocess); + console.log('\nshowState subprocess: \n', subprocess.stdout); await showESIndices(); await showAssets(); } @@ -789,7 +790,7 @@ export async function showState() { export async function showESIndices() { const subprocess = await execa.command('curl localhost:49200/_cat/indices'); signale.info(subprocess.stdout); - console.log('showESIndices subprocess: ', subprocess); + console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } export async function showAssets() { @@ -797,9 +798,10 @@ export async function showAssets() { const subprocess = await execa.command('curl localhost:45678/v1/assets'); signale.info(subprocess.stdout); - console.log('showAssets subprocess: ', subprocess); + console.log('\nshowAssets subprocess: \n', subprocess.stdout); } catch (err) { signale.info(err); + console.log('\nshowAssets subprocess: \n', err); } } diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index ed41ae72997..f32c084b261 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -11,7 +11,7 @@ import { getContainerInfo, dockerStop, dockerPull, - // kindLoadServiceImage, + kindLoadServiceImage, kindStartService, kindStopService } from '../scripts'; @@ -703,7 +703,7 @@ async function startService(options: TestOptions, service: Service): Promise<() if (options.testPlatform === 'kubernetes') { await kindStopService(service); - // await kindLoadServiceImage(service); + await kindLoadServiceImage(service, services[service].image, version); await kindStartService(service); return () => { }; } From 547a60a36bca82e117537e086acce4e1e1088976 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 07:14:07 -0700 Subject: [PATCH 102/142] Add keep open variable to e2e config --- e2e/test/config.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e/test/config.js b/e2e/test/config.js index 9d4040b1560..7e967403bff 100644 --- a/e2e/test/config.js +++ b/e2e/test/config.js @@ -35,7 +35,8 @@ const { HOST_IP = '127.0.0.1', GENERATE_ONLY, TEST_OPENSEARCH = false, - TEST_PLATFORM = 'native' + TEST_PLATFORM = 'native', + KEEP_OPEN = false } = process.env; const TEST_HOST = TEST_OPENSEARCH ? OPENSEARCH_HOST : ELASTICSEARCH_HOST; @@ -78,5 +79,6 @@ module.exports = { GENERATE_ONLY, newId, TEST_HOST, - TEST_PLATFORM + TEST_PLATFORM, + KEEP_OPEN }; From c4865af3dbc5b9d928864986ac7b6611d6a31861 Mon Sep 17 00:00:00 2001 From: sotojn Date: Fri, 27 Oct 2023 07:46:53 -0700 Subject: [PATCH 103/142] add cp kafka & zkeeper images to e2e tests --- e2e/package.json | 1 + packages/scripts/src/cmds/test.ts | 2 + packages/scripts/src/helpers/config.ts | 36 ++++++++++- packages/scripts/src/helpers/interfaces.ts | 1 + packages/scripts/src/helpers/scripts.ts | 3 +- .../src/helpers/test-runner/interfaces.ts | 2 + .../src/helpers/test-runner/services.ts | 62 ++++++++++++++++--- packages/scripts/test/service-spec.ts | 2 + packages/scripts/test/test-runner-spec.ts | 2 + 9 files changed, 97 insertions(+), 14 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index b5291bb2862..4d8e779ebe7 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -26,6 +26,7 @@ "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", + "test:k8sfast": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index 435932ae56b..701be4dda58 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -176,6 +176,8 @@ const cmd: CommandModule = { elasticsearchVersion, elasticsearchAPIVersion, kafkaVersion, + kafkaImageVersion: config.KAFKA_IMAGE_VERSION, + zookeeperVersion: config.ZOOKEEPER_VERSION, minioVersion, rabbitmqVersion, opensearchVersion, diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index 889cce8bc74..92d75731e6b 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -8,7 +8,23 @@ const forceColor = process.env.FORCE_COLOR || '1'; export const FORCE_COLOR = toBoolean(forceColor) ? '1' : '0'; - +const kafkaMapper = { + 3: { + 0: '7.0.11', + 1: '7.1.9', + 2: '7.2.7', + 3: '7.3.5', + 4: '7.4.2', + 5: '7.5.1' + }, + 2: { + 4: '5.4.10', + 5: '5.5.12', + 6: '6.0.15', + 7: '6.1.13', + 8: '6.2.12' + } +}; /** The timeout for how long a service has to stand up */ export const SERVICE_UP_TIMEOUT = process.env.SERVICE_UP_TIMEOUT ?? '2m'; @@ -25,7 +41,7 @@ export const ELASTICSEARCH_PORT = process.env.ELASTICSEARCH_PORT || '49200'; export const ELASTICSEARCH_HOST = `http://${ELASTICSEARCH_HOSTNAME}:${ELASTICSEARCH_PORT}`; export const ELASTICSEARCH_VERSION = process.env.ELASTICSEARCH_VERSION || '6.8.6'; export const ELASTICSEARCH_API_VERSION = process.env.ELASTICSEARCH_API_VERSION || '6.5'; -export const ELASTICSEARCH_DOCKER_IMAGE = process.env.ELASTICSEARCH_DOCKER_IMAGE || 'blacktop/elasticsearch'; +export const ELASTICSEARCH_DOCKER_IMAGE = process.env.ELASTICSEARCH_DOCKER_IMAGE || 'elasticsearch'; export const RESTRAINED_ELASTICSEARCH_PORT = process.env.RESTRAINED_ELASTICSEARCH_PORT || '49202'; export const RESTRAINED_ELASTICSEARCH_HOST = `http://${ELASTICSEARCH_HOSTNAME}:${RESTRAINED_ELASTICSEARCH_PORT}`; @@ -35,7 +51,19 @@ export const KAFKA_HOSTNAME = process.env.KAFKA_HOSTNAME || HOST_IP; export const KAFKA_PORT = process.env.KAFKA_PORT || '49092'; export const KAFKA_BROKER = `${KAFKA_HOSTNAME}:${KAFKA_PORT}`; export const KAFKA_VERSION = process.env.KAFKA_VERSION || '3.1'; -export const KAFKA_DOCKER_IMAGE = process.env.KAFKA_DOCKER_IMAGE || 'blacktop/kafka'; +export const KAFKA_IMAGE_VERSION = kafkaMapper[KAFKA_VERSION.charAt(0)][KAFKA_VERSION.charAt(2)]; +export const KAFKA_DOCKER_IMAGE = process.env.KAFKA_DOCKER_IMAGE || 'confluentinc/cp-kafka'; +export const ZOOKEEPER_VERSION = kafkaMapper[KAFKA_VERSION.charAt(0)][KAFKA_VERSION.charAt(2)]; +export const ZOOKEEPER_CLIENT_PORT = process.env.ZOOKEEPER_CLIENT_PORT || '42181'; +export const ZOOKEEPER_TICK_TIME = process.env.ZOOKEEPER_TICK_TIME || '2000'; +export const ZOOKEEPER_DOCKER_IMAGE = process.env.ZOOKEEPER_DOCKER_IMAGE || 'confluentinc/cp-zookeeper'; +export const KAFKA_BROKER_ID = process.env.KAFKA_BROKER_ID || '1'; +export const KAFKA_ZOOKEEPER_CONNECT = `${KAFKA_HOSTNAME}:${ZOOKEEPER_CLIENT_PORT}`; +export const KAFKA_LISTENERS = `INTERNAL://:${KAFKA_PORT}`; +export const KAFKA_ADVERTISED_LISTENERS = `INTERNAL://${KAFKA_HOSTNAME}:${KAFKA_PORT}`; +export const KAFKA_LISTENER_SECURITY_PROTOCOL_MAP = 'INTERNAL:PLAINTEXT'; +export const KAFKA_INTER_BROKER_LISTENER_NAME = process.env.KAFKA_INTER_BROKER_LISTENER_NAME || 'INTERNAL'; +export const KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR = process.env.KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR || '1'; export const MINIO_NAME = process.env.MINIO_NAME || 'minio'; export const MINIO_HOSTNAME = process.env.MINIO_HOSTNAME || HOST_IP; @@ -133,6 +161,8 @@ export const ENV_SERVICES = [ testOpensearch ? Service.Opensearch : undefined, testElasticsearch ? Service.Elasticsearch : undefined, toBoolean(TEST_KAFKA) ? Service.Kafka : undefined, + /// couple kafa with zookeeper + toBoolean(TEST_KAFKA) ? Service.Zookeeper : undefined, toBoolean(TEST_MINIO) ? Service.Minio : undefined, testRestrainedOpensearch ? Service.RestrainedOpensearch : undefined, testRestrainedElasticsearch ? Service.RestrainedElasticsearch : undefined, diff --git a/packages/scripts/src/helpers/interfaces.ts b/packages/scripts/src/helpers/interfaces.ts index 6c8c1022c86..605d0ff1b32 100644 --- a/packages/scripts/src/helpers/interfaces.ts +++ b/packages/scripts/src/helpers/interfaces.ts @@ -37,6 +37,7 @@ export type PackageInfo = { export enum Service { Kafka = 'kafka', + Zookeeper = 'zookeeper', Elasticsearch = 'elasticsearch', Minio = 'minio', RabbitMQ = 'rabbitmq', diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 0ccf6f1ea92..a0dfbc664d7 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -669,6 +669,7 @@ export async function deployKafka( kafkaDeploymentYaml: string, zookeeperDeploymentYaml: string ) { + await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, zookeeperDeploymentYaml)}`); const subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, kafkaDeploymentYaml)}`); signale.info(subprocess.stdout); // console.log('deployKafka subprocess: ', subprocess); @@ -686,7 +687,7 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { let kafkaRunning = false; try { - const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=kafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); + const kubectlResponse: execa.ExecaReturnValue = await execa.command('kubectl -n ts-dev1 get pods -l app=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); const kafkaReady = kubectlResponse.stdout; // console.log('kafka response: ', kafkaReady); if (kafkaReady === '"true"') { diff --git a/packages/scripts/src/helpers/test-runner/interfaces.ts b/packages/scripts/src/helpers/test-runner/interfaces.ts index fdb40b8a7af..e41efbc86f5 100644 --- a/packages/scripts/src/helpers/test-runner/interfaces.ts +++ b/packages/scripts/src/helpers/test-runner/interfaces.ts @@ -14,6 +14,8 @@ export type TestOptions = { elasticsearchVersion: string; elasticsearchAPIVersion: string; kafkaVersion: string; + kafkaImageVersion: any; + zookeeperVersion: string; minioVersion: string; rabbitmqVersion: string; opensearchVersion: string; diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index c5df90ac310..ea0fb3a6699 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -99,12 +99,28 @@ const services: Readonly>> = { : undefined, ports: [`${config.KAFKA_PORT}:${config.KAFKA_PORT}`], env: { - KAFKA_HEAP_OPTS: config.SERVICE_HEAP_OPTS, - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true', + // KAFKA_HEAP_OPTS: config.SERVICE_HEAP_OPTS, + KAFKA_BROKER_ID: config.KAFKA_BROKER_ID, KAFKA_ADVERTISED_HOST_NAME: config.HOST_IP, - KAFKA_ADVERTISED_PORT: config.KAFKA_PORT, - KAFKA_PORT: config.KAFKA_PORT, - KAFKA_NUM_PARTITIONS: '2', + KAFKA_ZOOKEEPER_CONNECT: config.KAFKA_ZOOKEEPER_CONNECT, + KAFKA_LISTENERS: config.KAFKA_LISTENERS, + KAFKA_ADVERTISED_LISTENERS: config.KAFKA_ADVERTISED_LISTENERS, + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: config.KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, + KAFKA_INTER_BROKER_LISTENER_NAME: config.KAFKA_INTER_BROKER_LISTENER_NAME, + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: config.KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + }, + network: config.DOCKER_NETWORK_NAME + }, + [Service.Zookeeper]: { + image: config.ZOOKEEPER_DOCKER_IMAGE, + name: `${config.TEST_NAMESPACE}_zookeeper`, + tmpfs: config.SERVICES_USE_TMPFS + ? ['/tmp/zookeeper-logs'] + : undefined, + ports: [`${config.ZOOKEEPER_CLIENT_PORT}:${config.ZOOKEEPER_CLIENT_PORT}`], + env: { + ZOOKEEPER_CLIENT_PORT: config.ZOOKEEPER_CLIENT_PORT, + ZOOKEEPER_TICK_TIME: config.ZOOKEEPER_TICK_TIME }, network: config.DOCKER_NETWORK_NAME }, @@ -163,7 +179,12 @@ export async function pullServices(suite: string, options: TestOptions): Promise } if (launchServices.includes(Service.Kafka)) { - const image = `${config.KAFKA_DOCKER_IMAGE}:${options.kafkaVersion}`; + const image = `${config.KAFKA_DOCKER_IMAGE}:${options.kafkaImageVersion}`; + images.push(image); + } + + if (launchServices.includes(Service.Zookeeper)) { + const image = `${config.ZOOKEEPER_DOCKER_IMAGE}:${options.zookeeperVersion}`; images.push(image); } @@ -223,6 +244,10 @@ export async function ensureServices(suite: string, options: TestOptions): Promi promises.push(ensureKafka(options)); } + if (launchServices.includes(Service.Zookeeper)) { + promises.push(ensureZookeeper(options)); + } + if (launchServices.includes(Service.Minio)) { promises.push(ensureMinio(options)); } @@ -251,6 +276,14 @@ export async function ensureKafka(options: TestOptions): Promise<() => void> { return fn; } +export async function ensureZookeeper(options: TestOptions): Promise<() => void> { + let fn = () => { }; + const startTime = Date.now(); + fn = await startService(options, Service.Zookeeper); + await checkZookeeper(options, startTime); + return fn; +} + export async function ensureMinio(options: TestOptions): Promise<() => void> { let fn = () => { }; const startTime = Date.now(); @@ -681,6 +714,11 @@ async function checkKafka(options: TestOptions, startTime: number) { signale.success(`kafka@${options.kafkaVersion} *might* be running at ${config.KAFKA_BROKER}, took ${took}`); } +async function checkZookeeper(options: TestOptions, startTime: number) { + const took = ts.toHumanTime(Date.now() - startTime); + signale.success(` zookeeper*might* be running, took ${took}`); +} + async function startService(options: TestOptions, service: Service): Promise<() => void> { let serviceName = service; @@ -691,15 +729,19 @@ async function startService(options: TestOptions, service: Service): Promise<() if (serviceName === 'restrained_opensearch') { serviceName = Service.Opensearch; } - - const version = options[`${serviceName}Version`] as string; + let version:string; + if (serviceName === 'kafka') { + version = options[`${serviceName}ImageVersion`] as string; + signale.pending(`starting ${service}@${options.kafkaVersion} service...`); + } else { + version = options[`${serviceName}Version`] as string; + signale.pending(`starting ${service}@${version} service...`); + } if (options.useExistingServices) { signale.warn(`expecting ${service}@${version} to be running (this can be dangerous)...`); return () => { }; } - signale.pending(`starting ${service}@${version} service...`); - if (process.env.TEST_PLATFORM === 'kubernetes') { await kindStopService(service); diff --git a/packages/scripts/test/service-spec.ts b/packages/scripts/test/service-spec.ts index a0587bfedba..893f10606ce 100644 --- a/packages/scripts/test/service-spec.ts +++ b/packages/scripts/test/service-spec.ts @@ -15,6 +15,8 @@ describe('services', () => { elasticsearchVersion: 'bad-version', elasticsearchAPIVersion: '6.8', kafkaVersion: 'very-bad-version', + kafkaImageVersion: 'very-bad-version', + zookeeperVersion: 'very-bad-version', minioVersion: 'very-bad-version', rabbitmqVersion: 'very-bad-version', opensearchVersion: 'very-bad-version', diff --git a/packages/scripts/test/test-runner-spec.ts b/packages/scripts/test/test-runner-spec.ts index 3c8259af576..743bdade915 100644 --- a/packages/scripts/test/test-runner-spec.ts +++ b/packages/scripts/test/test-runner-spec.ts @@ -21,6 +21,8 @@ describe('Test Runner Helpers', () => { elasticsearchAPIVersion: '', elasticsearchVersion: '', kafkaVersion: '', + kafkaImageVersion: '', + zookeeperVersion: '', minioVersion: '', rabbitmqVersion: '', opensearchVersion: '', From 63ce8c3bba055e35f511c5efeea29f07c1c1f51f Mon Sep 17 00:00:00 2001 From: sotojn Date: Fri, 27 Oct 2023 10:31:59 -0700 Subject: [PATCH 104/142] fix bug that duplicates e2e logs --- packages/scripts/src/helpers/test-runner/services.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index f0c37227040..022788fa09e 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -214,6 +214,7 @@ export async function pullServices(suite: string, options: TestOptions): Promise export async function ensureServices(suite: string, options: TestOptions): Promise<() => void> { const launchServices = getServicesForSuite(suite); + console.log('@@@@ Here it is: ', launchServices); const promises: Promise<(() => void)>[] = []; @@ -743,8 +744,6 @@ async function startService(options: TestOptions, service: Service): Promise<() return () => { }; } - signale.pending(`starting ${service}@${version} service...`); - if (options.testPlatform === 'kubernetes') { await kindStopService(service); await kindLoadServiceImage(service, services[service].image, version); From 5a8b2a8c3a5f7557123bcde78db46b2da51756df Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 13:37:45 -0700 Subject: [PATCH 105/142] merge error fix, don't load zookeeper separately --- e2e/package.json | 4 +-- e2e/test/global.setup.js | 14 +++------ e2e/test/teraslice-harness.js | 13 ++++---- packages/scripts/src/helpers/scripts.ts | 30 ++++++++----------- .../scripts/src/helpers/test-runner/index.ts | 5 ++-- 5 files changed, 27 insertions(+), 39 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index 2f9fec1e192..9df5fab7580 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,8 +25,8 @@ "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", - "test:k8sFast": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", + "test:k8s": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index dd8ac8f4e56..0f23cc8ac56 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -3,7 +3,7 @@ const { pDelay } = require('@terascope/utils'); const { deployK8sTeraslice, - showState + setAliasAndBaseAssets } = require('@terascope/scripts'); const fse = require('fs-extra'); const TerasliceHarness = require('./teraslice-harness'); @@ -37,8 +37,9 @@ module.exports = async () => { ]); if (TEST_PLATFORM === 'kubernetes') { - // await deployK8sTeraslice(); // here - // await showState(); + await deployK8sTeraslice(); // here + await teraslice.waitForTeraslice(); + await setAliasAndBaseAssets(); } else { await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); @@ -48,13 +49,6 @@ module.exports = async () => { await pDelay(2000); await teraslice.resetState(); - if (process.env.TEST_PLATFORM === 'kubernetes') { - await setAlias(); - await deployAssets('elasticsearch'); - await deployAssets('standard'); - await deployAssets('kafka'); - } - try { await teraslice.generateTestData(); } catch (err) { diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index b6bbf204d72..887b5e0f324 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { showState, deployK8sTeraslice, setAliasAndBaseAssets } = require('@terascope/scripts'); +const { showState, deployK8sTeraslice } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -89,9 +89,9 @@ module.exports = class TerasliceHarness { if (TEST_PLATFORM === 'kubernetes') { try { - console.log('@@@@ before state reset'); - await showState(); - await cleanupIndex(this.client, 'ts-dev1_*'); + // console.log('@@@@ before state reset'); + // await showState(); + // await cleanupIndex(this.client, 'ts-dev1__assets'); await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); } catch (err) { signale.error('Failure to clean indices', err); @@ -100,9 +100,8 @@ module.exports = class TerasliceHarness { try { await deployK8sTeraslice(); await this.waitForTeraslice(); - await setAliasAndBaseAssets(); - console.log('@@@@ after state reset'); - await showState(); + // console.log('@@@@ after state reset'); + // await showState(); } catch (err) { signale.error('Failure to reset teraslice state', err); throw err; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 587d09f6282..c099be40b90 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -539,6 +539,7 @@ export async function createKindCluster(): Promise { if (!e2eK8sDir) { throw new Error('Missing k8s e2e test directory'); } + const configPath = path.join(e2eK8sDir, 'kindConfig.yaml'); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); signale.info(subprocess.stderr); @@ -647,26 +648,21 @@ export async function kindLoadServiceImage( export async function kindStartService(serviceName: string): Promise { // Any new service's yaml file must be named 'Deployment.yaml' - const fileName = `${serviceName}Deployment.yaml`; - - async function createFromYaml(yamlFile: string) { - const e2eK8sDir = getE2eK8sDir(); - if (!e2eK8sDir) { - throw new Error('Missing k8s e2e test directory'); - } + const yamlFile = `${serviceName}Deployment.yaml`; - try { - const subprocess = await execa.command(`kubectl create -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); - signale.info(subprocess.stdout); - } catch (err) { - signale.error(`The service ${serviceName} could not be started.`); - } + const e2eK8sDir = getE2eK8sDir(); + if (!e2eK8sDir) { + throw new Error('Missing k8s e2e test directory'); } - await createFromYaml(fileName); + try { + const subprocess = await execa.command(`kubectl create -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); + signale.info(subprocess.stdout); + } catch (err) { + signale.error(`The service ${serviceName} could not be started: `, err); + } if (serviceName === 'kafka') { - await createFromYaml('zookeeperDeployment.yaml'); await waitForKafkaRunning(240000); } } @@ -773,7 +769,7 @@ async function setAlias() { } async function deployAssets(assetName: string) { - const subprocess = await execa.command(`earl assets deploy k8se2e --bundle --blocking terascope/${assetName}-assets`); + const subprocess = await execa.command(`earl assets deploy k8se2e --blocking terascope/${assetName}-assets`); signale.info(subprocess.stdout); // console.log('deployKafkaAssets subprocess: ', subprocess); } @@ -801,7 +797,7 @@ export async function showAssets() { console.log('\nshowAssets subprocess: \n', subprocess.stdout); } catch (err) { signale.info(err); - console.log('\nshowAssets subprocess: \n', err); + // console.log('\nshowAssets subprocess: \n', err); } } diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 24a1cbc1909..14d9310845b 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -16,7 +16,6 @@ import { isKindInstalled, isKubectlInstalled, createNamespace, - k8sSetup, loadTerasliceImage, destroyKindCluster, } from '../scripts'; @@ -200,7 +199,7 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { - // console.log('options: ', options); + // console.log('@@@@@@@@ options: ', options); tracker.expected++; const suite = 'e2e'; @@ -325,7 +324,7 @@ async function runE2ETest( }); } - // if (poptions.testPlatform === 'kubernetes') { + // if (options.testPlatform === 'kubernetes') { // await destroyKindCluster(); // } } From 162638c1f0e44752a893fccf9401ae957975855f Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 14:59:28 -0700 Subject: [PATCH 106/142] Use cpkafka images that match with kafka 3.1 --- e2e/k8s/kafkaDeployment.yaml | 2 +- e2e/k8s/zookeeperDeployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/k8s/kafkaDeployment.yaml b/e2e/k8s/kafkaDeployment.yaml index 73521fcc029..9324aa418f6 100644 --- a/e2e/k8s/kafkaDeployment.yaml +++ b/e2e/k8s/kafkaDeployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: cpkafka - image: confluentinc/cp-kafka:7.2.0 + image: confluentinc/cp-kafka:7.1.9 env: - name: KAFKA_BROKER_ID value: "1" diff --git a/e2e/k8s/zookeeperDeployment.yaml b/e2e/k8s/zookeeperDeployment.yaml index cbdfb47b72f..9199994e951 100644 --- a/e2e/k8s/zookeeperDeployment.yaml +++ b/e2e/k8s/zookeeperDeployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: zookeeper - image: confluentinc/cp-zookeeper:7.2.0 + image: confluentinc/cp-zookeeper:7.1.9 env: - name: ZOOKEEPER_CLIENT_PORT value: "2181" From ee11c1cef5919baaaac37d135ec47c880dc56115 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 15:00:35 -0700 Subject: [PATCH 107/142] Hard code kafka and ES versions for k8s e2e --- e2e/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index 9df5fab7580..f1ad047ba8d 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -25,8 +25,8 @@ "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", - "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", From 2758c41c8555b14459d5e236575191dc7af18c84 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 16:05:41 -0700 Subject: [PATCH 108/142] Remove unneeded namespace file --- e2e/k8s/ns.yaml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 e2e/k8s/ns.yaml diff --git a/e2e/k8s/ns.yaml b/e2e/k8s/ns.yaml deleted file mode 100644 index 1af803c0c4f..00000000000 --- a/e2e/k8s/ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -kind: Namespace -apiVersion: v1 -metadata: - name: ts-dev1 \ No newline at end of file From 3eeffda0ece3cba35074d586d1d8e7c1084d7143 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 27 Oct 2023 16:09:56 -0700 Subject: [PATCH 109/142] Add comments, delete logs. --- e2e/test/teraslice-harness.js | 18 +++++++++++++++++- packages/scripts/src/helpers/scripts.ts | 4 ++-- .../src/helpers/test-runner/services.ts | 1 - 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 887b5e0f324..d09d1ed3287 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -91,8 +91,8 @@ module.exports = class TerasliceHarness { try { // console.log('@@@@ before state reset'); // await showState(); - // await cleanupIndex(this.client, 'ts-dev1__assets'); await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); + await this.clearNonBaseAssets(); } catch (err) { signale.error('Failure to clean indices', err); throw err; @@ -507,4 +507,20 @@ module.exports = class TerasliceHarness { throw err; } } + + async clearNonBaseAssets() { + const assetList = await this.teraslice.assets.list(); + // console.log('@@@@@ assetList: ', assetList); + + const baseAssets = ['standard', 'elasticsearch', 'kafka']; + const assetsToDelete = assetList.filter((assetObj) => !baseAssets.includes(assetObj.name)); + // console.log('@@@@@ assetsToDelete: ', assetsToDelete); + + for (const asset of assetsToDelete) { + // console.log('@@@@@ asset: ', asset); + const response = await this.teraslice.assets.remove(asset.id); + // console.log('@@@@@ response: ', response); + signale.success(`Deleted asset with id ${response}`); + } + } }; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index c099be40b90..08c3235a2c6 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -629,11 +629,11 @@ export async function kindStopService(serviceName: string): Promise { signale.info(subprocess.stdout); // console.log('stopElasticsearch subprocess: ', subprocess); } catch (err) { - signale.info(`The service ${serviceName} could not be deleted because it does not exist.`); + // Do nothing. This should fail because no services should be up yet. } } -// FIXME: implement this function with cp-kafka support +// TODO: Image versions are currently hard coded into yaml files export async function kindLoadServiceImage( serviceName: string, serviceImage: string, version: string ): Promise { diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 022788fa09e..9f4458fe2ca 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -214,7 +214,6 @@ export async function pullServices(suite: string, options: TestOptions): Promise export async function ensureServices(suite: string, options: TestOptions): Promise<() => void> { const launchServices = getServicesForSuite(suite); - console.log('@@@@ Here it is: ', launchServices); const promises: Promise<(() => void)>[] = []; From 29158357cc44b4dbde8f9e69a870ed30ce7ff99d Mon Sep 17 00:00:00 2001 From: busma13 Date: Mon, 30 Oct 2023 14:48:06 -0700 Subject: [PATCH 110/142] Add comments. --- e2e/jest.config.js | 4 +- packages/scripts/src/helpers/scripts.ts | 66 ++++++------------------- 2 files changed, 17 insertions(+), 53 deletions(-) diff --git a/e2e/jest.config.js b/e2e/jest.config.js index f211efc9d58..f6947d860d2 100644 --- a/e2e/jest.config.js +++ b/e2e/jest.config.js @@ -2,7 +2,9 @@ const config = require('../jest.config.base')(__dirname); -// FIXME update arrays to run tests specific to platform +// TODO: update arrays to run tests specific to platform. +// First array is for tests skipped in kubernetes. +// Second array is for tests skipped in native. config.testPathIgnorePatterns = process.env.TEST_PLATFORM === 'kubernetes' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : []; config.collectCoverage = false; delete config.transform; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 08c3235a2c6..0249a9bfdbf 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -570,51 +570,12 @@ export async function isKubectlInstalled(): Promise { } } +// TODO: check that image is loaded before we continue export async function loadTerasliceImage(terasliceImage: string): Promise { const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); // console.log('load teraslice image subprocess: ', subprocess); signale.info(subprocess.stderr); - - // FIXME: is it necessary to wait? - // const imageLoaded = await waitForTerasliceImageLoaded(); - // if (imageLoaded) { - // signale.success('Teraslice image successfully loaded.'); - // } -} - -// function waitForTerasliceImageLoaded(timeoutMs = 120000): Promise { -// const endAt = Date.now() + timeoutMs; - -// const _waitForTerasliceImageLoaded = async (): Promise => { -// if (Date.now() > endAt) { -// throw new Error(`Failure to load teraslice image after ${timeoutMs}ms`); -// } - -// let terasliceLoaded = false; -// try { -// // docker exec into kind-control-plane -// // docker exec -it k8se2e crictl images -o json -// // find repo tag "docker.io/library/teraslice-workspace:e2e" -// const kubectlResponse = await execa.command(''); -// const kindFinished = kubectlResponse.stdout; -// console.log('kubectl response: ', kubectlResponse); -// if (kindFinished === '"true"') { -// terasliceLoaded = true; -// } -// } catch (err) { -// await pDelay(3000); -// return _waitForTerasliceImageLoaded(); -// } - -// if (terasliceLoaded) { -// return true; -// } -// await pDelay(3000); -// return _waitForTerasliceImageLoaded(); -// }; - -// return _waitForTerasliceImageLoaded(); -// } +} export async function kindStopService(serviceName: string): Promise { const e2eK8sDir = getE2eK8sDir(); @@ -634,6 +595,7 @@ export async function kindStopService(serviceName: string): Promise { } // TODO: Image versions are currently hard coded into yaml files +// TODO: check that image is loaded before we continue export async function kindLoadServiceImage( serviceName: string, serviceImage: string, version: string ): Promise { @@ -774,6 +736,15 @@ async function deployAssets(assetName: string) { // console.log('deployKafkaAssets subprocess: ', subprocess); } +export async function deleteTerasliceNamespace() { + try { + const subprocess = await execa.command('kubectl delete namespace ts-dev1'); + signale.info(subprocess.stdout); + } catch (err) { + signale.info('Teraslice namespace cannot be deleted because it does not exist'); + } +} + // FIXME: delete before merging? - for testing export async function showState() { const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels'); @@ -783,13 +754,13 @@ export async function showState() { await showAssets(); } -export async function showESIndices() { +async function showESIndices() { const subprocess = await execa.command('curl localhost:49200/_cat/indices'); signale.info(subprocess.stdout); console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } -export async function showAssets() { +async function showAssets() { try { const subprocess = await execa.command('curl localhost:45678/v1/assets'); signale.info(subprocess.stdout); @@ -800,12 +771,3 @@ export async function showAssets() { // console.log('\nshowAssets subprocess: \n', err); } } - -export async function deleteTerasliceNamespace() { - try { - const subprocess = await execa.command('kubectl delete namespace ts-dev1'); - signale.info(subprocess.stdout); - } catch (err) { - signale.info('Teraslice namespace cannot be deleted because it does not exist'); - } -} From 86e789f215de61c6f23d86fa839c5ede2732e696 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 10:23:22 -0700 Subject: [PATCH 111/142] Add e2e-k8s-tests to github workflows test.yml --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82abc4d29d5..3adcc855148 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -245,3 +245,36 @@ jobs: - name: Test ${{ matrix.search-version }} run: yarn --silent test:${{ matrix.search-version }} --node-version ${{ matrix.node-version }} working-directory: ./e2e + + e2e-k8s-tests: + runs-on: ubuntu-latest + strategy: + # opensearch is finiky, keep testing others if it fails + fail-fast: false + matrix: + node-version: [16.19.1, 18.16.0] + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Setup Node ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'yarn' + + # we login to docker to avoid docker pull limit rates + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Install and build packages + run: yarn setup + env: + YARN_SETUP_ARGS: "--prod=false --silent" + + - name: Test k8s elasticsearch7 + run: yarn --silent test:k8s --node-version ${{ matrix.node-version }} + working-directory: ./e2e From 21a915f1ea86cc17e47b6746003f1cdc22aa89be Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 10:24:36 -0700 Subject: [PATCH 112/142] Update test to destroy kind cluster after tests --- packages/scripts/src/helpers/test-runner/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 14d9310845b..a34e087c52b 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -324,9 +324,9 @@ async function runE2ETest( }); } - // if (options.testPlatform === 'kubernetes') { - // await destroyKindCluster(); - // } + if (options.testPlatform === 'kubernetes') { + await destroyKindCluster(); + } } function printAndGetEnv(suite: string, options: TestOptions) { From ea42b239c8fd65ccf529e6a821a273b27584e23f Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 11:25:10 -0700 Subject: [PATCH 113/142] Fix e2e k8s test wirkflow to install kind first. --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3adcc855148..fc43cf03a8b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -275,6 +275,11 @@ jobs: env: YARN_SETUP_ARGS: "--prod=false --silent" + - name: Install Kind and Kubectl + uses: helm/kind-action@v1.8.0' + with: + install_only: "true" + - name: Test k8s elasticsearch7 run: yarn --silent test:k8s --node-version ${{ matrix.node-version }} working-directory: ./e2e From 204c40db626ffb773b68e1da0758a0551b068abf Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 11:32:44 -0700 Subject: [PATCH 114/142] Trying an older version of kind-action --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc43cf03a8b..147dfc8a1e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -276,7 +276,7 @@ jobs: YARN_SETUP_ARGS: "--prod=false --silent" - name: Install Kind and Kubectl - uses: helm/kind-action@v1.8.0' + uses: helm/kind-action@v1.5.0' with: install_only: "true" From cf682b8f74b64e7da58b197fb6557f62c304b6f1 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 12:17:55 -0700 Subject: [PATCH 115/142] Remove console logs --- e2e/test/teraslice-harness.js | 2 +- packages/scripts/src/helpers/scripts.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index d09d1ed3287..80056b64d06 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { showState, deployK8sTeraslice } = require('@terascope/scripts'); +const { deployK8sTeraslice } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 0249a9bfdbf..5fcbe6d42b4 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -749,7 +749,7 @@ export async function deleteTerasliceNamespace() { export async function showState() { const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels'); signale.info(subprocess.stdout); - console.log('\nshowState subprocess: \n', subprocess.stdout); + // console.log('\nshowState subprocess: \n', subprocess.stdout); await showESIndices(); await showAssets(); } @@ -757,7 +757,7 @@ export async function showState() { async function showESIndices() { const subprocess = await execa.command('curl localhost:49200/_cat/indices'); signale.info(subprocess.stdout); - console.log('\nshowESIndices subprocess: \n', subprocess.stdout); + // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } async function showAssets() { @@ -765,7 +765,7 @@ async function showAssets() { const subprocess = await execa.command('curl localhost:45678/v1/assets'); signale.info(subprocess.stdout); - console.log('\nshowAssets subprocess: \n', subprocess.stdout); + // console.log('\nshowAssets subprocess: \n', subprocess.stdout); } catch (err) { signale.info(err); // console.log('\nshowAssets subprocess: \n', err); From 97510fd59a922e524c6dd75309a02b42a9f33f78 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 12:57:59 -0700 Subject: [PATCH 116/142] Fix typo causing error in github action --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 147dfc8a1e5..e489c736560 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -276,7 +276,7 @@ jobs: YARN_SETUP_ARGS: "--prod=false --silent" - name: Install Kind and Kubectl - uses: helm/kind-action@v1.5.0' + uses: helm/kind-action@v1.8.0 with: install_only: "true" From 3263a55d9db7a9007a69337535c2806a06fb1196 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 13:26:51 -0700 Subject: [PATCH 117/142] Change expected ES cluster_name. New elasticsearch image uses 'docker-cluster' instead of 'elasticsearch' as its cluster name. --- packages/elasticsearch-store/test/client-functions-spec.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/elasticsearch-store/test/client-functions-spec.ts b/packages/elasticsearch-store/test/client-functions-spec.ts index b7c229ab799..91feab5dd35 100644 --- a/packages/elasticsearch-store/test/client-functions-spec.ts +++ b/packages/elasticsearch-store/test/client-functions-spec.ts @@ -56,11 +56,7 @@ describe('creates client that exposes elasticsearch and opensearch functions', ( const resp = await client.info(); if (clientMetadata.distribution === 'elasticsearch') { - if (clientMetadata.version.split('.')[0] === '8') { - expect(resp.cluster_name).toBe('docker-cluster'); - } else { - expect(resp.cluster_name).toBe(clientMetadata.distribution); - } + expect(resp.cluster_name).toBe('docker-cluster'); } if (clientMetadata.distribution === 'opensearch') { From 182b8f347548e9d52a51cdd72b219ca679154d7c Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 13:41:53 -0700 Subject: [PATCH 118/142] Add CI check to kind and kubectl install tests. --- packages/scripts/src/helpers/test-runner/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index a34e087c52b..ae3a54272a0 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -31,7 +31,7 @@ import { import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; import { TestTracker } from './tracker'; -import { MAX_PROJECTS_PER_BATCH, SKIP_DOCKER_BUILD_IN_E2E } from '../config'; +import { KEEP_OPEN, MAX_PROJECTS_PER_BATCH, SKIP_DOCKER_BUILD_IN_E2E } from '../config'; const logger = debugLogger('ts-scripts:cmd:test'); @@ -213,13 +213,13 @@ async function runE2ETest( if (options.testPlatform === 'kubernetes') { try { const kindInstalled = await isKindInstalled(); - if (!kindInstalled) { + if (!kindInstalled && !isCI) { signale.error('Please install Kind before running k8s tests. https://kind.sigs.k8s.io/docs/user/quick-start'); process.exit(1); } const kubectlInstalled = await isKubectlInstalled(); - if (!kubectlInstalled) { + if (!kubectlInstalled && !isCI) { signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); process.exit(1); } @@ -324,7 +324,7 @@ async function runE2ETest( }); } - if (options.testPlatform === 'kubernetes') { + if (options.testPlatform === 'kubernetes' && !KEEP_OPEN) { await destroyKindCluster(); } } From f8645aac53c6935bf15b0bc2151b57596b2efcd0 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 13:43:33 -0700 Subject: [PATCH 119/142] sync e2e package.json file --- e2e/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index f1ad047ba8d..64ddcf5ca17 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -24,12 +24,12 @@ "logs": "./scripts/logs.sh", "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", - "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", - "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --keep-open='true' --suite e2e --", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --" From 85e9d2c26efd920eb299a0a2a4b4f232eee77c1f Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 13:48:43 -0700 Subject: [PATCH 120/142] Get keep-open variable from options. --- packages/scripts/src/helpers/test-runner/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index ae3a54272a0..99051902ba0 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -31,7 +31,7 @@ import { import { buildDevDockerImage } from '../publish/utils'; import { PublishOptions, PublishType } from '../publish/interfaces'; import { TestTracker } from './tracker'; -import { KEEP_OPEN, MAX_PROJECTS_PER_BATCH, SKIP_DOCKER_BUILD_IN_E2E } from '../config'; +import { MAX_PROJECTS_PER_BATCH, SKIP_DOCKER_BUILD_IN_E2E } from '../config'; const logger = debugLogger('ts-scripts:cmd:test'); @@ -324,7 +324,7 @@ async function runE2ETest( }); } - if (options.testPlatform === 'kubernetes' && !KEEP_OPEN) { + if (options.testPlatform === 'kubernetes' && !options.keepOpen) { await destroyKindCluster(); } } From 7ef0ac37a3ce2d6e78edf20dbcd66b5a55d8fa70 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 14:49:38 -0700 Subject: [PATCH 121/142] Adding debug of HOST_IP --- e2e/test/global.setup.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 0f23cc8ac56..2a81c14d790 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -12,9 +12,12 @@ const { dockerUp } = require('./docker-helpers'); const signale = require('./signale'); const setupTerasliceConfig = require('./setup-config'); const downloadAssets = require('./download-assets'); -const { CONFIG_PATH, ASSETS_PATH, TEST_PLATFORM } = require('./config'); +const { + CONFIG_PATH, ASSETS_PATH, TEST_PLATFORM, HOST_IP +} = require('./config'); module.exports = async () => { + signale.debug('@@@@ HOST_IP: ', HOST_IP); const teraslice = new TerasliceHarness(); await teraslice.init(); From 8b74f5952c75de6a6ab92a98d15707fbbe1699e1 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 15:04:09 -0700 Subject: [PATCH 122/142] Add delays --- e2e/test/global.setup.js | 3 +++ e2e/test/teraslice-harness.js | 2 ++ 2 files changed, 5 insertions(+) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 2a81c14d790..2c64eccd4f9 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -41,8 +41,11 @@ module.exports = async () => { if (TEST_PLATFORM === 'kubernetes') { await deployK8sTeraslice(); // here + await pDelay(2000); await teraslice.waitForTeraslice(); + await pDelay(2000); await setAliasAndBaseAssets(); + await pDelay(2000); } else { await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 80056b64d06..0b4f68cead4 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -99,7 +99,9 @@ module.exports = class TerasliceHarness { } try { await deployK8sTeraslice(); + await pDelay(2000); await this.waitForTeraslice(); + await pDelay(2000); // console.log('@@@@ after state reset'); // await showState(); } catch (err) { From 05a2893d7773c0d47b18d8205bc2f846281ddd26 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 15:12:58 -0700 Subject: [PATCH 123/142] Remove keep-open from test script --- e2e/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/package.json b/e2e/package.json index 64ddcf5ca17..7c418160713 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -24,7 +24,7 @@ "logs": "./scripts/logs.sh", "logs-follow": "./scripts/logs.sh -f", "setup": "yarn --silent", - "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --keep-open='true' --suite e2e --", + "test": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", From 6b54826f4a1d38b48ac0d1f2183fe1fa9710c26a Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 15:20:36 -0700 Subject: [PATCH 124/142] Add more debugs --- packages/scripts/src/helpers/scripts.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 5fcbe6d42b4..d98a1105441 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -724,31 +724,31 @@ export async function setAliasAndBaseAssets() { async function setAlias() { let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); subprocess = await execa.command('earl aliases add k8se2e http://localhost:45678'); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); // console.log('setAlias subprocess: ', subprocess1, subprocess2); } async function deployAssets(assetName: string) { const subprocess = await execa.command(`earl assets deploy k8se2e --blocking terascope/${assetName}-assets`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); // console.log('deployKafkaAssets subprocess: ', subprocess); } export async function deleteTerasliceNamespace() { try { const subprocess = await execa.command('kubectl delete namespace ts-dev1'); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); } catch (err) { - signale.info('Teraslice namespace cannot be deleted because it does not exist'); + signale.debug('Teraslice namespace cannot be deleted because it does not exist'); } } // FIXME: delete before merging? - for testing export async function showState() { const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels'); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); // console.log('\nshowState subprocess: \n', subprocess.stdout); await showESIndices(); await showAssets(); @@ -756,18 +756,18 @@ export async function showState() { async function showESIndices() { const subprocess = await execa.command('curl localhost:49200/_cat/indices'); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } async function showAssets() { try { const subprocess = await execa.command('curl localhost:45678/v1/assets'); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); // console.log('\nshowAssets subprocess: \n', subprocess.stdout); } catch (err) { - signale.info(err); + signale.debug(err); // console.log('\nshowAssets subprocess: \n', err); } } From 82e39b7237c617d47c3f7550048297920c56041e Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 31 Oct 2023 15:26:41 -0700 Subject: [PATCH 125/142] Add host ip to setAlias function. --- e2e/test/global.setup.js | 2 +- packages/scripts/src/helpers/scripts.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 2c64eccd4f9..906f900bb2a 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -44,7 +44,7 @@ module.exports = async () => { await pDelay(2000); await teraslice.waitForTeraslice(); await pDelay(2000); - await setAliasAndBaseAssets(); + await setAliasAndBaseAssets(HOST_IP); await pDelay(2000); } else { await Promise.all([setupTerasliceConfig(), downloadAssets()]); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index d98a1105441..ebd83671f69 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -715,17 +715,17 @@ export async function deployK8sTeraslice() { } } -export async function setAliasAndBaseAssets() { - await setAlias(); +export async function setAliasAndBaseAssets(hostIP: string) { + await setAlias(hostIP); await deployAssets('elasticsearch'); await deployAssets('standard'); await deployAssets('kafka'); } -async function setAlias() { +async function setAlias(hostIP: string) { let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); signale.debug(subprocess.stdout); - subprocess = await execa.command('earl aliases add k8se2e http://localhost:45678'); + subprocess = await execa.command(`earl aliases add k8se2e http://${hostIP}:45678`); signale.debug(subprocess.stdout); // console.log('setAlias subprocess: ', subprocess1, subprocess2); } From 91b94040c64a65edf1c81394d2dc9cfcd1414e21 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 08:08:27 -0700 Subject: [PATCH 126/142] Add extra logging and debugging --- e2e/package.json | 4 +- e2e/test/global.setup.js | 3 -- e2e/test/teraslice-harness.js | 6 +-- packages/scripts/src/helpers/scripts.ts | 68 +++++++++++++++++-------- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index 7c418160713..e2066c0a92e 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -28,8 +28,8 @@ "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", - "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e -- --silent=false", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --", + "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --debug -- --silent=false", "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --" diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 906f900bb2a..0d052e3fb5d 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -41,11 +41,8 @@ module.exports = async () => { if (TEST_PLATFORM === 'kubernetes') { await deployK8sTeraslice(); // here - await pDelay(2000); await teraslice.waitForTeraslice(); - await pDelay(2000); await setAliasAndBaseAssets(HOST_IP); - await pDelay(2000); } else { await Promise.all([setupTerasliceConfig(), downloadAssets()]); await dockerUp(); diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 0b4f68cead4..046d25fa7f2 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { deployK8sTeraslice } = require('@terascope/scripts'); +const { deployK8sTeraslice, showState } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -24,7 +24,7 @@ const generateOnly = GENERATE_ONLY ? parseInt(GENERATE_ONLY, 10) : null; module.exports = class TerasliceHarness { async init() { - const { client } = await createClient({ node: TEST_HOST });// create ES or OS db + const { client } = await createClient({ node: TEST_HOST }); this.client = client; this.teraslice = new TerasliceClient({ host: `http://${HOST_IP}:45678`, @@ -103,7 +103,7 @@ module.exports = class TerasliceHarness { await this.waitForTeraslice(); await pDelay(2000); // console.log('@@@@ after state reset'); - // await showState(); + await showState(HOST_IP); } catch (err) { signale.error('Failure to reset teraslice state', err); throw err; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index ebd83671f69..da5c206facd 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -542,12 +542,14 @@ export async function createKindCluster(): Promise { const configPath = path.join(e2eK8sDir, 'kindConfig.yaml'); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); - signale.info(subprocess.stderr); + signale.debug(subprocess.stderr); + logger.debug(subprocess.stderr); } export async function destroyKindCluster(): Promise { const subprocess = await execa.command('kind delete cluster --name k8se2e'); - signale.info(subprocess.stderr); + signale.debug(subprocess.stderr); + logger.debug(subprocess.stderr); } export async function isKindInstalled(): Promise { @@ -574,7 +576,8 @@ export async function isKubectlInstalled(): Promise { export async function loadTerasliceImage(terasliceImage: string): Promise { const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); // console.log('load teraslice image subprocess: ', subprocess); - signale.info(subprocess.stderr); + signale.debug(subprocess.stderr); + logger.debug(subprocess.stderr); } export async function kindStopService(serviceName: string): Promise { @@ -587,7 +590,8 @@ export async function kindStopService(serviceName: string): Promise { // Any new service's yaml file must be named 'Deployment.yaml' const yamlFile = `${serviceName}Deployment.yaml`; const subprocess = await execa.command(`kubectl delete -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('stopElasticsearch subprocess: ', subprocess); } catch (err) { // Do nothing. This should fail because no services should be up yet. @@ -602,9 +606,11 @@ export async function kindLoadServiceImage( try { const subprocess = await execa.command(`kind load docker-image ${serviceImage}:${version} --name k8se2e`); // console.log('load service image subprocess: ', subprocess); - signale.info(subprocess.stderr); + signale.debug(subprocess.stderr); + logger.debug(subprocess.stderr); } catch (err) { - signale.info(`The service ${serviceName} could not be loaded. It may not be present locally`); + signale.debug(`The service ${serviceName} could not be loaded. It may not be present locally`); + logger.debug(`The service ${serviceName} could not be loaded. It may not be present locally`); } } @@ -619,9 +625,11 @@ export async function kindStartService(serviceName: string): Promise { try { const subprocess = await execa.command(`kubectl create -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); } catch (err) { signale.error(`The service ${serviceName} could not be started: `, err); + logger.error(`The service ${serviceName} could not be started: `, err); } if (serviceName === 'kafka') { @@ -666,7 +674,8 @@ export async function createNamespace(namespaceYaml: string) { throw new Error('Missing k8s e2e test directory'); } const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('namespace subprocess: ', subprocess); } @@ -677,11 +686,14 @@ export async function k8sSetup(): Promise { } let subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'role.yaml')}`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'roleBinding.yaml')}`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); subprocess = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, 'priorityClass.yaml')}`); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); } export async function deployK8sTeraslice() { @@ -697,20 +709,25 @@ export async function deployK8sTeraslice() { /// Creates configmap for terasclice-master let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); // console.log('masterConfig subprocess: ', subprocess); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); /// Creates configmap for teraslice-worker subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); // console.log('workerConfig subprocess: ', subprocess); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); /// Creates deployment for teraslice subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, 'masterDeployment.yaml')}`); // console.log('masterDeploy subprocess: ', subprocess); - signale.info(subprocess.stdout); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); } catch (err) { signale.error('Error deploying Teraslice'); signale.error(err); + logger.error('Error deploying Teraslice'); + logger.error(err); process.exit(1); } } @@ -725,14 +742,17 @@ export async function setAliasAndBaseAssets(hostIP: string) { async function setAlias(hostIP: string) { let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); subprocess = await execa.command(`earl aliases add k8se2e http://${hostIP}:45678`); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('setAlias subprocess: ', subprocess1, subprocess2); } async function deployAssets(assetName: string) { const subprocess = await execa.command(`earl assets deploy k8se2e --blocking terascope/${assetName}-assets`); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('deployKafkaAssets subprocess: ', subprocess); } @@ -740,34 +760,40 @@ export async function deleteTerasliceNamespace() { try { const subprocess = await execa.command('kubectl delete namespace ts-dev1'); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); } catch (err) { signale.debug('Teraslice namespace cannot be deleted because it does not exist'); + logger.debug('Teraslice namespace cannot be deleted because it does not exist'); } } // FIXME: delete before merging? - for testing -export async function showState() { +export async function showState(hostIP: string) { const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels'); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('\nshowState subprocess: \n', subprocess.stdout); - await showESIndices(); - await showAssets(); + await showESIndices(hostIP); + await showAssets(hostIP); } -async function showESIndices() { - const subprocess = await execa.command('curl localhost:49200/_cat/indices'); +async function showESIndices(hostIP: string) { + const subprocess = await execa.command(`curl ${hostIP}:49200/_cat/indices`); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } -async function showAssets() { +async function showAssets(hostIP: string) { try { - const subprocess = await execa.command('curl localhost:45678/v1/assets'); + const subprocess = await execa.command(`curl ${hostIP}:45678/v1/assets`); signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); // console.log('\nshowAssets subprocess: \n', subprocess.stdout); } catch (err) { signale.debug(err); + logger.debug(err); // console.log('\nshowAssets subprocess: \n', err); } } From 06d5c0b9cd101231f8d04eb2288dbcfc02315894 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 09:48:31 -0700 Subject: [PATCH 127/142] Add more logs --- e2e/package.json | 4 ++-- e2e/test/teraslice-harness.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index e2066c0a92e..3f9d84fa863 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -28,8 +28,8 @@ "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --", - "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --debug -- --silent=false", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug -- --silent-false", + "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug -- --silent=false", "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --" diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 046d25fa7f2..8f69ee87312 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -90,7 +90,7 @@ module.exports = class TerasliceHarness { if (TEST_PLATFORM === 'kubernetes') { try { // console.log('@@@@ before state reset'); - // await showState(); + // await showState(HOST_IP); await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); await this.clearNonBaseAssets(); } catch (err) { @@ -478,7 +478,9 @@ module.exports = class TerasliceHarness { await Promise.all(executions.map((ex) => this.waitForExStatus(ex, 'completed'))); } else { const ex = await this.postJob(jobSpec); + showState(HOST_IP); await this.waitForExStatus(ex, 'completed'); + showState(HOST_IP); } signale.info(`Generated ${indexName} example data`, getElapsed(genStartTime)); From bbd27246e0ea462ee86a6be22c07eca65e3356fb Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 10:20:53 -0700 Subject: [PATCH 128/142] More log fixes --- e2e/package.json | 2 +- e2e/test/teraslice-harness.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index 3f9d84fa863..f832c0e7825 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -28,7 +28,7 @@ "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug -- --silent-false", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug --", "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug -- --silent=false", "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 8f69ee87312..0b045d225cc 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -478,9 +478,9 @@ module.exports = class TerasliceHarness { await Promise.all(executions.map((ex) => this.waitForExStatus(ex, 'completed'))); } else { const ex = await this.postJob(jobSpec); - showState(HOST_IP); + if (process.env.TEST_PLATFORM === 'kubernetes') showState(HOST_IP); await this.waitForExStatus(ex, 'completed'); - showState(HOST_IP); + if (process.env.TEST_PLATFORM === 'kubernetes') showState(HOST_IP); } signale.info(`Generated ${indexName} example data`, getElapsed(genStartTime)); From bcb2f4fd73d56e2a0673c6238b0bbcbac0f04e13 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 10:45:20 -0700 Subject: [PATCH 129/142] more logging --- e2e/test/teraslice-harness.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 0b045d225cc..d96ffd02b38 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -478,9 +478,15 @@ module.exports = class TerasliceHarness { await Promise.all(executions.map((ex) => this.waitForExStatus(ex, 'completed'))); } else { const ex = await this.postJob(jobSpec); - if (process.env.TEST_PLATFORM === 'kubernetes') showState(HOST_IP); + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.debug('@@@ after postJob'); + signale.debug(await showState(HOST_IP)); + } await this.waitForExStatus(ex, 'completed'); - if (process.env.TEST_PLATFORM === 'kubernetes') showState(HOST_IP); + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.debug('@@@ after waitForExStatus'); + signale.debug(await showState(HOST_IP)); + } } signale.info(`Generated ${indexName} example data`, getElapsed(genStartTime)); From 32e82d8e6ae960f23638be737cd2e558a8b63c3a Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 10:54:41 -0700 Subject: [PATCH 130/142] more logs --- e2e/test/teraslice-harness.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index d96ffd02b38..65809537531 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -57,6 +57,10 @@ module.exports = class TerasliceHarness { const start = Date.now(); try { + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.debug('@@@ before ex.waitForExStatus'); + signale.debug(await showState(HOST_IP)); + } const result = await ex.waitForStatus(status, interval, 2 * 60 * 1000); if (endDelay) { // since most of the time we are chaining this with other actions @@ -64,8 +68,16 @@ module.exports = class TerasliceHarness { // it a little bit of time await pDelay(endDelay); } + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.debug('@@@ after ex.waitForExStatus'); + signale.debug(await showState(HOST_IP)); + } return result; } catch (err) { + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.debug('@@@ in catch of waitForExStatus'); + signale.debug(await showState(HOST_IP)); + } err.message = `Execution: ${ex.id()}: ${err.message}`; this.warn(err.stack, { From aa5155b94569388afdaecb650098fac12eed32bb Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 11:05:56 -0700 Subject: [PATCH 131/142] log es data --- e2e/test/teraslice-harness.js | 3 ++- packages/scripts/src/helpers/scripts.ts | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 65809537531..eeeea7ff51f 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { deployK8sTeraslice, showState } = require('@terascope/scripts'); +const { deployK8sTeraslice, showState, showESData } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -77,6 +77,7 @@ module.exports = class TerasliceHarness { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ in catch of waitForExStatus'); signale.debug(await showState(HOST_IP)); + signale.debug(await showESData(HOST_IP)); } err.message = `Execution: ${ex.id()}: ${err.message}`; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index da5c206facd..6117201528b 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -784,6 +784,19 @@ async function showESIndices(hostIP: string) { // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } +export async function showESData(hostIP: string) { + let subprocess = await execa.command(`curl ${hostIP}:49200/ts-dev1__ex/_search`); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); + subprocess = await execa.command(`curl ${hostIP}:49200/ts-dev1__analytics-2023.11/_search`); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); + subprocess = await execa.command(`curl ${hostIP}:49200/ts-dev1__state-2023.11/_search`); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); + // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); +} + async function showAssets(hostIP: string) { try { const subprocess = await execa.command(`curl ${hostIP}:45678/v1/assets`); From f3df25740b8b94cd4515833f38d2f0ca62b0ed04 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 13:07:45 -0700 Subject: [PATCH 132/142] show master logs --- e2e/test/teraslice-harness.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index eeeea7ff51f..2d750b6a41f 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { deployK8sTeraslice, showState, showESData } = require('@terascope/scripts'); +const { deployK8sTeraslice, showState, showTSMasterLogs } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -77,7 +77,7 @@ module.exports = class TerasliceHarness { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ in catch of waitForExStatus'); signale.debug(await showState(HOST_IP)); - signale.debug(await showESData(HOST_IP)); + signale.debug(await showTSMasterLogs(HOST_IP)); } err.message = `Execution: ${ex.id()}: ${err.message}`; @@ -445,7 +445,13 @@ module.exports = class TerasliceHarness { } async postJob(jobSpec) { - return this.teraslice.executions.submit(jobSpec); + const result = this.teraslice.executions.submit(jobSpec); + signale.debug('@@@ postJob result: ', result); + if (process.env.TEST_PLATFORM === 'kubernetes') { + signale.debug('@@@ after postJob result'); + signale.debug(await showState(HOST_IP)); + } + return result; } async generate(count, hex) { From 05b3be5abb66bafc8c62bcf6ac0595c65b760330 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 13:14:06 -0700 Subject: [PATCH 133/142] Add worker and ex logs --- e2e/test/teraslice-harness.js | 6 ++++-- packages/scripts/src/helpers/scripts.ts | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 2d750b6a41f..02ef7cb8edd 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,7 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { deployK8sTeraslice, showState, showTSMasterLogs } = require('@terascope/scripts'); +const { deployK8sTeraslice, showState, showTSMasterLogs, showTSExLogs, showTSWorkerLogs } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -77,7 +77,9 @@ module.exports = class TerasliceHarness { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ in catch of waitForExStatus'); signale.debug(await showState(HOST_IP)); - signale.debug(await showTSMasterLogs(HOST_IP)); + signale.debug(await showTSMasterLogs()); + signale.debug(await showTSExLogs()); + signale.debug(await showTSWorkerLogs()); } err.message = `Execution: ${ex.id()}: ${err.message}`; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 6117201528b..0e410c9bfb7 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -797,6 +797,24 @@ export async function showESData(hostIP: string) { // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); } +export async function showTSMasterLogs() { + const subprocess = await execa.command('kubectl -n tsdev1 logs -l app.kubernetes.io/component=master'); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); +} + +export async function showTSExLogs() { + const subprocess = await execa.command('kubectl -n tsdev1 logs -l app.kubernetes.io/component=execution_controller'); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); +} + +export async function showTSWorkerLogs() { + const subprocess = await execa.command('kubectl -n tsdev1 logs -l app.kubernetes.io/component=worker'); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); +} + async function showAssets(hostIP: string) { try { const subprocess = await execa.command(`curl ${hostIP}:45678/v1/assets`); From 131ca885724d0d1f5dcd25b3e08cb72abbb254c2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 13:26:56 -0700 Subject: [PATCH 134/142] fix typo --- packages/scripts/src/helpers/scripts.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 0e410c9bfb7..59be5c0530a 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -798,19 +798,19 @@ export async function showESData(hostIP: string) { } export async function showTSMasterLogs() { - const subprocess = await execa.command('kubectl -n tsdev1 logs -l app.kubernetes.io/component=master'); + const subprocess = await execa.command('kubectl -n ts-dev1 logs -l app.kubernetes.io/component=master'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } export async function showTSExLogs() { - const subprocess = await execa.command('kubectl -n tsdev1 logs -l app.kubernetes.io/component=execution_controller'); + const subprocess = await execa.command('kubectl -n ts-dev1 logs -l app.kubernetes.io/component=execution_controller'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } export async function showTSWorkerLogs() { - const subprocess = await execa.command('kubectl -n tsdev1 logs -l app.kubernetes.io/component=worker'); + const subprocess = await execa.command('kubectl -n ts-dev1 logs -l app.kubernetes.io/component=worker'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } From ecf3cbed896b0e322a6896dbde0c17ddb4fc00c8 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 14:04:24 -0700 Subject: [PATCH 135/142] change log length to 1000 --- packages/scripts/src/helpers/scripts.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 59be5c0530a..bd0f44d97cf 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -798,19 +798,19 @@ export async function showESData(hostIP: string) { } export async function showTSMasterLogs() { - const subprocess = await execa.command('kubectl -n ts-dev1 logs -l app.kubernetes.io/component=master'); + const subprocess = await execa.command('kubectl -n ts-dev1 logs --tail 1000 -l app.kubernetes.io/component=master'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } export async function showTSExLogs() { - const subprocess = await execa.command('kubectl -n ts-dev1 logs -l app.kubernetes.io/component=execution_controller'); + const subprocess = await execa.command('kubectl -n ts-dev1 logs --tail 1000 -l app.kubernetes.io/component=execution_controller'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } export async function showTSWorkerLogs() { - const subprocess = await execa.command('kubectl -n ts-dev1 logs -l app.kubernetes.io/component=worker'); + const subprocess = await execa.command('kubectl -n ts-dev1 logs --tail 1000 -l app.kubernetes.io/component=worker'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } From fdbbf9f530b315b7746f93260d6f7837a39e9c70 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 15:23:10 -0700 Subject: [PATCH 136/142] Print out kubectl describe for worker --- e2e/test/teraslice-harness.js | 6 +++++- packages/scripts/src/helpers/scripts.ts | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 02ef7cb8edd..fa98b4fb1eb 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -5,7 +5,9 @@ const { pDelay, uniq, toString, cloneDeep, isEmpty, castArray } = require('@terascope/utils'); -const { deployK8sTeraslice, showState, showTSMasterLogs, showTSExLogs, showTSWorkerLogs } = require('@terascope/scripts'); +const { + deployK8sTeraslice, showState, showTSMasterLogs, showTSExLogs, showTSWorkerLogs, describeWorker +} = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); const path = require('path'); @@ -60,6 +62,7 @@ module.exports = class TerasliceHarness { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ before ex.waitForExStatus'); signale.debug(await showState(HOST_IP)); + signale.debug(await describeWorker()); } const result = await ex.waitForStatus(status, interval, 2 * 60 * 1000); if (endDelay) { @@ -71,6 +74,7 @@ module.exports = class TerasliceHarness { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ after ex.waitForExStatus'); signale.debug(await showState(HOST_IP)); + signale.debug(await describeWorker()); } return result; } catch (err) { diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index bd0f44d97cf..5a3006578ce 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -769,7 +769,7 @@ export async function deleteTerasliceNamespace() { // FIXME: delete before merging? - for testing export async function showState(hostIP: string) { - const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels'); + const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels -o wide'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); // console.log('\nshowState subprocess: \n', subprocess.stdout); @@ -815,6 +815,12 @@ export async function showTSWorkerLogs() { logger.debug(subprocess.stdout); } +export async function describeWorker() { + const subprocess = await execa.command('kubectl -n ts-dev1 describe -l app.kubernetes.io/component=worker'); + signale.debug(subprocess.stdout); + logger.debug(subprocess.stdout); +} + async function showAssets(hostIP: string) { try { const subprocess = await execa.command(`curl ${hostIP}:45678/v1/assets`); From ceafc800e0422e2e884da6a4c15f3365ceca1589 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 15:31:39 -0700 Subject: [PATCH 137/142] typo in command --- .github/workflows/testK8sOnly.yml | 44 +++++++++++++++++++++++++ packages/scripts/src/helpers/scripts.ts | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/testK8sOnly.yml diff --git a/.github/workflows/testK8sOnly.yml b/.github/workflows/testK8sOnly.yml new file mode 100644 index 00000000000..b0def6debf8 --- /dev/null +++ b/.github/workflows/testK8sOnly.yml @@ -0,0 +1,44 @@ +name: Teraslice CI Tests +# this will technically run again on merge to master, should limit it +on: + pull_request: + branches: [ master ] + +jobs: + e2e-k8s-tests: + runs-on: ubuntu-latest + strategy: + # opensearch is finiky, keep testing others if it fails + fail-fast: false + matrix: + node-version: [16.19.1, 18.16.0] + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Setup Node ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: 'yarn' + + # we login to docker to avoid docker pull limit rates + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Install and build packages + run: yarn setup + env: + YARN_SETUP_ARGS: "--prod=false --silent" + + - name: Install Kind and Kubectl + uses: helm/kind-action@v1.8.0 + with: + install_only: "true" + + - name: Test k8s elasticsearch7 + run: yarn --silent test:k8s --node-version ${{ matrix.node-version }} + working-directory: ./e2e diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 5a3006578ce..ca23e26ab07 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -816,7 +816,7 @@ export async function showTSWorkerLogs() { } export async function describeWorker() { - const subprocess = await execa.command('kubectl -n ts-dev1 describe -l app.kubernetes.io/component=worker'); + const subprocess = await execa.command('kubectl -n ts-dev1 describe pod -l app.kubernetes.io/component=worker'); signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } From 74103aaf16fed5ba7c91c55cdb906de8093c534d Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 1 Nov 2023 15:54:26 -0700 Subject: [PATCH 138/142] more logs --- e2e/test/teraslice-harness.js | 1 + packages/scripts/src/helpers/scripts.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index fa98b4fb1eb..40f020e5286 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -80,6 +80,7 @@ module.exports = class TerasliceHarness { } catch (err) { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ in catch of waitForExStatus'); + signale.debug(await describeWorker()); signale.debug(await showState(HOST_IP)); signale.debug(await showTSMasterLogs()); signale.debug(await showTSExLogs()); diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index ca23e26ab07..aecb2bf60f5 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -816,9 +816,10 @@ export async function showTSWorkerLogs() { } export async function describeWorker() { + signale.debug('@@@ descripbeWorker'); const subprocess = await execa.command('kubectl -n ts-dev1 describe pod -l app.kubernetes.io/component=worker'); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); + signale.debug(subprocess); + logger.debug(subprocess); } async function showAssets(hostIP: string) { From 07b44efbaa5cf68c34304837e42a9e05b570d771 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Nov 2023 10:29:34 -0700 Subject: [PATCH 139/142] Update resource requests and limits for e2e jobs --- e2e/test/fixtures/jobs/generate-to-es.json | 3 +++ e2e/test/fixtures/jobs/generator-asset.json | 3 +++ e2e/test/fixtures/jobs/generator.json | 3 +++ e2e/test/fixtures/jobs/id.json | 3 +++ e2e/test/fixtures/jobs/kafka-reader.json | 3 +++ e2e/test/fixtures/jobs/kafka-sender.json | 3 +++ e2e/test/fixtures/jobs/multisend.json | 3 +++ e2e/test/fixtures/jobs/reindex.json | 3 +++ 8 files changed, 24 insertions(+) diff --git a/e2e/test/fixtures/jobs/generate-to-es.json b/e2e/test/fixtures/jobs/generate-to-es.json index d8301baa243..6942f681560 100644 --- a/e2e/test/fixtures/jobs/generate-to-es.json +++ b/e2e/test/fixtures/jobs/generate-to-es.json @@ -5,6 +5,9 @@ "workers": 2, "analytics": false, "assets": ["elasticsearch", "standard"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "max_retries": 0, "operations": [ { diff --git a/e2e/test/fixtures/jobs/generator-asset.json b/e2e/test/fixtures/jobs/generator-asset.json index e0cbf88bdfc..831eee754da 100644 --- a/e2e/test/fixtures/jobs/generator-asset.json +++ b/e2e/test/fixtures/jobs/generator-asset.json @@ -4,6 +4,9 @@ "lifecycle": "persistent", "workers": 2, "assets": ["ex1", "standard"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "max_retries": 0, "analytics": false, "operations": [ diff --git a/e2e/test/fixtures/jobs/generator.json b/e2e/test/fixtures/jobs/generator.json index 28bcf98d2c1..f88e73801aa 100644 --- a/e2e/test/fixtures/jobs/generator.json +++ b/e2e/test/fixtures/jobs/generator.json @@ -5,6 +5,9 @@ "workers": 3, "analytics": false, "assets": ["standard"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "max_retries": 0, "operations": [ { diff --git a/e2e/test/fixtures/jobs/id.json b/e2e/test/fixtures/jobs/id.json index 844482d35e3..218b3619d4b 100644 --- a/e2e/test/fixtures/jobs/id.json +++ b/e2e/test/fixtures/jobs/id.json @@ -5,6 +5,9 @@ "slicers": 2, "workers": 4, "assets": ["elasticsearch"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "operations": [ { "_op": "id_reader", diff --git a/e2e/test/fixtures/jobs/kafka-reader.json b/e2e/test/fixtures/jobs/kafka-reader.json index f1eeb469ceb..23fd02344c2 100644 --- a/e2e/test/fixtures/jobs/kafka-reader.json +++ b/e2e/test/fixtures/jobs/kafka-reader.json @@ -4,6 +4,9 @@ "workers": 1, "analytics": true, "assets": ["kafka", "elasticsearch"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "max_retries": 0, "operations": [ { diff --git a/e2e/test/fixtures/jobs/kafka-sender.json b/e2e/test/fixtures/jobs/kafka-sender.json index f799564a770..d2d4085bf6d 100644 --- a/e2e/test/fixtures/jobs/kafka-sender.json +++ b/e2e/test/fixtures/jobs/kafka-sender.json @@ -4,6 +4,9 @@ "workers": 1, "analytics": true, "assets": ["kafka", "elasticsearch"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "max_retries": 0, "operations": [ { diff --git a/e2e/test/fixtures/jobs/multisend.json b/e2e/test/fixtures/jobs/multisend.json index 52be220596b..482e9fd8c92 100644 --- a/e2e/test/fixtures/jobs/multisend.json +++ b/e2e/test/fixtures/jobs/multisend.json @@ -4,6 +4,9 @@ "lifecycle": "once", "analytics": false, "assets": ["elasticsearch"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "operations": [ { "_op": "elasticsearch_reader", diff --git a/e2e/test/fixtures/jobs/reindex.json b/e2e/test/fixtures/jobs/reindex.json index 9fda19219c7..a93030347ea 100644 --- a/e2e/test/fixtures/jobs/reindex.json +++ b/e2e/test/fixtures/jobs/reindex.json @@ -4,6 +4,9 @@ "workers": 1, "analytics": true, "assets": ["elasticsearch"], + "resources_requests_cpu": 0.1, + "resources_limits_cpu": 0.5, + "cpu_execution_controller": 0.2, "operations": [ { "_op": "elasticsearch_reader", From 2da4fa4e3229541ab60ed560c6795f67555a6a03 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Nov 2023 10:30:13 -0700 Subject: [PATCH 140/142] update resource requests for data generator jobs --- .github/workflows/testK8sOnly.yml | 2 +- e2e/test/teraslice-harness.js | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testK8sOnly.yml b/.github/workflows/testK8sOnly.yml index b0def6debf8..f5c0ceb5aa4 100644 --- a/.github/workflows/testK8sOnly.yml +++ b/.github/workflows/testK8sOnly.yml @@ -1,4 +1,4 @@ -name: Teraslice CI Tests +name: Teraslice CI Tests K8sOnly # this will technically run again on merge to master, should limit it on: pull_request: diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index 40f020e5286..db7a8fa6c5e 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -6,7 +6,8 @@ const { cloneDeep, isEmpty, castArray } = require('@terascope/utils'); const { - deployK8sTeraslice, showState, showTSMasterLogs, showTSExLogs, showTSWorkerLogs, describeWorker + deployK8sTeraslice, showState, showTSMasterLogs, + showTSExLogs, showTSWorkerLogs, describeWorker, getPodYamls, describeNode } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); @@ -63,6 +64,8 @@ module.exports = class TerasliceHarness { signale.debug('@@@ before ex.waitForExStatus'); signale.debug(await showState(HOST_IP)); signale.debug(await describeWorker()); + signale.debug(await describeNode()); + signale.debug(await getPodYamls()); } const result = await ex.waitForStatus(status, interval, 2 * 60 * 1000); if (endDelay) { @@ -75,12 +78,16 @@ module.exports = class TerasliceHarness { signale.debug('@@@ after ex.waitForExStatus'); signale.debug(await showState(HOST_IP)); signale.debug(await describeWorker()); + signale.debug(await describeNode()); + signale.debug(await getPodYamls()); } return result; } catch (err) { if (process.env.TEST_PLATFORM === 'kubernetes') { signale.debug('@@@ in catch of waitForExStatus'); signale.debug(await describeWorker()); + signale.debug(await describeNode()); + signale.debug(await getPodYamls()); signale.debug(await showState(HOST_IP)); signale.debug(await showTSMasterLogs()); signale.debug(await showTSExLogs()); @@ -476,6 +483,9 @@ module.exports = class TerasliceHarness { lifecycle: 'once', workers: 1, assets: ['elasticsearch', 'standard'], + resources_requests_cpu: 0.1, + resources_limits_cpu: 0.5, + cpu_execution_controller: 0.2, operations: [ { _op: 'data_generator', From afb04f9fd1d91e825641cf14e94f5cce92057b1d Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Nov 2023 10:31:02 -0700 Subject: [PATCH 141/142] Add logging functions --- packages/scripts/src/helpers/scripts.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index aecb2bf60f5..474479775d1 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -816,12 +816,26 @@ export async function showTSWorkerLogs() { } export async function describeWorker() { - signale.debug('@@@ descripbeWorker'); + signale.debug('@@@ describeWorker'); const subprocess = await execa.command('kubectl -n ts-dev1 describe pod -l app.kubernetes.io/component=worker'); signale.debug(subprocess); logger.debug(subprocess); } +export async function describeNode() { + signale.debug('@@@ describeNode'); + const subprocess = await execa.command('kubectl -n ts-dev1 describe node'); + signale.debug(subprocess); + logger.debug(subprocess); +} + +export async function getPodYamls() { + signale.debug('@@@ getPodYamls'); + const subprocess = await execa.command('kubectl -n ts-dev1 get pods -l app.kubernetes.io/component=worker -o yaml'); + signale.debug(subprocess); + logger.debug(subprocess); +} + async function showAssets(hostIP: string) { try { const subprocess = await execa.command(`curl ${hostIP}:45678/v1/assets`); From 8ce6270a36accd25858000269296a172627d650a Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Nov 2023 14:02:58 -0700 Subject: [PATCH 142/142] Remove all logs and testing env changes. --- .github/workflows/testK8sOnly.yml | 44 --------- e2e/package.json | 4 +- e2e/test/global.setup.js | 1 - e2e/test/global.teardown.js | 2 +- e2e/test/teraslice-harness.js | 54 +---------- packages/scripts/src/helpers/scripts.ts | 95 ------------------- .../scripts/src/helpers/test-runner/index.ts | 1 - .../src/helpers/test-runner/services.ts | 1 - 8 files changed, 7 insertions(+), 195 deletions(-) delete mode 100644 .github/workflows/testK8sOnly.yml diff --git a/.github/workflows/testK8sOnly.yml b/.github/workflows/testK8sOnly.yml deleted file mode 100644 index f5c0ceb5aa4..00000000000 --- a/.github/workflows/testK8sOnly.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Teraslice CI Tests K8sOnly -# this will technically run again on merge to master, should limit it -on: - pull_request: - branches: [ master ] - -jobs: - e2e-k8s-tests: - runs-on: ubuntu-latest - strategy: - # opensearch is finiky, keep testing others if it fails - fail-fast: false - matrix: - node-version: [16.19.1, 18.16.0] - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Setup Node ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'yarn' - - # we login to docker to avoid docker pull limit rates - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Install and build packages - run: yarn setup - env: - YARN_SETUP_ARGS: "--prod=false --silent" - - - name: Install Kind and Kubectl - uses: helm/kind-action@v1.8.0 - with: - install_only: "true" - - - name: Test k8s elasticsearch7 - run: yarn --silent test:k8s --node-version ${{ matrix.node-version }} - working-directory: ./e2e diff --git a/e2e/package.json b/e2e/package.json index f832c0e7825..91565c7e313 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -28,8 +28,8 @@ "test:debug": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --debug --", "test:elasticsearch6": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug --", - "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' DEBUG='*' ts-scripts test --suite e2e --debug -- --silent=false", + "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' KAFKA_VERSION='3.1' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --" diff --git a/e2e/test/global.setup.js b/e2e/test/global.setup.js index 0d052e3fb5d..7c0512ff716 100644 --- a/e2e/test/global.setup.js +++ b/e2e/test/global.setup.js @@ -17,7 +17,6 @@ const { } = require('./config'); module.exports = async () => { - signale.debug('@@@@ HOST_IP: ', HOST_IP); const teraslice = new TerasliceHarness(); await teraslice.init(); diff --git a/e2e/test/global.teardown.js b/e2e/test/global.teardown.js index 38dfa656e31..b4a7e2563dd 100644 --- a/e2e/test/global.teardown.js +++ b/e2e/test/global.teardown.js @@ -26,7 +26,7 @@ async function globalTeardown(testClient) { const errors = []; try { - if (TEST_PLATFORM === 'kubernetes' && !process.env.KEEP_OPEN) { + if (TEST_PLATFORM === 'kubernetes') { await deleteTerasliceNamespace(); await cleanupIndex(client, 'ts-dev1_*'); } else { diff --git a/e2e/test/teraslice-harness.js b/e2e/test/teraslice-harness.js index db7a8fa6c5e..95fe62a470c 100644 --- a/e2e/test/teraslice-harness.js +++ b/e2e/test/teraslice-harness.js @@ -6,8 +6,7 @@ const { cloneDeep, isEmpty, castArray } = require('@terascope/utils'); const { - deployK8sTeraslice, showState, showTSMasterLogs, - showTSExLogs, showTSWorkerLogs, describeWorker, getPodYamls, describeNode + deployK8sTeraslice, showState } = require('@terascope/scripts'); const { createClient, ElasticsearchTestHelpers } = require('elasticsearch-store'); const { TerasliceClient } = require('teraslice-client-js'); @@ -60,13 +59,6 @@ module.exports = class TerasliceHarness { const start = Date.now(); try { - if (process.env.TEST_PLATFORM === 'kubernetes') { - signale.debug('@@@ before ex.waitForExStatus'); - signale.debug(await showState(HOST_IP)); - signale.debug(await describeWorker()); - signale.debug(await describeNode()); - signale.debug(await getPodYamls()); - } const result = await ex.waitForStatus(status, interval, 2 * 60 * 1000); if (endDelay) { // since most of the time we are chaining this with other actions @@ -74,25 +66,8 @@ module.exports = class TerasliceHarness { // it a little bit of time await pDelay(endDelay); } - if (process.env.TEST_PLATFORM === 'kubernetes') { - signale.debug('@@@ after ex.waitForExStatus'); - signale.debug(await showState(HOST_IP)); - signale.debug(await describeWorker()); - signale.debug(await describeNode()); - signale.debug(await getPodYamls()); - } return result; } catch (err) { - if (process.env.TEST_PLATFORM === 'kubernetes') { - signale.debug('@@@ in catch of waitForExStatus'); - signale.debug(await describeWorker()); - signale.debug(await describeNode()); - signale.debug(await getPodYamls()); - signale.debug(await showState(HOST_IP)); - signale.debug(await showTSMasterLogs()); - signale.debug(await showTSExLogs()); - signale.debug(await showTSWorkerLogs()); - } err.message = `Execution: ${ex.id()}: ${err.message}`; this.warn(err.stack, { @@ -116,12 +91,11 @@ module.exports = class TerasliceHarness { if (TEST_PLATFORM === 'kubernetes') { try { - // console.log('@@@@ before state reset'); // await showState(HOST_IP); await cleanupIndex(this.client, `${SPEC_INDEX_PREFIX}*`); await this.clearNonBaseAssets(); } catch (err) { - signale.error('Failure to clean indices', err); + signale.error('Failure to clean indices and assets', err); throw err; } try { @@ -129,7 +103,6 @@ module.exports = class TerasliceHarness { await pDelay(2000); await this.waitForTeraslice(); await pDelay(2000); - // console.log('@@@@ after state reset'); await showState(HOST_IP); } catch (err) { signale.error('Failure to reset teraslice state', err); @@ -459,13 +432,7 @@ module.exports = class TerasliceHarness { } async postJob(jobSpec) { - const result = this.teraslice.executions.submit(jobSpec); - signale.debug('@@@ postJob result: ', result); - if (process.env.TEST_PLATFORM === 'kubernetes') { - signale.debug('@@@ after postJob result'); - signale.debug(await showState(HOST_IP)); - } - return result; + return this.teraslice.executions.submit(jobSpec); } async generate(count, hex) { @@ -514,15 +481,7 @@ module.exports = class TerasliceHarness { await Promise.all(executions.map((ex) => this.waitForExStatus(ex, 'completed'))); } else { const ex = await this.postJob(jobSpec); - if (process.env.TEST_PLATFORM === 'kubernetes') { - signale.debug('@@@ after postJob'); - signale.debug(await showState(HOST_IP)); - } await this.waitForExStatus(ex, 'completed'); - if (process.env.TEST_PLATFORM === 'kubernetes') { - signale.debug('@@@ after waitForExStatus'); - signale.debug(await showState(HOST_IP)); - } } signale.info(`Generated ${indexName} example data`, getElapsed(genStartTime)); @@ -556,17 +515,12 @@ module.exports = class TerasliceHarness { async clearNonBaseAssets() { const assetList = await this.teraslice.assets.list(); - // console.log('@@@@@ assetList: ', assetList); - const baseAssets = ['standard', 'elasticsearch', 'kafka']; const assetsToDelete = assetList.filter((assetObj) => !baseAssets.includes(assetObj.name)); - // console.log('@@@@@ assetsToDelete: ', assetsToDelete); for (const asset of assetsToDelete) { - // console.log('@@@@@ asset: ', asset); const response = await this.teraslice.assets.remove(asset.id); - // console.log('@@@@@ response: ', response); - signale.success(`Deleted asset with id ${response}`); + signale.debug(`Deleted asset with id ${response._id}`); } } }; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 474479775d1..0a065973470 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -542,20 +542,17 @@ export async function createKindCluster(): Promise { const configPath = path.join(e2eK8sDir, 'kindConfig.yaml'); const subprocess = await execa.command(`kind create cluster --config ${configPath}`); - signale.debug(subprocess.stderr); logger.debug(subprocess.stderr); } export async function destroyKindCluster(): Promise { const subprocess = await execa.command('kind delete cluster --name k8se2e'); - signale.debug(subprocess.stderr); logger.debug(subprocess.stderr); } export async function isKindInstalled(): Promise { try { const subprocess = await execa.command('command -v kind'); - // console.log('kind subprocess: ', subprocess); return !!subprocess.stdout; } catch (err) { return false; @@ -565,7 +562,6 @@ export async function isKindInstalled(): Promise { export async function isKubectlInstalled(): Promise { try { const subprocess = await execa.command('command -v kubectl'); - // console.log('kubectl subprocess: ', subprocess); return !!subprocess.stdout; } catch (err) { return false; @@ -575,8 +571,6 @@ export async function isKubectlInstalled(): Promise { // TODO: check that image is loaded before we continue export async function loadTerasliceImage(terasliceImage: string): Promise { const subprocess = await execa.command(`kind load docker-image ${terasliceImage} --name k8se2e`); - // console.log('load teraslice image subprocess: ', subprocess); - signale.debug(subprocess.stderr); logger.debug(subprocess.stderr); } @@ -590,9 +584,7 @@ export async function kindStopService(serviceName: string): Promise { // Any new service's yaml file must be named 'Deployment.yaml' const yamlFile = `${serviceName}Deployment.yaml`; const subprocess = await execa.command(`kubectl delete -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - // console.log('stopElasticsearch subprocess: ', subprocess); } catch (err) { // Do nothing. This should fail because no services should be up yet. } @@ -605,11 +597,8 @@ export async function kindLoadServiceImage( ): Promise { try { const subprocess = await execa.command(`kind load docker-image ${serviceImage}:${version} --name k8se2e`); - // console.log('load service image subprocess: ', subprocess); - signale.debug(subprocess.stderr); logger.debug(subprocess.stderr); } catch (err) { - signale.debug(`The service ${serviceName} could not be loaded. It may not be present locally`); logger.debug(`The service ${serviceName} could not be loaded. It may not be present locally`); } } @@ -625,10 +614,8 @@ export async function kindStartService(serviceName: string): Promise { try { const subprocess = await execa.command(`kubectl create -n services-dev1 -f ${path.join(e2eK8sDir, yamlFile)}`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } catch (err) { - signale.error(`The service ${serviceName} could not be started: `, err); logger.error(`The service ${serviceName} could not be started: `, err); } @@ -649,7 +636,6 @@ function waitForKafkaRunning(timeoutMs = 120000): Promise { try { const kubectlResponse = await execa.command('kubectl -n services-dev1 get pods -l app.kubernetes.io/name=cpkafka -o=jsonpath="{.items[?(@.status.containerStatuses)].status.containerStatuses[0].ready}"'); const kafkaReady = kubectlResponse.stdout; - // console.log('kafka response: ', kafkaReady); if (kafkaReady === '"true"') { kafkaRunning = true; } @@ -674,9 +660,7 @@ export async function createNamespace(namespaceYaml: string) { throw new Error('Missing k8s e2e test directory'); } const subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, namespaceYaml)}`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - // console.log('namespace subprocess: ', subprocess); } export async function k8sSetup(): Promise { @@ -686,13 +670,10 @@ export async function k8sSetup(): Promise { } let subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'role.yaml')}`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); subprocess = await execa.command(`kubectl create -f ${path.join(e2eK8sDir, 'roleBinding.yaml')}`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); subprocess = await execa.command(`kubectl apply -f ${path.join(e2eK8sDir, 'priorityClass.yaml')}`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } @@ -708,24 +689,16 @@ export async function deployK8sTeraslice() { try { /// Creates configmap for terasclice-master let subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-master --from-file=${path.join(e2eK8sDir, 'masterConfig', 'teraslice.yaml')}`); - // console.log('masterConfig subprocess: ', subprocess); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); /// Creates configmap for teraslice-worker subprocess = await execa.command(`kubectl create -n ts-dev1 configmap teraslice-worker --from-file=${path.join(e2eK8sDir, 'workerConfig', 'teraslice.yaml')}`); - // console.log('workerConfig subprocess: ', subprocess); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); /// Creates deployment for teraslice subprocess = await execa.command(`kubectl create -n ts-dev1 -f ${path.join(e2eK8sDir, 'masterDeployment.yaml')}`); - // console.log('masterDeploy subprocess: ', subprocess); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } catch (err) { - signale.error('Error deploying Teraslice'); - signale.error(err); logger.error('Error deploying Teraslice'); logger.error(err); process.exit(1); @@ -741,28 +714,21 @@ export async function setAliasAndBaseAssets(hostIP: string) { async function setAlias(hostIP: string) { let subprocess = await execa.command('earl aliases remove k8se2e 2> /dev/null || true', { shell: true }); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); subprocess = await execa.command(`earl aliases add k8se2e http://${hostIP}:45678`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - // console.log('setAlias subprocess: ', subprocess1, subprocess2); } async function deployAssets(assetName: string) { const subprocess = await execa.command(`earl assets deploy k8se2e --blocking terascope/${assetName}-assets`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - // console.log('deployKafkaAssets subprocess: ', subprocess); } export async function deleteTerasliceNamespace() { try { const subprocess = await execa.command('kubectl delete namespace ts-dev1'); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); } catch (err) { - signale.debug('Teraslice namespace cannot be deleted because it does not exist'); logger.debug('Teraslice namespace cannot be deleted because it does not exist'); } } @@ -770,82 +736,21 @@ export async function deleteTerasliceNamespace() { // FIXME: delete before merging? - for testing export async function showState(hostIP: string) { const subprocess = await execa.command('kubectl get deployments,po,svc --all-namespaces --show-labels -o wide'); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - // console.log('\nshowState subprocess: \n', subprocess.stdout); await showESIndices(hostIP); await showAssets(hostIP); } async function showESIndices(hostIP: string) { const subprocess = await execa.command(`curl ${hostIP}:49200/_cat/indices`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); -} - -export async function showESData(hostIP: string) { - let subprocess = await execa.command(`curl ${hostIP}:49200/ts-dev1__ex/_search`); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); - subprocess = await execa.command(`curl ${hostIP}:49200/ts-dev1__analytics-2023.11/_search`); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); - subprocess = await execa.command(`curl ${hostIP}:49200/ts-dev1__state-2023.11/_search`); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); - // console.log('\nshowESIndices subprocess: \n', subprocess.stdout); -} - -export async function showTSMasterLogs() { - const subprocess = await execa.command('kubectl -n ts-dev1 logs --tail 1000 -l app.kubernetes.io/component=master'); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); -} - -export async function showTSExLogs() { - const subprocess = await execa.command('kubectl -n ts-dev1 logs --tail 1000 -l app.kubernetes.io/component=execution_controller'); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); -} - -export async function showTSWorkerLogs() { - const subprocess = await execa.command('kubectl -n ts-dev1 logs --tail 1000 -l app.kubernetes.io/component=worker'); - signale.debug(subprocess.stdout); - logger.debug(subprocess.stdout); -} - -export async function describeWorker() { - signale.debug('@@@ describeWorker'); - const subprocess = await execa.command('kubectl -n ts-dev1 describe pod -l app.kubernetes.io/component=worker'); - signale.debug(subprocess); - logger.debug(subprocess); -} - -export async function describeNode() { - signale.debug('@@@ describeNode'); - const subprocess = await execa.command('kubectl -n ts-dev1 describe node'); - signale.debug(subprocess); - logger.debug(subprocess); -} - -export async function getPodYamls() { - signale.debug('@@@ getPodYamls'); - const subprocess = await execa.command('kubectl -n ts-dev1 get pods -l app.kubernetes.io/component=worker -o yaml'); - signale.debug(subprocess); - logger.debug(subprocess); } async function showAssets(hostIP: string) { try { const subprocess = await execa.command(`curl ${hostIP}:45678/v1/assets`); - signale.debug(subprocess.stdout); logger.debug(subprocess.stdout); - - // console.log('\nshowAssets subprocess: \n', subprocess.stdout); } catch (err) { - signale.debug(err); logger.debug(err); - // console.log('\nshowAssets subprocess: \n', err); } } diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 99051902ba0..b5c32831b00 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -199,7 +199,6 @@ async function runTestSuite( async function runE2ETest( options: TestOptions, tracker: TestTracker ): Promise { - // console.log('@@@@@@@@ options: ', options); tracker.expected++; const suite = 'e2e'; diff --git a/packages/scripts/src/helpers/test-runner/services.ts b/packages/scripts/src/helpers/test-runner/services.ts index 9f4458fe2ca..6c2b13490c2 100644 --- a/packages/scripts/src/helpers/test-runner/services.ts +++ b/packages/scripts/src/helpers/test-runner/services.ts @@ -752,7 +752,6 @@ async function startService(options: TestOptions, service: Service): Promise<() await stopService(service); - // console.log(`@@@@@@@ loading ${service} via docker`); const fn = await dockerRun( services[service], version,