Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[teraslice] fix serial job starts in kubernetesV2 #3867

Merged
merged 3 commits into from
Dec 10, 2024
Merged

[teraslice] fix serial job starts in kubernetesV2 #3867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b18e99
remove waitForSelectedPods() from allocateWorkers()
busma13 Dec 10, 2024
abc9212
remove unnecessary code used to get podIP from allocateSlicer()
busma13 Dec 10, 2024
6d48aec
release: (patch) [email protected]
busma13 Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"devDependencies": {
"@terascope/types": "~1.3.1",
"bunyan": "~1.8.15",
"elasticsearch-store": "~1.4.2",
"elasticsearch-store": "~1.4.3",
"fs-extra": "~11.2.0",
"ms": "~2.1.3",
"nanoid": "~5.0.9",
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "teraslice-workspace",
"displayName": "Teraslice",
"version": "2.9.1",
"version": "2.9.2",
"private": true,
"homepage": "https://github.com/terascope/teraslice",
"bugs": {
Expand Down
8 changes: 4 additions & 4 deletions packages/data-mate/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/data-mate",
"displayName": "Data-Mate",
"version": "1.4.2",
"version": "1.4.3",
"description": "Library of data validations/transformations",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
"repository": {
Expand Down Expand Up @@ -30,9 +30,9 @@
"test:watch": "ts-scripts test --watch . --"
},
"dependencies": {
"@terascope/data-types": "~1.4.1",
"@terascope/data-types": "~1.4.2",
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"@types/validator": "~13.12.2",
"awesome-phonenumber": "~7.2.0",
"date-fns": "~4.1.0",
Expand All @@ -46,7 +46,7 @@
"uuid": "~11.0.3",
"valid-url": "~1.0.9",
"validator": "~13.12.0",
"xlucene-parser": "~1.4.2"
"xlucene-parser": "~1.4.3"
},
"devDependencies": {
"@types/ip6addr": "~0.2.6",
Expand Down
4 changes: 2 additions & 2 deletions packages/data-types/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/data-types",
"displayName": "Data Types",
"version": "1.4.1",
"version": "1.4.2",
"description": "A library for defining the data structures and mapping",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
"bugs": {
Expand All @@ -28,7 +28,7 @@
},
"dependencies": {
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"graphql": "~16.9.0",
"lodash": "~4.17.21",
"yargs": "~17.7.2"
Expand Down
2 changes: 1 addition & 1 deletion packages/docker-compose-js/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/docker-compose-js",
"displayName": "Docker Compose Js",
"version": "1.4.1",
"version": "1.4.2",
"description": "Node.js driver for controlling docker-compose testing environments.",
"keywords": [
"docker",
Expand Down
6 changes: 3 additions & 3 deletions packages/elasticsearch-api/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/elasticsearch-api",
"displayName": "Elasticsearch API",
"version": "4.4.1",
"version": "4.4.2",
"description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
"bugs": {
Expand All @@ -25,15 +25,15 @@
},
"dependencies": {
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"bluebird": "~3.7.2",
"setimmediate": "~1.0.5"
},
"devDependencies": {
"@opensearch-project/opensearch": "~1.2.0",
"@types/elasticsearch": "~5.0.43",
"elasticsearch": "~15.4.1",
"elasticsearch-store": "~1.4.2",
"elasticsearch-store": "~1.4.3",
"elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0",
"elasticsearch7": "npm:@elastic/elasticsearch@~7.17.0",
"elasticsearch8": "npm:@elastic/elasticsearch@~8.15.0"
Expand Down
10 changes: 5 additions & 5 deletions packages/elasticsearch-store/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "elasticsearch-store",
"displayName": "Elasticsearch Store",
"version": "1.4.2",
"version": "1.4.3",
"description": "An API for managing an elasticsearch index, with versioning and migration support.",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
"bugs": {
Expand Down Expand Up @@ -30,10 +30,10 @@
"test:watch": "ts-scripts test --watch . --"
},
"dependencies": {
"@terascope/data-mate": "~1.4.2",
"@terascope/data-types": "~1.4.1",
"@terascope/data-mate": "~1.4.3",
"@terascope/data-types": "~1.4.2",
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"ajv": "~8.17.1",
"ajv-formats": "~3.0.1",
"elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0",
Expand All @@ -43,7 +43,7 @@
"opensearch2": "npm:@opensearch-project/opensearch@~2.12.0",
"setimmediate": "~1.0.5",
"uuid": "~11.0.3",
"xlucene-translator": "~1.4.2"
"xlucene-translator": "~1.4.3"
},
"devDependencies": {
"@types/uuid": "~10.0.0"
Expand Down
2 changes: 1 addition & 1 deletion packages/eslint-config/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/eslint-config",
"displayName": "Terascope ESLint Config",
"version": "1.1.2",
"version": "1.1.3",
"description": "A shared eslint config based on eslint-config-airbnb",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/eslint-config#readme",
"bugs": {
Expand Down
4 changes: 2 additions & 2 deletions packages/job-components/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/job-components",
"displayName": "Job Components",
"version": "1.6.2",
"version": "1.6.3",
"description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
"bugs": {
Expand Down Expand Up @@ -33,7 +33,7 @@
},
"dependencies": {
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"convict": "~6.2.4",
"convict-format-with-moment": "~6.2.0",
"convict-format-with-validator": "~6.2.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/scripts/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/scripts",
"displayName": "Scripts",
"version": "1.5.3",
"version": "1.5.4",
"description": "A collection of terascope monorepo scripts",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
"bugs": {
Expand Down Expand Up @@ -33,7 +33,7 @@
},
"dependencies": {
"@kubernetes/client-node": "~0.22.3",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"codecov": "~3.8.3",
"execa": "~9.5.2",
"fs-extra": "~11.2.0",
Expand Down
6 changes: 3 additions & 3 deletions packages/terafoundation/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "terafoundation",
"displayName": "Terafoundation",
"version": "1.6.2",
"version": "1.6.3",
"description": "A Clustering and Foundation tool for Terascope Tools",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
"bugs": {
Expand Down Expand Up @@ -30,14 +30,14 @@
"dependencies": {
"@terascope/file-asset-apis": "~1.0.3",
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"bluebird": "~3.7.2",
"bunyan": "~1.8.15",
"convict": "~6.2.4",
"convict-format-with-moment": "~6.2.0",
"convict-format-with-validator": "~6.2.0",
"elasticsearch": "~15.4.1",
"elasticsearch-store": "~1.4.2",
"elasticsearch-store": "~1.4.3",
"express": "~4.21.2",
"js-yaml": "~4.1.0",
"nanoid": "~5.0.9",
Expand Down
4 changes: 2 additions & 2 deletions packages/teraslice-cli/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
"devDependencies": {
"@terascope/fetch-github-release": "~1.0.0",
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"@types/decompress": "~4.2.7",
"@types/diff": "~6.0.0",
"@types/ejs": "~3.1.5",
Expand All @@ -67,7 +67,7 @@
"pretty-bytes": "~6.1.1",
"prompts": "~2.4.2",
"signale": "~1.4.0",
"teraslice-client-js": "~1.4.1",
"teraslice-client-js": "~1.4.2",
"tmp": "~0.2.0",
"tty-table": "~4.2.3",
"yargs": "~17.7.2"
Expand Down
4 changes: 2 additions & 2 deletions packages/teraslice-client-js/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "teraslice-client-js",
"displayName": "Teraslice Client (JavaScript)",
"version": "1.4.1",
"version": "1.4.2",
"description": "A Node.js client for teraslice jobs, assets, and cluster references.",
"keywords": [
"elasticsearch",
Expand Down Expand Up @@ -33,7 +33,7 @@
},
"dependencies": {
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"auto-bind": "~5.0.1",
"got": "~13.0.0"
},
Expand Down
4 changes: 2 additions & 2 deletions packages/teraslice-messaging/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/teraslice-messaging",
"displayName": "Teraslice Messaging",
"version": "1.7.1",
"version": "1.7.2",
"description": "An internal teraslice messaging library using socket.io",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme",
"bugs": {
Expand Down Expand Up @@ -36,7 +36,7 @@
},
"dependencies": {
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"ms": "~2.1.3",
"nanoid": "~5.0.9",
"p-event": "~6.0.1",
Expand Down
6 changes: 3 additions & 3 deletions packages/teraslice-state-storage/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@terascope/teraslice-state-storage",
"displayName": "Teraslice State Storage",
"version": "1.4.1",
"version": "1.4.2",
"description": "State storage operation api for teraslice",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme",
"bugs": {
Expand All @@ -24,8 +24,8 @@
"test:watch": "ts-scripts test --watch . --"
},
"dependencies": {
"@terascope/elasticsearch-api": "~4.4.1",
"@terascope/utils": "~1.4.1"
"@terascope/elasticsearch-api": "~4.4.2",
"@terascope/utils": "~1.4.2"
},
"engines": {
"node": ">=18.18.0",
Expand Down
4 changes: 2 additions & 2 deletions packages/teraslice-test-harness/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@
"fs-extra": "~11.2.0"
},
"devDependencies": {
"@terascope/job-components": "~1.6.2"
"@terascope/job-components": "~1.6.3"
},
"peerDependencies": {
"@terascope/job-components": ">=1.6.2"
"@terascope/job-components": ">=1.6.3"
},
"engines": {
"node": ">=18.18.0",
Expand Down
12 changes: 6 additions & 6 deletions packages/teraslice/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "teraslice",
"displayName": "Teraslice",
"version": "2.9.1",
"version": "2.9.2",
"description": "Distributed computing platform for processing JSON data",
"homepage": "https://github.com/terascope/teraslice#readme",
"bugs": {
Expand Down Expand Up @@ -39,11 +39,11 @@
},
"dependencies": {
"@kubernetes/client-node": "~0.22.3",
"@terascope/elasticsearch-api": "~4.4.1",
"@terascope/job-components": "~1.6.2",
"@terascope/teraslice-messaging": "~1.7.1",
"@terascope/elasticsearch-api": "~4.4.2",
"@terascope/job-components": "~1.6.3",
"@terascope/teraslice-messaging": "~1.7.2",
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"async-mutex": "~0.5.0",
"barbe": "~3.0.16",
"body-parser": "~1.20.2",
Expand All @@ -63,7 +63,7 @@
"semver": "~7.6.3",
"socket.io": "~1.7.4",
"socket.io-client": "~1.7.4",
"terafoundation": "~1.6.2",
"terafoundation": "~1.6.3",
"uuid": "~11.0.3"
},
"devDependencies": {
Expand Down
34 changes: 0 additions & 34 deletions packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,32 +138,6 @@ export class KubernetesClusterBackendV2 {

this.logger.debug(jobResult, 'k8s slicer job submitted');

if (!jobResult.spec.selector?.matchLabels) {
throw new Error('Required field matchLabels missing from jobResult.spec.selector');
}

let controllerLabel: string;
if (jobResult.spec.selector.matchLabels['controller-uid'] !== undefined) {
/// If running on kubernetes < v1.27.0
controllerLabel = 'controller-uid';
} else {
/// If running on kubernetes v1.27.0 or later
controllerLabel = 'batch.kubernetes.io/controller-uid';
}

const controllerUid = jobResult.spec.selector.matchLabels[controllerLabel];

const pod = await this.k8s.waitForSelectedPod(
`${controllerLabel}=${controllerUid}`,
'pod-status',
undefined,
this.context.sysconfig.teraslice.slicer_timeout
);

if (!pod.status?.podIP) {
const error = new Error('pod.status.podIP must be defined');
return Promise.reject(error);
}
const exServiceName = serviceResult.metadata.name;
const exServiceHostName = `${exServiceName}.${this.k8s.defaultNamespace}`;
this.logger.debug(`Slicer is using host name: ${exServiceHostName}`);
Expand All @@ -189,14 +163,6 @@ export class KubernetesClusterBackendV2 {
() => this.k8s.nonEmptyJobList(selector), getRetryConfig()
);

/// Wait for ex readiness probe to return 'Ready'
await this.k8s.waitForSelectedPod(
selector,
'readiness-probe',
undefined,
this.context.sysconfig.teraslice.slicer_timeout
);

if (!jobs.items[0].metadata.uid) {
throw new Error('Required field uid missing from kubernetes job metadata');
}
Expand Down
6 changes: 3 additions & 3 deletions packages/ts-transforms/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "ts-transforms",
"displayName": "TS Transforms",
"version": "1.4.2",
"version": "1.4.3",
"description": "An ETL framework built upon xlucene-evaluator",
"homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme",
"bugs": {
Expand Down Expand Up @@ -36,9 +36,9 @@
"test:watch": "ts-scripts test --watch . --"
},
"dependencies": {
"@terascope/data-mate": "~1.4.2",
"@terascope/data-mate": "~1.4.3",
"@terascope/types": "~1.3.1",
"@terascope/utils": "~1.4.1",
"@terascope/utils": "~1.4.2",
"awesome-phonenumber": "~7.2.0",
"graphlib": "~2.1.8",
"jexl": "~2.3.0",
Expand Down
Loading
Loading