From fc687b790b77ec4472d7a03004e273f64d10a4e3 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 21 Jan 2021 14:13:19 +0100 Subject: [PATCH] Handle another node already having deleted the temporary index (#88332) (#88949) * Handle another node already having deleted the temporary index * Make run_multiple_kibana_nodes.sh script more generic * Add note about dependency on jq Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../saved_objects/migrationsv2/model.test.ts | 12 ++- .../saved_objects/migrationsv2/model.ts | 20 +++-- test/scripts/run_multiple_kibana_nodes.sh | 86 +++++++++++++++++++ 3 files changed, 111 insertions(+), 7 deletions(-) create mode 100755 test/scripts/run_multiple_kibana_nodes.sh diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index 43e32fd611074..d5ab85c54a728 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -911,7 +911,7 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); - test('MARK_VERSION_INDEX_READY -> MARK_VERSION_INDEX_CONFLICT if someone else removed the current alias from the source index', () => { + test('MARK_VERSION_INDEX_READY -> MARK_VERSION_INDEX_CONFLICT if another removed the current alias from the source index', () => { const res: ResponseType<'MARK_VERSION_INDEX_READY'> = Either.left({ type: 'alias_not_found_exception', }); @@ -920,6 +920,16 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + test('MARK_VERSION_INDEX_READY -> MARK_VERSION_INDEX_CONFLICT if another node removed the temporary index', () => { + const res: ResponseType<'MARK_VERSION_INDEX_READY'> = Either.left({ + type: 'index_not_found_exception', + index: '.kibana_7.11.0_reindex_temp', + }); + const newState = model(markVersionIndexReadyState, res); + expect(newState.controlState).toEqual('MARK_VERSION_INDEX_READY_CONFLICT'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); }); describe('MARK_VERSION_INDEX_READY_CONFLICT', () => { const aliasActions = Option.some([Symbol('alias action')] as unknown) as Option.Some< diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index ba2508bf73cce..1119edde8e268 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -638,12 +638,20 @@ export const model = (currentState: State, resW: ResponseType): // alias_not_found_exception another instance has completed a // migration from the same source. return { ...stateP, controlState: 'MARK_VERSION_INDEX_READY_CONFLICT' }; - } else if ( - left.type === 'remove_index_not_a_concrete_index' || - left.type === 'index_not_found_exception' - ) { - // We don't handle these errors as the migration algorithm will never - // cause them to occur (these are only relevant to the LEGACY_DELETE + } else if (left.type === 'index_not_found_exception') { + if (left.index === stateP.tempIndex) { + // another instance has already completed the migration and deleted + // the temporary index + return { ...stateP, controlState: 'MARK_VERSION_INDEX_READY_CONFLICT' }; + } else { + // The migration algorithm will never cause a + // index_not_found_exception for an index other than the temporary + // index handled above. + throwBadResponse(stateP, left as never); + } + } else if (left.type === 'remove_index_not_a_concrete_index') { + // We don't handle this error as the migration algorithm will never + // cause it to occur (this error is only relevant to the LEGACY_DELETE // step). throwBadResponse(stateP, left as never); } else { diff --git a/test/scripts/run_multiple_kibana_nodes.sh b/test/scripts/run_multiple_kibana_nodes.sh new file mode 100755 index 0000000000000..f5661c19bed11 --- /dev/null +++ b/test/scripts/run_multiple_kibana_nodes.sh @@ -0,0 +1,86 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#!/bin/bash + +# +# Script to run multiple kibana nodes in parallel on the same machine. +# Make sure to run the script from kibana root directory. Some functions depend on the jq command-line utility +# being installed. +# +# bash test/scripts/run_multiple_kibana_nodes.sh [options] +# functions: +# start [instances] [args] - start multiple kibanas (3 default) +# es [args] - run elasticsearch +# tail - show logs of all kibanas +# kill - kills all started kibana processes +# clean - clean up nohup files +# kibana_index - search .kibana index against es +# + +FN="$1" + +if [ "${FN}" == "kill" ]; then + echo "killing main processes" + for pid in $(cat processes.out); do kill -9 $pid; done + echo "killing trailing processes" + for pid in $(pgrep -f scripts/kibana); do kill -9 $pid; done + exit 0; +fi + +if [ "${FN}" == "tail" ]; then + tail -f nohup_* + exit 0; +fi + +if [ "${FN}" == "clean" ]; then + rm -r nohup_*.out + rm processes.out + exit 0; +fi + +if [ "${FN}" == "es" ]; then + ARGS="$2" + yarn es snapshot $ARGS + exit 0; +fi + +if [ "${FN}" == "kibana_index" ]; then + # search the kibana index + curl -XPOST http://elastic:changeme@localhost:9200/.kibana/_search -u elastic:changeme -d '' | jq + exit 0; +fi + +if [ "${FN}" == "start" ]; then + NUM="$2" + ARGS="$3" + if test ! "${NUM-}"; then + NUM=3 + fi + node scripts/build_kibana_platform_plugins --no-examples + rm processes.out + for i in $(seq 0 $(expr $NUM - 1)) + do + PORT="56${i}1" + PROXY="56${i}3" + echo "starting kibana on port $PORT" + nohup node scripts/kibana.js --dev.basePathProxyTarget=$PROXY --server.port=$PORT --dev --no-watch --no-optimizer --no-base-path $ARGS > nohup_$i.out & + PROCESS_ID=$! + echo "${PROCESS_ID}" >> processes.out + done + exit 0; +fi