diff --git a/.ci/Jenkinsfile_coverage b/.ci/Jenkinsfile_coverage index fa1e141be93ea..6b8dc31bab34e 100644 --- a/.ci/Jenkinsfile_coverage +++ b/.ci/Jenkinsfile_coverage @@ -3,99 +3,91 @@ library 'kibana-pipeline-library' kibanaLibrary.load() // load from the Jenkins instance -stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit - timeout(time: 180, unit: 'MINUTES') { - timestamps { - ansiColor('xterm') { - catchError { +kibanaPipeline(timeoutMinutes: 180) { + catchErrors { + withEnv([ + 'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc. + ]) { + parallel([ + 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), + 'x-pack-intake-agent': { withEnv([ - 'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc. + 'NODE_ENV=test' // Needed for jest tests only ]) { - parallel([ - 'kibana-intake-agent': { - kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh')() - }, - 'x-pack-intake-agent': { - withEnv([ - 'NODE_ENV=test' // Needed for jest tests only - ]) { - kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh')() - } - }, - 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), - 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - ]), - 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - ]), - ]) - kibanaPipeline.jobRunner('tests-l', false) { - kibanaPipeline.downloadCoverageArtifacts() - kibanaPipeline.bash( - ''' - # bootstrap from x-pack folder - source src/dev/ci_setup/setup_env.sh - cd x-pack - yarn kbn bootstrap --prefer-offline - cd .. - # extract archives - mkdir -p /tmp/extracted_coverage - echo extracting intakes - tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage - tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage - echo extracting kibana-oss-tests - tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage - echo extracting kibana-xpack-tests - tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage - # replace path in json files to have valid html report - pwd=$(pwd) - du -sh /tmp/extracted_coverage/target/kibana-coverage/ - echo replacing path in json files - for i in {1..9}; do - sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json & - done - wait - # merge oss & x-pack reports - echo merging coverage reports - yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary - yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary - echo copy mocha reports - mkdir -p target/kibana-coverage/mocha-combined - cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined - ''', - "run `yarn kbn bootstrap && merge coverage`" - ) - sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*' - kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz') - sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*' - kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz') - sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*' - kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz') - } + workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh')() } - } - kibanaPipeline.sendMail() + }, + 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), + 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), + 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), + 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), + 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), + 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), + 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), + 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), + 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), + 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), + 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), + 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), + ]), + 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), + 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), + 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), + 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), + 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), + 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), + 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), + 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), + 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), + 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), + ]), + ]) + workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) { + kibanaPipeline.downloadCoverageArtifacts() + kibanaPipeline.bash( + ''' + # bootstrap from x-pack folder + source src/dev/ci_setup/setup_env.sh + cd x-pack + yarn kbn bootstrap --prefer-offline + cd .. + # extract archives + mkdir -p /tmp/extracted_coverage + echo extracting intakes + tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage + tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage + echo extracting kibana-oss-tests + tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage + echo extracting kibana-xpack-tests + tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage + # replace path in json files to have valid html report + pwd=$(pwd) + du -sh /tmp/extracted_coverage/target/kibana-coverage/ + echo replacing path in json files + for i in {1..9}; do + sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json & + done + wait + # merge oss & x-pack reports + echo merging coverage reports + yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary + yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary + echo copy mocha reports + mkdir -p target/kibana-coverage/mocha-combined + cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined + ''', + "run `yarn kbn bootstrap && merge coverage`" + ) + sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*' + kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz') + sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*' + kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz') + sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*' + kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz') } } } + kibanaPipeline.sendMail() } diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index f702405aad69e..befb8d259b5b6 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -21,53 +21,47 @@ def workerFailures = [] currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24) currentBuild.description = "${params.CI_GROUP}
Agents: ${AGENT_COUNT}
Executions: ${params.NUMBER_EXECUTIONS}" -stage("Kibana Pipeline") { - timeout(time: 180, unit: 'MINUTES') { - timestamps { - ansiColor('xterm') { - def agents = [:] - for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) { - def agentNumberInside = agentNumber - def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0) - agents["agent-${agentNumber}"] = { - catchError { - print "Agent ${agentNumberInside} - ${agentExecutions} executions" - - kibanaPipeline.withWorkers('flaky-test-runner', { - if (NEED_BUILD) { - if (!IS_XPACK) { - kibanaPipeline.buildOss() - if (CI_GROUP == '1') { - runbld("./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1") - } - } else { - kibanaPipeline.buildXpack() - } - } - }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))() +kibanaPipeline(timeoutMinutes: 180) { + def agents = [:] + for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) { + def agentNumberInside = agentNumber + def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0) + agents["agent-${agentNumber}"] = { + catchErrors { + print "Agent ${agentNumberInside} - ${agentExecutions} executions" + + workers.functional('flaky-test-runner', { + if (NEED_BUILD) { + if (!IS_XPACK) { + kibanaPipeline.buildOss() + if (CI_GROUP == '1') { + runbld("./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1") + } + } else { + kibanaPipeline.buildXpack() } } - } + }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))() + } + } + } - parallel(agents) + parallel(agents) - currentBuild.description += ", Failures: ${workerFailures.size()}" + currentBuild.description += ", Failures: ${workerFailures.size()}" - if (workerFailures.size() > 0) { - print "There were ${workerFailures.size()} test suite failures." - print "The executions that failed were:" - print workerFailures.join("\n") - print "Please check 'Test Result' and 'Pipeline Steps' pages for more info" - } - } - } + if (workerFailures.size() > 0) { + print "There were ${workerFailures.size()} test suite failures." + print "The executions that failed were:" + print workerFailures.join("\n") + print "Please check 'Test Result' and 'Pipeline Steps' pages for more info" } } def getWorkerFromParams(isXpack, job, ciGroup) { if (!isXpack) { if (job == 'serverMocha') { - return kibanaPipeline.getPostBuildWorker('serverMocha', { + return kibanaPipeline.functionalTestProcess('serverMocha', { kibanaPipeline.bash( """ source src/dev/ci_setup/setup_env.sh @@ -77,20 +71,20 @@ def getWorkerFromParams(isXpack, job, ciGroup) { ) }) } else if (job == 'firefoxSmoke') { - return kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }) + return kibanaPipeline.functionalTestProcess('firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh') } else if(job == 'visualRegression') { - return kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }) + return kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh') } else { - return kibanaPipeline.getOssCiGroupWorker(ciGroup) + return kibanaPipeline.ossCiGroupProcess(ciGroup) } } if (job == 'firefoxSmoke') { - return kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }) + return kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh') } else if(job == 'visualRegression') { - return kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }) + return kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh') } else { - return kibanaPipeline.getXpackCiGroupWorker(ciGroup) + return kibanaPipeline.xpackCiGroupProcess(ciGroup) } } @@ -105,10 +99,9 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor for(def j = 0; j < workerExecutions; j++) { print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}" withEnv([ - "JOB=agent-${agentNumber}-worker-${workerNumber}-${j}", "REMOVE_KIBANA_INSTALL_DIR=1", ]) { - catchError { + catchErrors { try { worker(workerNumber) } catch (ex) { diff --git a/.ci/es-snapshots/Jenkinsfile_build_es b/.ci/es-snapshots/Jenkinsfile_build_es index ad0ad54275e12..a00bcb3bbc946 100644 --- a/.ci/es-snapshots/Jenkinsfile_build_es +++ b/.ci/es-snapshots/Jenkinsfile_build_es @@ -26,7 +26,7 @@ timeout(time: 120, unit: 'MINUTES') { timestamps { ansiColor('xterm') { node('linux && immutable') { - catchError { + catchErrors { def VERSION def SNAPSHOT_ID def DESTINATION diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index 30d52a56547bd..ce472a404c053 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -19,50 +19,45 @@ currentBuild.description = "ES: ${SNAPSHOT_VERSION}
Kibana: ${params.branch def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${SNAPSHOT_VERSION}/archives/${SNAPSHOT_ID}/manifest.json" -timeout(time: 120, unit: 'MINUTES') { - timestamps { - ansiColor('xterm') { - catchError { - withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { - parallel([ - // TODO we just need to run integration tests from intake? - 'kibana-intake-agent': kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh'), - 'x-pack-intake-agent': kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh'), - 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), - 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - ]), - 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - ]), - ]) - } - - promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID) - } - - kibanaPipeline.sendMail() +kibanaPipeline(timeoutMinutes: 120) { + catchErrors { + withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) { + parallel([ + 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), + 'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'), + 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), + 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), + 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), + 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), + 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), + 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), + 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), + 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), + 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), + 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), + 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), + 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), + ]), + 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), + 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), + 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), + 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), + 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), + 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), + 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), + 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), + 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), + 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), + ]), + ]) } + + promoteSnapshot(SNAPSHOT_VERSION, SNAPSHOT_ID) } + + kibanaPipeline.sendMail() } def promoteSnapshot(snapshotVersion, snapshotId) { diff --git a/Jenkinsfile b/Jenkinsfile index 1b4350d5b91e9..85502369b07be 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,71 +3,49 @@ library 'kibana-pipeline-library' kibanaLibrary.load() -stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit - timeout(time: 135, unit: 'MINUTES') { - timestamps { - ansiColor('xterm') { - githubPr.withDefaultPrComments { - catchError { - retryable.enable() - parallel([ - 'kibana-intake-agent': kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh'), - 'x-pack-intake-agent': kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh'), - 'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ - // 'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { - // retryable('kibana-firefoxSmoke') { - // runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') - // } - // }), - 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1), - 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2), - 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3), - 'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4), - 'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5), - 'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6), - 'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7), - 'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8), - 'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9), - 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10), - 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11), - 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12), - 'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { - retryable('kibana-accessibility') { - runbld('./test/scripts/jenkins_accessibility.sh', 'Execute kibana-accessibility') - } - }), - // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }), - ]), - 'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ - // 'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { - // retryable('xpack-firefoxSmoke') { - // runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') - // } - // }), - 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1), - 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2), - 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3), - 'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4), - 'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5), - 'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6), - 'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7), - 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8), - 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9), - 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10), - 'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { - retryable('xpack-accessibility') { - runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility') - } - }), - // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }), - ]), - ]) - } - } - - retryable.printFlakyFailures() - kibanaPipeline.sendMail() - } +kibanaPipeline(timeoutMinutes: 135) { + githubPr.withDefaultPrComments { + catchError { + retryable.enable() + parallel([ + 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), + 'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'), + 'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [ + // 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'), + 'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1), + 'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2), + 'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3), + 'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4), + 'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5), + 'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6), + 'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7), + 'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8), + 'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9), + 'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10), + 'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11), + 'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12), + 'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'), + // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'), + ]), + 'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [ + // 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'), + 'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1), + 'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2), + 'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3), + 'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4), + 'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5), + 'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6), + 'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7), + 'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8), + 'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9), + 'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10), + 'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'), + // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'), + ]), + ]) } } + + retryable.printFlakyFailures() + kibanaPipeline.sendMail() } diff --git a/docs/development/core/public/kibana-plugin-public.appmountparameters.history.md b/docs/development/core/public/kibana-plugin-public.appmountparameters.history.md index 9a3fa1a1bb48a..f22e70b0e7bee 100644 --- a/docs/development/core/public/kibana-plugin-public.appmountparameters.history.md +++ b/docs/development/core/public/kibana-plugin-public.appmountparameters.history.md @@ -44,7 +44,6 @@ import { MyPluginDepsStart } from './plugin'; export renderApp = ({ element, history }: AppMountParameters) => { ReactDOM.render( - // pass `appBasePath` to `basename` , diff --git a/docs/development/core/public/kibana-plugin-public.appmountparameters.onappleave.md b/docs/development/core/public/kibana-plugin-public.appmountparameters.onappleave.md index 283ae34f14c54..6c5b89ffda05b 100644 --- a/docs/development/core/public/kibana-plugin-public.appmountparameters.onappleave.md +++ b/docs/development/core/public/kibana-plugin-public.appmountparameters.onappleave.md @@ -26,7 +26,7 @@ import { BrowserRouter, Route } from 'react-router-dom'; import { CoreStart, AppMountParams } from 'src/core/public'; import { MyPluginDepsStart } from './plugin'; -export renderApp = ({ appBasePath, element, onAppLeave }: AppMountParams) => { +export renderApp = ({ element, history, onAppLeave }: AppMountParams) => { const { renderApp, hasUnsavedChanges } = await import('./application'); onAppLeave(actions => { if(hasUnsavedChanges()) { @@ -34,7 +34,7 @@ export renderApp = ({ appBasePath, element, onAppLeave }: AppMountParams) => { } return actions.default(); }); - return renderApp(params); + return renderApp({ element, history }); } ``` diff --git a/docs/development/core/server/kibana-plugin-server.md b/docs/development/core/server/kibana-plugin-server.md index 15a1fd0506256..c948c89920796 100644 --- a/docs/development/core/server/kibana-plugin-server.md +++ b/docs/development/core/server/kibana-plugin-server.md @@ -88,11 +88,16 @@ The plugin integrates with the core system via lifecycle events: `setup` | [Logger](./kibana-plugin-server.logger.md) | Logger exposes all the necessary methods to log any type of information and this is the interface used by the logging consumers including plugins. | | [LoggerFactory](./kibana-plugin-server.loggerfactory.md) | The single purpose of LoggerFactory interface is to define a way to retrieve a context-based logger instance. | | [LogMeta](./kibana-plugin-server.logmeta.md) | Contextual metadata | +| [MetricsServiceSetup](./kibana-plugin-server.metricsservicesetup.md) | APIs to retrieves metrics gathered and exposed by the core platform. | | [OnPostAuthToolkit](./kibana-plugin-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. | | [OnPreAuthToolkit](./kibana-plugin-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. | | [OnPreResponseExtensions](./kibana-plugin-server.onpreresponseextensions.md) | Additional data to extend a response. | | [OnPreResponseInfo](./kibana-plugin-server.onpreresponseinfo.md) | Response status code. | | [OnPreResponseToolkit](./kibana-plugin-server.onpreresponsetoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. | +| [OpsMetrics](./kibana-plugin-server.opsmetrics.md) | Regroups metrics gathered by all the collectors. This contains metrics about the os/runtime, the kibana process and the http server. | +| [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) | OS related metrics | +| [OpsProcessMetrics](./kibana-plugin-server.opsprocessmetrics.md) | Process related metrics | +| [OpsServerMetrics](./kibana-plugin-server.opsservermetrics.md) | server related metrics | | [PackageInfo](./kibana-plugin-server.packageinfo.md) | | | [Plugin](./kibana-plugin-server.plugin.md) | The interface that should be returned by a PluginInitializer. | | [PluginConfigDescriptor](./kibana-plugin-server.pluginconfigdescriptor.md) | Describes a plugin configuration properties. | diff --git a/docs/development/core/server/kibana-plugin-server.metricsservicesetup.getopsmetrics_.md b/docs/development/core/server/kibana-plugin-server.metricsservicesetup.getopsmetrics_.md new file mode 100644 index 0000000000000..454b8c905451e --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.metricsservicesetup.getopsmetrics_.md @@ -0,0 +1,24 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [MetricsServiceSetup](./kibana-plugin-server.metricsservicesetup.md) > [getOpsMetrics$](./kibana-plugin-server.metricsservicesetup.getopsmetrics_.md) + +## MetricsServiceSetup.getOpsMetrics$ property + +Retrieve an observable emitting the [OpsMetrics](./kibana-plugin-server.opsmetrics.md) gathered. The observable will emit an initial value during core's `start` phase, and a new value every fixed interval of time, based on the `opts.interval` configuration property. + +Signature: + +```typescript +getOpsMetrics$: () => Observable; +``` + +## Example + + +```ts +core.metrics.getOpsMetrics$().subscribe(metrics => { + // do something with the metrics +}) + +``` + diff --git a/docs/development/core/server/kibana-plugin-server.metricsservicesetup.md b/docs/development/core/server/kibana-plugin-server.metricsservicesetup.md new file mode 100644 index 0000000000000..270c56402a390 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.metricsservicesetup.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [MetricsServiceSetup](./kibana-plugin-server.metricsservicesetup.md) + +## MetricsServiceSetup interface + +APIs to retrieves metrics gathered and exposed by the core platform. + +Signature: + +```typescript +export interface MetricsServiceSetup +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [getOpsMetrics$](./kibana-plugin-server.metricsservicesetup.getopsmetrics_.md) | () => Observable<OpsMetrics> | Retrieve an observable emitting the [OpsMetrics](./kibana-plugin-server.opsmetrics.md) gathered. The observable will emit an initial value during core's start phase, and a new value every fixed interval of time, based on the opts.interval configuration property. | + diff --git a/docs/development/core/server/kibana-plugin-server.opsmetrics.concurrent_connections.md b/docs/development/core/server/kibana-plugin-server.opsmetrics.concurrent_connections.md new file mode 100644 index 0000000000000..cfd39a551ad34 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsmetrics.concurrent_connections.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsMetrics](./kibana-plugin-server.opsmetrics.md) > [concurrent\_connections](./kibana-plugin-server.opsmetrics.concurrent_connections.md) + +## OpsMetrics.concurrent\_connections property + +number of current concurrent connections to the server + +Signature: + +```typescript +concurrent_connections: OpsServerMetrics['concurrent_connections']; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsmetrics.md b/docs/development/core/server/kibana-plugin-server.opsmetrics.md new file mode 100644 index 0000000000000..e23bd8d431d3f --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsmetrics.md @@ -0,0 +1,24 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsMetrics](./kibana-plugin-server.opsmetrics.md) + +## OpsMetrics interface + +Regroups metrics gathered by all the collectors. This contains metrics about the os/runtime, the kibana process and the http server. + +Signature: + +```typescript +export interface OpsMetrics +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [concurrent\_connections](./kibana-plugin-server.opsmetrics.concurrent_connections.md) | OpsServerMetrics['concurrent_connections'] | number of current concurrent connections to the server | +| [os](./kibana-plugin-server.opsmetrics.os.md) | OpsOsMetrics | OS related metrics | +| [process](./kibana-plugin-server.opsmetrics.process.md) | OpsProcessMetrics | Process related metrics | +| [requests](./kibana-plugin-server.opsmetrics.requests.md) | OpsServerMetrics['requests'] | server requests stats | +| [response\_times](./kibana-plugin-server.opsmetrics.response_times.md) | OpsServerMetrics['response_times'] | server response time stats | + diff --git a/docs/development/core/server/kibana-plugin-server.opsmetrics.os.md b/docs/development/core/server/kibana-plugin-server.opsmetrics.os.md new file mode 100644 index 0000000000000..993a1d7a2d7b7 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsmetrics.os.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsMetrics](./kibana-plugin-server.opsmetrics.md) > [os](./kibana-plugin-server.opsmetrics.os.md) + +## OpsMetrics.os property + +OS related metrics + +Signature: + +```typescript +os: OpsOsMetrics; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsmetrics.process.md b/docs/development/core/server/kibana-plugin-server.opsmetrics.process.md new file mode 100644 index 0000000000000..53d3a33d66e06 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsmetrics.process.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsMetrics](./kibana-plugin-server.opsmetrics.md) > [process](./kibana-plugin-server.opsmetrics.process.md) + +## OpsMetrics.process property + +Process related metrics + +Signature: + +```typescript +process: OpsProcessMetrics; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsmetrics.requests.md b/docs/development/core/server/kibana-plugin-server.opsmetrics.requests.md new file mode 100644 index 0000000000000..9cd6b85e507f0 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsmetrics.requests.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsMetrics](./kibana-plugin-server.opsmetrics.md) > [requests](./kibana-plugin-server.opsmetrics.requests.md) + +## OpsMetrics.requests property + +server requests stats + +Signature: + +```typescript +requests: OpsServerMetrics['requests']; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsmetrics.response_times.md b/docs/development/core/server/kibana-plugin-server.opsmetrics.response_times.md new file mode 100644 index 0000000000000..358699071b1c3 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsmetrics.response_times.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsMetrics](./kibana-plugin-server.opsmetrics.md) > [response\_times](./kibana-plugin-server.opsmetrics.response_times.md) + +## OpsMetrics.response\_times property + +server response time stats + +Signature: + +```typescript +response_times: OpsServerMetrics['response_times']; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.distro.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.distro.md new file mode 100644 index 0000000000000..338164f173d02 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.distro.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [distro](./kibana-plugin-server.opsosmetrics.distro.md) + +## OpsOsMetrics.distro property + +The os distrib. Only present for linux platforms + +Signature: + +```typescript +distro?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.distrorelease.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.distrorelease.md new file mode 100644 index 0000000000000..24c5a1f00b64c --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.distrorelease.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [distroRelease](./kibana-plugin-server.opsosmetrics.distrorelease.md) + +## OpsOsMetrics.distroRelease property + +The os distrib release, prefixed by the os distrib. Only present for linux platforms + +Signature: + +```typescript +distroRelease?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.load.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.load.md new file mode 100644 index 0000000000000..0bf17502ce34e --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.load.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [load](./kibana-plugin-server.opsosmetrics.load.md) + +## OpsOsMetrics.load property + +cpu load metrics + +Signature: + +```typescript +load: { + '1m': number; + '5m': number; + '15m': number; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.md new file mode 100644 index 0000000000000..0fb4e59fdf539 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.md @@ -0,0 +1,26 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) + +## OpsOsMetrics interface + +OS related metrics + +Signature: + +```typescript +export interface OpsOsMetrics +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [distro](./kibana-plugin-server.opsosmetrics.distro.md) | string | The os distrib. Only present for linux platforms | +| [distroRelease](./kibana-plugin-server.opsosmetrics.distrorelease.md) | string | The os distrib release, prefixed by the os distrib. Only present for linux platforms | +| [load](./kibana-plugin-server.opsosmetrics.load.md) | {
'1m': number;
'5m': number;
'15m': number;
} | cpu load metrics | +| [memory](./kibana-plugin-server.opsosmetrics.memory.md) | {
total_in_bytes: number;
free_in_bytes: number;
used_in_bytes: number;
} | system memory usage metrics | +| [platform](./kibana-plugin-server.opsosmetrics.platform.md) | NodeJS.Platform | The os platform | +| [platformRelease](./kibana-plugin-server.opsosmetrics.platformrelease.md) | string | The os platform release, prefixed by the platform name | +| [uptime\_in\_millis](./kibana-plugin-server.opsosmetrics.uptime_in_millis.md) | number | the OS uptime | + diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.memory.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.memory.md new file mode 100644 index 0000000000000..4a1becaeeaec7 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.memory.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [memory](./kibana-plugin-server.opsosmetrics.memory.md) + +## OpsOsMetrics.memory property + +system memory usage metrics + +Signature: + +```typescript +memory: { + total_in_bytes: number; + free_in_bytes: number; + used_in_bytes: number; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.platform.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.platform.md new file mode 100644 index 0000000000000..411d0fc546dc0 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.platform.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [platform](./kibana-plugin-server.opsosmetrics.platform.md) + +## OpsOsMetrics.platform property + +The os platform + +Signature: + +```typescript +platform: NodeJS.Platform; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.platformrelease.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.platformrelease.md new file mode 100644 index 0000000000000..1071b4a38f588 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.platformrelease.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [platformRelease](./kibana-plugin-server.opsosmetrics.platformrelease.md) + +## OpsOsMetrics.platformRelease property + +The os platform release, prefixed by the platform name + +Signature: + +```typescript +platformRelease: string; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsosmetrics.uptime_in_millis.md b/docs/development/core/server/kibana-plugin-server.opsosmetrics.uptime_in_millis.md new file mode 100644 index 0000000000000..dfff1a1f1da0b --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsosmetrics.uptime_in_millis.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsOsMetrics](./kibana-plugin-server.opsosmetrics.md) > [uptime\_in\_millis](./kibana-plugin-server.opsosmetrics.uptime_in_millis.md) + +## OpsOsMetrics.uptime\_in\_millis property + +the OS uptime + +Signature: + +```typescript +uptime_in_millis: number; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.event_loop_delay.md b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.event_loop_delay.md new file mode 100644 index 0000000000000..f61c8b0995324 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.event_loop_delay.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsProcessMetrics](./kibana-plugin-server.opsprocessmetrics.md) > [event\_loop\_delay](./kibana-plugin-server.opsprocessmetrics.event_loop_delay.md) + +## OpsProcessMetrics.event\_loop\_delay property + +node event loop delay + +Signature: + +```typescript +event_loop_delay: number; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.md b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.md new file mode 100644 index 0000000000000..92fd8471cce7d --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.md @@ -0,0 +1,23 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsProcessMetrics](./kibana-plugin-server.opsprocessmetrics.md) + +## OpsProcessMetrics interface + +Process related metrics + +Signature: + +```typescript +export interface OpsProcessMetrics +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [event\_loop\_delay](./kibana-plugin-server.opsprocessmetrics.event_loop_delay.md) | number | node event loop delay | +| [memory](./kibana-plugin-server.opsprocessmetrics.memory.md) | {
heap: {
total_in_bytes: number;
used_in_bytes: number;
size_limit: number;
};
resident_set_size_in_bytes: number;
} | process memory usage | +| [pid](./kibana-plugin-server.opsprocessmetrics.pid.md) | number | pid of the kibana process | +| [uptime\_in\_millis](./kibana-plugin-server.opsprocessmetrics.uptime_in_millis.md) | number | uptime of the kibana process | + diff --git a/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.memory.md b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.memory.md new file mode 100644 index 0000000000000..5c1a8de70dc01 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.memory.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsProcessMetrics](./kibana-plugin-server.opsprocessmetrics.md) > [memory](./kibana-plugin-server.opsprocessmetrics.memory.md) + +## OpsProcessMetrics.memory property + +process memory usage + +Signature: + +```typescript +memory: { + heap: { + total_in_bytes: number; + used_in_bytes: number; + size_limit: number; + }; + resident_set_size_in_bytes: number; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.pid.md b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.pid.md new file mode 100644 index 0000000000000..a34187f372018 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.pid.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsProcessMetrics](./kibana-plugin-server.opsprocessmetrics.md) > [pid](./kibana-plugin-server.opsprocessmetrics.pid.md) + +## OpsProcessMetrics.pid property + +pid of the kibana process + +Signature: + +```typescript +pid: number; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.uptime_in_millis.md b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.uptime_in_millis.md new file mode 100644 index 0000000000000..24db2f017a663 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsprocessmetrics.uptime_in_millis.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsProcessMetrics](./kibana-plugin-server.opsprocessmetrics.md) > [uptime\_in\_millis](./kibana-plugin-server.opsprocessmetrics.uptime_in_millis.md) + +## OpsProcessMetrics.uptime\_in\_millis property + +uptime of the kibana process + +Signature: + +```typescript +uptime_in_millis: number; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsservermetrics.concurrent_connections.md b/docs/development/core/server/kibana-plugin-server.opsservermetrics.concurrent_connections.md new file mode 100644 index 0000000000000..ade79fedfa1b5 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsservermetrics.concurrent_connections.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsServerMetrics](./kibana-plugin-server.opsservermetrics.md) > [concurrent\_connections](./kibana-plugin-server.opsservermetrics.concurrent_connections.md) + +## OpsServerMetrics.concurrent\_connections property + +number of current concurrent connections to the server + +Signature: + +```typescript +concurrent_connections: number; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsservermetrics.md b/docs/development/core/server/kibana-plugin-server.opsservermetrics.md new file mode 100644 index 0000000000000..4e35c02bd9f28 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsservermetrics.md @@ -0,0 +1,22 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsServerMetrics](./kibana-plugin-server.opsservermetrics.md) + +## OpsServerMetrics interface + +server related metrics + +Signature: + +```typescript +export interface OpsServerMetrics +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [concurrent\_connections](./kibana-plugin-server.opsservermetrics.concurrent_connections.md) | number | number of current concurrent connections to the server | +| [requests](./kibana-plugin-server.opsservermetrics.requests.md) | {
disconnects: number;
total: number;
statusCodes: Record<number, number>;
} | server requests stats | +| [response\_times](./kibana-plugin-server.opsservermetrics.response_times.md) | {
avg_in_millis: number;
max_in_millis: number;
} | server response time stats | + diff --git a/docs/development/core/server/kibana-plugin-server.opsservermetrics.requests.md b/docs/development/core/server/kibana-plugin-server.opsservermetrics.requests.md new file mode 100644 index 0000000000000..5ad2abc869557 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsservermetrics.requests.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsServerMetrics](./kibana-plugin-server.opsservermetrics.md) > [requests](./kibana-plugin-server.opsservermetrics.requests.md) + +## OpsServerMetrics.requests property + +server requests stats + +Signature: + +```typescript +requests: { + disconnects: number; + total: number; + statusCodes: Record; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-server.opsservermetrics.response_times.md b/docs/development/core/server/kibana-plugin-server.opsservermetrics.response_times.md new file mode 100644 index 0000000000000..5008efc6ad4da --- /dev/null +++ b/docs/development/core/server/kibana-plugin-server.opsservermetrics.response_times.md @@ -0,0 +1,16 @@ + + +[Home](./index.md) > [kibana-plugin-server](./kibana-plugin-server.md) > [OpsServerMetrics](./kibana-plugin-server.opsservermetrics.md) > [response\_times](./kibana-plugin-server.opsservermetrics.response_times.md) + +## OpsServerMetrics.response\_times property + +server response time stats + +Signature: + +```typescript +response_times: { + avg_in_millis: number; + max_in_millis: number; + }; +``` diff --git a/docs/redirects.asciidoc b/docs/redirects.asciidoc index 3843cc27defd5..8ad5330f3fda5 100644 --- a/docs/redirects.asciidoc +++ b/docs/redirects.asciidoc @@ -56,6 +56,11 @@ This page has moved. Please see <>. This page has moved. Please see <>. +[role="exclude",id="add-sample-data"] +== Add sample data + +This page has moved. Please see <>. + [role="exclude",id="tilemap"] == Coordinate map diff --git a/docs/user/getting-started.asciidoc b/docs/user/getting-started.asciidoc index c6fe5b5b92d69..d426ec111351c 100644 --- a/docs/user/getting-started.asciidoc +++ b/docs/user/getting-started.asciidoc @@ -1,54 +1,65 @@ [[getting-started]] -= Getting Started += Get started [partintro] -- -You’re new to Kibana and want to give it a try. {kib} has sample data sets and -tutorials to help you get started. +Ready to try out {kib} and see what it can do? To quickest way to get started with {kib} is to set up on Cloud, then add a sample data set that helps you get a handle on the full range of {kib} features. [float] -=== Sample data +[[cloud-set-up]] +== Set up on Cloud -You can use the <> to take {kib} for a test ride without having -to go through the process of loading data yourself. With one click, -you can install a sample data set and start interacting with -{kib} visualizations in seconds. You can access the sample data -from the {kib} home page. +To access {kib} in a single click, run our hosted Elasticsearch Service on Elastic Cloud. -[float] +. Log into the link:https://cloud.elastic.co/[Elasticsearch Service Console]. +If you need an account, register for a link:https://www.elastic.co/cloud/elasticsearch-service/signup[free 14-day trial]. + +. Click *Create deployment*, then give your deployment a name. -=== Add data tutorials -{kib} has built-in *Add Data* tutorials to help you set up -data flows in the Elastic Stack. These tutorials are available -from the Kibana home page. In *Add Data to Kibana*, find the data type -you’re interested in, and click its button to view a list of available tutorials. +. To use the default options, click *Create deployment*. You can modify the other deployment options, but the default options are great to get started. + +Be sure to copy down the password for the `elastic` user and Cloud ID information. You'll need that later. [float] -=== Hands-on experience +[[get-data-in]] +== Get data into {kib} + +The easiest way to get data into {kib} is to add a sample data set. + +{kib} has several sample data sets that you can use before loading your own data: + +* *Sample eCommerce orders* includes visualizations for tracking product-related information, +such as cost, revenue, and price. + +* *Sample flight data* includes visualizations for monitoring flight routes. -The following tutorials walk you through searching, analyzing, -and visualizing data. +* *Sample web logs* includes visualizations for monitoring website traffic. -* <>. You'll -learn to filter and query data, edit visualizations, and interact with dashboards. +To use the sample data sets: -* <>. You'll manually load a data set and build -your own visualizations and dashboard. +. Go to the {kib} home page. + +. Click *Load a data set and a {kib} dashboard*. + +. Click *View data* and view the prepackaged dashboards, maps, and more. + +[role="screenshot"] +image::images/add-sample-data.png[] + +NOTE: The timestamps in the sample data sets are relative to when they are installed. +If you uninstall and reinstall a data set, the timestamps change to reflect the most recent installation. [float] -=== Before you begin +[[getting-started-next-steps]] +== Next steps -Make sure you've <> and established -a <>. +* To get a hands-on experience creating visualizations, follow the <> tutorial. -If you are running our hosted Elasticsearch Service on Elastic Cloud, you access Kibana with a single click. (You can {ess-trial}[sign up for a free trial] and start exploring data in minutes.) +* If you're ready to load an actual data set and build a dashboard, follow the <> tutorial. -- -include::{kib-repo-dir}/getting-started/add-sample-data.asciidoc[] - include::{kib-repo-dir}/getting-started/tutorial-sample-data.asciidoc[] include::{kib-repo-dir}/getting-started/tutorial-full-experience.asciidoc[] @@ -60,4 +71,3 @@ include::{kib-repo-dir}/getting-started/tutorial-discovering.asciidoc[] include::{kib-repo-dir}/getting-started/tutorial-visualizing.asciidoc[] include::{kib-repo-dir}/getting-started/tutorial-dashboard.asciidoc[] - diff --git a/docs/user/index.asciidoc b/docs/user/index.asciidoc index 3911d57e05c9a..ff100d0763368 100644 --- a/docs/user/index.asciidoc +++ b/docs/user/index.asciidoc @@ -1,13 +1,13 @@ include::introduction.asciidoc[] +include::getting-started.asciidoc[] + include::setup.asciidoc[] include::monitoring/configuring-monitoring.asciidoc[] include::security/securing-kibana.asciidoc[] -include::getting-started.asciidoc[] - include::discover.asciidoc[] include::visualize.asciidoc[] diff --git a/package.json b/package.json index 0f04a2fba3b65..2c401724c72cd 100644 --- a/package.json +++ b/package.json @@ -323,6 +323,7 @@ "@types/fetch-mock": "^7.3.1", "@types/flot": "^0.0.31", "@types/getopts": "^2.0.1", + "@types/getos": "^3.0.0", "@types/glob": "^7.1.1", "@types/globby": "^8.0.0", "@types/graphql": "^0.13.2", @@ -349,6 +350,7 @@ "@types/mustache": "^0.8.31", "@types/node": "^10.12.27", "@types/node-forge": "^0.9.0", + "@types/normalize-path": "^3.0.0", "@types/numeral": "^0.0.26", "@types/opn": "^5.1.0", "@types/pegjs": "^0.10.1", @@ -384,7 +386,7 @@ "@typescript-eslint/parser": "^2.15.0", "angular-mocks": "^1.7.9", "archiver": "^3.1.1", - "axe-core": "^3.3.2", + "axe-core": "^3.4.1", "babel-eslint": "^10.0.3", "babel-jest": "^24.9.0", "babel-plugin-dynamic-import-node": "^2.3.0", diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts index 66fa55479f3b9..817c4796562e8 100644 --- a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/index.ts @@ -17,6 +17,7 @@ * under the License. */ +import './legacy/styles.scss'; import { fooLibFn } from '../../foo/public/index'; export * from './lib'; export { fooLibFn }; diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss new file mode 100644 index 0000000000000..e71a2d485a2f8 --- /dev/null +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss @@ -0,0 +1,4 @@ +body { + width: $globalStyleConstant; + background-image: url("ui/icon.svg"); +} diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/icon.svg b/packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/icon.svg new file mode 100644 index 0000000000000..ae7d5b958bbad --- /dev/null +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/styles/_styling_constants.scss b/packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/styles/_styling_constants.scss new file mode 100644 index 0000000000000..83995ca65211b --- /dev/null +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/src/legacy/ui/public/styles/_styling_constants.scss @@ -0,0 +1 @@ +$globalStyleConstant: 10; diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 706f79978beee..1a974d3e81092 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -5,553 +5,56 @@ OptimizerConfig { "bundles": Array [ Bundle { "cache": BundleCache { - "path": /plugins/bar/target/public/.kbn-optimizer-cache, + "path": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public/.kbn-optimizer-cache, "state": undefined, }, - "contextDir": /plugins/bar, + "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, "entry": "./public/index", "id": "bar", - "outputDir": /plugins/bar/target/public, - "sourceRoot": , + "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public, + "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, Bundle { "cache": BundleCache { - "path": /plugins/foo/target/public/.kbn-optimizer-cache, + "path": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public/.kbn-optimizer-cache, "state": undefined, }, - "contextDir": /plugins/foo, + "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, "entry": "./public/index", "id": "foo", - "outputDir": /plugins/foo/target/public, - "sourceRoot": , + "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public, + "sourceRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "type": "plugin", }, ], "cache": true, - "dist": false, + "dist": true, "inspectWorkers": false, "maxWorkerCount": 1, "plugins": Array [ Object { - "directory": /plugins/bar, + "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, "id": "bar", "isUiPlugin": true, }, Object { - "directory": /plugins/baz, + "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/baz, "id": "baz", "isUiPlugin": false, }, Object { - "directory": /plugins/foo, + "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, "id": "foo", "isUiPlugin": true, }, ], "profileWebpack": false, - "repoRoot": , + "repoRoot": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo, "watch": false, } `; -exports[`builds expected bundles, saves bundle counts to metadata: bar bundle 1`] = ` -"var __kbnBundles__ = typeof __kbnBundles__ === \\"object\\" ? __kbnBundles__ : {}; __kbnBundles__[\\"plugin/bar\\"] = -/******/ (function(modules) { // webpackBootstrap -/******/ // The module cache -/******/ var installedModules = {}; -/******/ -/******/ // The require function -/******/ function __webpack_require__(moduleId) { -/******/ -/******/ // Check if module is in cache -/******/ if(installedModules[moduleId]) { -/******/ return installedModules[moduleId].exports; -/******/ } -/******/ // Create a new module (and put it into the cache) -/******/ var module = installedModules[moduleId] = { -/******/ i: moduleId, -/******/ l: false, -/******/ exports: {} -/******/ }; -/******/ -/******/ // Execute the module function -/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); -/******/ -/******/ // Flag the module as loaded -/******/ module.l = true; -/******/ -/******/ // Return the exports of the module -/******/ return module.exports; -/******/ } -/******/ -/******/ -/******/ // expose the modules object (__webpack_modules__) -/******/ __webpack_require__.m = modules; -/******/ -/******/ // expose the module cache -/******/ __webpack_require__.c = installedModules; -/******/ -/******/ // define getter function for harmony exports -/******/ __webpack_require__.d = function(exports, name, getter) { -/******/ if(!__webpack_require__.o(exports, name)) { -/******/ Object.defineProperty(exports, name, { enumerable: true, get: getter }); -/******/ } -/******/ }; -/******/ -/******/ // define __esModule on exports -/******/ __webpack_require__.r = function(exports) { -/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { -/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); -/******/ } -/******/ Object.defineProperty(exports, '__esModule', { value: true }); -/******/ }; -/******/ -/******/ // create a fake namespace object -/******/ // mode & 1: value is a module id, require it -/******/ // mode & 2: merge all properties of value into the ns -/******/ // mode & 4: return value when already ns object -/******/ // mode & 8|1: behave like require -/******/ __webpack_require__.t = function(value, mode) { -/******/ if(mode & 1) value = __webpack_require__(value); -/******/ if(mode & 8) return value; -/******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value; -/******/ var ns = Object.create(null); -/******/ __webpack_require__.r(ns); -/******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value }); -/******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key)); -/******/ return ns; -/******/ }; -/******/ -/******/ // getDefaultExport function for compatibility with non-harmony modules -/******/ __webpack_require__.n = function(module) { -/******/ var getter = module && module.__esModule ? -/******/ function getDefault() { return module['default']; } : -/******/ function getModuleExports() { return module; }; -/******/ __webpack_require__.d(getter, 'a', getter); -/******/ return getter; -/******/ }; -/******/ -/******/ // Object.prototype.hasOwnProperty.call -/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; -/******/ -/******/ // __webpack_public_path__ -/******/ __webpack_require__.p = \\"__REPLACE_WITH_PUBLIC_PATH__\\"; -/******/ -/******/ -/******/ // Load entry module and return exports -/******/ return __webpack_require__(__webpack_require__.s = \\"./public/index.ts\\"); -/******/ }) -/************************************************************************/ -/******/ ({ +exports[`builds expected bundles, saves bundle counts to metadata: bar bundle 1`] = `"var __kbnBundles__=typeof __kbnBundles__===\\"object\\"?__kbnBundles__:{};__kbnBundles__[\\"plugin/bar\\"]=function(modules){var installedModules={};function __webpack_require__(moduleId){if(installedModules[moduleId]){return installedModules[moduleId].exports}var module=installedModules[moduleId]={i:moduleId,l:false,exports:{}};modules[moduleId].call(module.exports,module,module.exports,__webpack_require__);module.l=true;return module.exports}__webpack_require__.m=modules;__webpack_require__.c=installedModules;__webpack_require__.d=function(exports,name,getter){if(!__webpack_require__.o(exports,name)){Object.defineProperty(exports,name,{enumerable:true,get:getter})}};__webpack_require__.r=function(exports){if(typeof Symbol!==\\"undefined\\"&&Symbol.toStringTag){Object.defineProperty(exports,Symbol.toStringTag,{value:\\"Module\\"})}Object.defineProperty(exports,\\"__esModule\\",{value:true})};__webpack_require__.t=function(value,mode){if(mode&1)value=__webpack_require__(value);if(mode&8)return value;if(mode&4&&typeof value===\\"object\\"&&value&&value.__esModule)return value;var ns=Object.create(null);__webpack_require__.r(ns);Object.defineProperty(ns,\\"default\\",{enumerable:true,value:value});if(mode&2&&typeof value!=\\"string\\")for(var key in value)__webpack_require__.d(ns,key,function(key){return value[key]}.bind(null,key));return ns};__webpack_require__.n=function(module){var getter=module&&module.__esModule?function getDefault(){return module[\\"default\\"]}:function getModuleExports(){return module};__webpack_require__.d(getter,\\"a\\",getter);return getter};__webpack_require__.o=function(object,property){return Object.prototype.hasOwnProperty.call(object,property)};__webpack_require__.p=\\"__REPLACE_WITH_PUBLIC_PATH__\\";return __webpack_require__(__webpack_require__.s=4)}([function(module,exports,__webpack_require__){\\"use strict\\";var isOldIE=function isOldIE(){var memo;return function memorize(){if(typeof memo===\\"undefined\\"){memo=Boolean(window&&document&&document.all&&!window.atob)}return memo}}();var getTarget=function getTarget(){var memo={};return function memorize(target){if(typeof memo[target]===\\"undefined\\"){var styleTarget=document.querySelector(target);if(window.HTMLIFrameElement&&styleTarget instanceof window.HTMLIFrameElement){try{styleTarget=styleTarget.contentDocument.head}catch(e){styleTarget=null}}memo[target]=styleTarget}return memo[target]}}();var stylesInDom=[];function getIndexByIdentifier(identifier){var result=-1;for(var i=0;i { await del(TMP_DIR); @@ -51,20 +51,25 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { repoRoot: MOCK_REPO_DIR, pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')], maxWorkerCount: 1, + dist: true, }); expect(config).toMatchSnapshot('OptimizerConfig'); - const msgs = await runOptimizer(config) - .pipe( - tap(state => { - if (state.event?.type === 'worker stdio') { - // eslint-disable-next-line no-console - console.log('worker', state.event.stream, state.event.chunk.toString('utf8')); + const log = new ToolingLog({ + level: 'error', + writeTo: { + write(chunk) { + if (chunk.endsWith('\n')) { + chunk = chunk.slice(0, -1); } - }), - toArray() - ) + // eslint-disable-next-line no-console + console.error(chunk); + }, + }, + }); + const msgs = await runOptimizer(config) + .pipe(logOptimizerState(log, config), toArray()) .toPromise(); const assert = (statement: string, truth: boolean, altStates?: OptimizerUpdate[]) => { @@ -133,23 +138,31 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { expect(foo.cache.getModuleCount()).toBe(3); expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(` Array [ - /plugins/foo/public/ext.ts, - /plugins/foo/public/index.ts, - /plugins/foo/public/lib.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, ] `); const bar = config.bundles.find(b => b.id === 'bar')!; expect(bar).toBeTruthy(); bar.cache.refresh(); - expect(bar.cache.getModuleCount()).toBe(5); + expect(bar.cache.getModuleCount()).toBe( + // code + styles + style/css-loader runtime + 14 + ); + expect(bar.cache.getReferencedFiles()).toMatchInlineSnapshot(` Array [ - /plugins/foo/public/ext.ts, - /plugins/foo/public/index.ts, - /plugins/foo/public/lib.ts, - /plugins/bar/public/index.ts, - /plugins/bar/public/lib.ts, + /node_modules/css-loader/package.json, + /node_modules/style-loader/package.json, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/lib.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg, ] `); }); @@ -159,6 +172,7 @@ it('uses cache on second run and exist cleanly', async () => { repoRoot: MOCK_REPO_DIR, pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')], maxWorkerCount: 1, + dist: true, }); const msgs = await runOptimizer(config) diff --git a/packages/kbn-optimizer/src/worker/__snapshots__/parse_path.test.ts.snap b/packages/kbn-optimizer/src/worker/__snapshots__/parse_path.test.ts.snap new file mode 100644 index 0000000000000..2973ac116d6bd --- /dev/null +++ b/packages/kbn-optimizer/src/worker/__snapshots__/parse_path.test.ts.snap @@ -0,0 +1,156 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`parseDirPath() parses / 1`] = ` +Object { + "dirs": Array [], + "filename": undefined, + "root": "/", +} +`; + +exports[`parseDirPath() parses /foo 1`] = ` +Object { + "dirs": Array [ + "foo", + ], + "filename": undefined, + "root": "/", +} +`; + +exports[`parseDirPath() parses /foo/bar/baz 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + "baz", + ], + "filename": undefined, + "root": "/", +} +`; + +exports[`parseDirPath() parses /foo/bar/baz/ 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + "baz", + ], + "filename": undefined, + "root": "/", +} +`; + +exports[`parseDirPath() parses c:\\ 1`] = ` +Object { + "dirs": Array [], + "filename": undefined, + "root": "c:", +} +`; + +exports[`parseDirPath() parses c:\\foo 1`] = ` +Object { + "dirs": Array [ + "foo", + ], + "filename": undefined, + "root": "c:", +} +`; + +exports[`parseDirPath() parses c:\\foo\\bar\\baz 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + "baz", + ], + "filename": undefined, + "root": "c:", +} +`; + +exports[`parseDirPath() parses c:\\foo\\bar\\baz\\ 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + "baz", + ], + "filename": undefined, + "root": "c:", +} +`; + +exports[`parseFilePath() parses /foo 1`] = ` +Object { + "dirs": Array [], + "filename": "foo", + "root": "/", +} +`; + +exports[`parseFilePath() parses /foo/bar/baz 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + ], + "filename": "baz", + "root": "/", +} +`; + +exports[`parseFilePath() parses /foo/bar/baz.json 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + ], + "filename": "baz.json", + "root": "/", +} +`; + +exports[`parseFilePath() parses c:/foo/bar/baz.json 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + ], + "filename": "baz.json", + "root": "c:", +} +`; + +exports[`parseFilePath() parses c:\\foo 1`] = ` +Object { + "dirs": Array [], + "filename": "foo", + "root": "c:", +} +`; + +exports[`parseFilePath() parses c:\\foo\\bar\\baz 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + ], + "filename": "baz", + "root": "c:", +} +`; + +exports[`parseFilePath() parses c:\\foo\\bar\\baz.json 1`] = ` +Object { + "dirs": Array [ + "foo", + "bar", + ], + "filename": "baz.json", + "root": "c:", +} +`; diff --git a/packages/kbn-optimizer/src/worker/parse_path.test.ts b/packages/kbn-optimizer/src/worker/parse_path.test.ts new file mode 100644 index 0000000000000..72197e8c8fb07 --- /dev/null +++ b/packages/kbn-optimizer/src/worker/parse_path.test.ts @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { parseFilePath, parseDirPath } from './parse_path'; + +const DIRS = ['/', '/foo/bar/baz/', 'c:\\', 'c:\\foo\\bar\\baz\\']; +const AMBIGUOUS = ['/foo', '/foo/bar/baz', 'c:\\foo', 'c:\\foo\\bar\\baz']; +const FILES = ['/foo/bar/baz.json', 'c:/foo/bar/baz.json', 'c:\\foo\\bar\\baz.json']; + +describe('parseFilePath()', () => { + it.each([...FILES, ...AMBIGUOUS])('parses %s', path => { + expect(parseFilePath(path)).toMatchSnapshot(); + }); +}); + +describe('parseDirPath()', () => { + it.each([...DIRS, ...AMBIGUOUS])('parses %s', path => { + expect(parseDirPath(path)).toMatchSnapshot(); + }); +}); diff --git a/packages/kbn-optimizer/src/worker/parse_path.ts b/packages/kbn-optimizer/src/worker/parse_path.ts new file mode 100644 index 0000000000000..88152df55b84f --- /dev/null +++ b/packages/kbn-optimizer/src/worker/parse_path.ts @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import normalizePath from 'normalize-path'; + +/** + * Parse an absolute path, supporting normalized paths from webpack, + * into a list of directories and root + */ +export function parseDirPath(path: string) { + const filePath = parseFilePath(path); + return { + ...filePath, + dirs: [...filePath.dirs, ...(filePath.filename ? [filePath.filename] : [])], + filename: undefined, + }; +} + +export function parseFilePath(path: string) { + const normalized = normalizePath(path); + const [root, ...others] = normalized.split('/'); + return { + root: root === '' ? '/' : root, + dirs: others.slice(0, -1), + filename: others[others.length - 1] || undefined, + }; +} diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index 7dcce8a0fae8d..7a8097fd2b2c7 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -27,9 +27,10 @@ import webpack, { Stats } from 'webpack'; import * as Rx from 'rxjs'; import { mergeMap, map, mapTo, takeUntil } from 'rxjs/operators'; -import { CompilerMsgs, CompilerMsg, maybeMap, Bundle, WorkerConfig } from '../common'; +import { CompilerMsgs, CompilerMsg, maybeMap, Bundle, WorkerConfig, ascending } from '../common'; import { getWebpackConfig } from './webpack.config'; import { isFailureStats, failedStatsToErrorMessage } from './webpack_helpers'; +import { parseFilePath } from './parse_path'; import { isExternalModule, isNormalModule, @@ -108,20 +109,19 @@ const observeCompiler = ( for (const module of normalModules) { const path = getModulePath(module); + const parsedPath = parseFilePath(path); - const parsedPath = Path.parse(path); - const dirSegments = parsedPath.dir.split(Path.sep); - if (!dirSegments.includes('node_modules')) { + if (!parsedPath.dirs.includes('node_modules')) { referencedFiles.add(path); continue; } - const nmIndex = dirSegments.lastIndexOf('node_modules'); - const isScoped = dirSegments[nmIndex + 1].startsWith('@'); + const nmIndex = parsedPath.dirs.lastIndexOf('node_modules'); + const isScoped = parsedPath.dirs[nmIndex + 1].startsWith('@'); referencedFiles.add( Path.join( parsedPath.root, - ...dirSegments.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), + ...parsedPath.dirs.slice(0, nmIndex + 1 + (isScoped ? 2 : 1)), 'package.json' ) ); @@ -146,7 +146,7 @@ const observeCompiler = ( optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), moduleCount: normalModules.length, - files, + files: files.sort(ascending(f => f)), }); return compilerMsgs.compilerSuccess({ diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 3c6ae78bc4d91..5d8ef7626f630 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -30,6 +30,7 @@ import { CleanWebpackPlugin } from 'clean-webpack-plugin'; import * as SharedDeps from '@kbn/ui-shared-deps'; import { Bundle, WorkerConfig } from '../common'; +import { parseDirPath } from './parse_path'; const IS_CODE_COVERAGE = !!process.env.CODE_COVERAGE; const ISTANBUL_PRESET_PATH = require.resolve('@kbn/babel-preset/istanbul_preset'); @@ -135,7 +136,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { } // manually force ui/* urls in legacy styles to resolve to ui/legacy/public - if (uri.startsWith('ui/') && base.split(Path.sep).includes('legacy')) { + if (uri.startsWith('ui/') && parseDirPath(base).dirs.includes('legacy')) { return Path.resolve( worker.repoRoot, 'src/legacy/ui/public', @@ -150,7 +151,9 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) { { loader: 'sass-loader', options: { - sourceMap: !worker.dist, + // must always be enabled as long as we're using the `resolve-url-loader` to + // rewrite `ui/*` urls. They're dropped by subsequent loaders though + sourceMap: true, prependData(loaderContext: webpack.loader.LoaderContext) { return `@import ${stringifyRequest( loaderContext, diff --git a/packages/kbn-plugin-generator/integration_tests/generate_plugin.test.js b/packages/kbn-plugin-generator/integration_tests/generate_plugin.test.js index 771bf43c4020a..d7d4dc14519c3 100644 --- a/packages/kbn-plugin-generator/integration_tests/generate_plugin.test.js +++ b/packages/kbn-plugin-generator/integration_tests/generate_plugin.test.js @@ -102,6 +102,7 @@ describe(`running the plugin-generator via 'node scripts/generate_plugin.js plug 'start', '--optimize.enabled=false', '--logging.json=false', + '--logging.verbose=true', '--migrations.skip=true', ], cwd: generatedPath, diff --git a/packages/kbn-test/src/junit_report_path.ts b/packages/kbn-test/src/junit_report_path.ts index 11eaf3d2b14a5..d46c9455dcff0 100644 --- a/packages/kbn-test/src/junit_report_path.ts +++ b/packages/kbn-test/src/junit_report_path.ts @@ -20,7 +20,9 @@ import { resolve } from 'path'; const job = process.env.JOB ? `job-${process.env.JOB}-` : ''; -const num = process.env.CI_WORKER_NUMBER ? `worker-${process.env.CI_WORKER_NUMBER}-` : ''; +const num = process.env.CI_PARALLEL_PROCESS_NUMBER + ? `worker-${process.env.CI_PARALLEL_PROCESS_NUMBER}-` + : ''; export function makeJunitReportPath(rootDirectory: string, reportName: string) { return resolve( diff --git a/renovate.json5 b/renovate.json5 index 58a64a5d0f967..ca2cd2e6bcd93 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -665,6 +665,14 @@ '@types/nodemailer', ], }, + { + groupSlug: 'normalize-path', + groupName: 'normalize-path related packages', + packageNames: [ + 'normalize-path', + '@types/normalize-path', + ], + }, { groupSlug: 'numeral', groupName: 'numeral related packages', diff --git a/src/core/CONVENTIONS.md b/src/core/CONVENTIONS.md index 2769079757bc3..0f592d108c561 100644 --- a/src/core/CONVENTIONS.md +++ b/src/core/CONVENTIONS.md @@ -148,8 +148,8 @@ import { MyAppRoot } from './components/app.ts'; /** * This module will be loaded asynchronously to reduce the bundle size of your plugin's main bundle. */ -export const renderApp = (core: CoreStart, deps: MyPluginDepsStart, { element, appBasePath }: AppMountParams) => { - ReactDOM.render(, element); +export const renderApp = (core: CoreStart, deps: MyPluginDepsStart, { element, history }: AppMountParams) => { + ReactDOM.render(, element); return () => ReactDOM.unmountComponentAtNode(element); } ``` diff --git a/src/core/TESTING.md b/src/core/TESTING.md index 9abc2bb77d7d1..cb38dac0e20ce 100644 --- a/src/core/TESTING.md +++ b/src/core/TESTING.md @@ -453,7 +453,7 @@ describe('Plugin', () => { const [coreStartMock, startDepsMock] = await coreSetup.getStartServices(); const unmountMock = jest.fn(); renderAppMock.mockReturnValue(unmountMock); - const params = { element: document.createElement('div'), appBasePath: '/fake/base/path' }; + const params = coreMock.createAppMountParamters('/fake/base/path'); new Plugin(coreMock.createPluginInitializerContext()).setup(coreSetup); // Grab registered mount function @@ -478,7 +478,7 @@ import ReactDOM from 'react-dom'; import { AppMountParams, CoreStart } from 'src/core/public'; import { AppRoot } from './components/app_root'; -export const renderApp = ({ element, appBasePath }: AppMountParams, core: CoreStart, plugins: MyPluginDepsStart) => { +export const renderApp = ({ element, history }: AppMountParams, core: CoreStart, plugins: MyPluginDepsStart) => { // Hide the chrome while this app is mounted for a full screen experience core.chrome.setIsVisible(false); @@ -491,7 +491,7 @@ export const renderApp = ({ element, appBasePath }: AppMountParams, core: CoreSt // Render app ReactDOM.render( - , + , element ); @@ -512,12 +512,14 @@ In testing `renderApp` you should be verifying that: ```typescript /** public/application.test.ts */ +import { createMemoryHistory } from 'history'; +import { ScopedHistory } from 'src/core/public'; import { coreMock } from 'src/core/public/mocks'; import { renderApp } from './application'; describe('renderApp', () => { it('mounts and unmounts UI', () => { - const params = { element: document.createElement('div'), appBasePath: '/fake/base/path' }; + const params = coreMock.createAppMountParamters('/fake/base/path'); const core = coreMock.createStart(); // Verify some expected DOM element is rendered into the element @@ -529,7 +531,7 @@ describe('renderApp', () => { }); it('unsubscribes from uiSettings', () => { - const params = { element: document.createElement('div'), appBasePath: '/fake/base/path' }; + const params = coreMock.createAppMountParamters('/fake/base/path'); const core = coreMock.createStart(); // Create a fake Subject you can use to monitor observers const settings$ = new Subject(); @@ -544,7 +546,7 @@ describe('renderApp', () => { }); it('resets chrome visibility', () => { - const params = { element: document.createElement('div'), appBasePath: '/fake/base/path' }; + const params = coreMock.createAppMountParamters('/fake/base/path'); const core = coreMock.createStart(); // Verify stateful Core API was called on mount diff --git a/src/core/public/application/types.ts b/src/core/public/application/types.ts index facb818c60ff9..318afb652999e 100644 --- a/src/core/public/application/types.ts +++ b/src/core/public/application/types.ts @@ -347,7 +347,6 @@ export interface AppMountParameters { * * export renderApp = ({ element, history }: AppMountParameters) => { * ReactDOM.render( - * // pass `appBasePath` to `basename` * * * , @@ -429,7 +428,7 @@ export interface AppMountParameters { * import { CoreStart, AppMountParams } from 'src/core/public'; * import { MyPluginDepsStart } from './plugin'; * - * export renderApp = ({ appBasePath, element, onAppLeave }: AppMountParams) => { + * export renderApp = ({ element, history, onAppLeave }: AppMountParams) => { * const { renderApp, hasUnsavedChanges } = await import('./application'); * onAppLeave(actions => { * if(hasUnsavedChanges()) { @@ -437,7 +436,7 @@ export interface AppMountParameters { * } * return actions.default(); * }); - * return renderApp(params); + * return renderApp({ element, history }); * } * ``` */ diff --git a/src/core/public/mocks.ts b/src/core/public/mocks.ts index 8ea672890ca29..c860e9de8334e 100644 --- a/src/core/public/mocks.ts +++ b/src/core/public/mocks.ts @@ -16,9 +16,15 @@ * specific language governing permissions and limitations * under the License. */ +import { createMemoryHistory } from 'history'; + +// Only import types from '.' to avoid triggering default Jest mocks. +import { CoreContext, PluginInitializerContext, AppMountParameters } from '.'; +// Import values from their individual modules instead. +import { ScopedHistory } from './application'; + import { applicationServiceMock } from './application/application_service.mock'; import { chromeServiceMock } from './chrome/chrome_service.mock'; -import { CoreContext, PluginInitializerContext } from '.'; import { docLinksServiceMock } from './doc_links/doc_links_service.mock'; import { fatalErrorsServiceMock } from './fatal_errors/fatal_errors_service.mock'; import { httpServiceMock } from './http/http_service.mock'; @@ -139,10 +145,27 @@ function createStorageMock() { return storageMock; } +function createAppMountParametersMock(appBasePath = '') { + // Assemble an in-memory history mock using the provided basePath + const rawHistory = createMemoryHistory(); + rawHistory.push(appBasePath); + const history = new ScopedHistory(rawHistory, appBasePath); + + const params: jest.Mocked = { + appBasePath, + element: document.createElement('div'), + history, + onAppLeave: jest.fn(), + }; + + return params; +} + export const coreMock = { createCoreContext, createSetup: createCoreSetupMock, createStart: createCoreStartMock, createPluginInitializerContext: pluginInitializerContextMock, createStorage: createStorageMock, + createAppMountParamters: createAppMountParametersMock, }; diff --git a/src/core/server/elasticsearch/retry_call_cluster.test.ts b/src/core/server/elasticsearch/retry_call_cluster.test.ts index b5a5185ab39d9..4f391f0aba34b 100644 --- a/src/core/server/elasticsearch/retry_call_cluster.test.ts +++ b/src/core/server/elasticsearch/retry_call_cluster.test.ts @@ -89,6 +89,19 @@ describe('migrationsRetryCallCluster', () => { }); }); + it('retries ES API calls that rejects with snapshot_in_progress_exception', () => { + expect.assertions(1); + const callEsApi = jest.fn(); + let i = 0; + callEsApi.mockImplementation(() => { + return i++ <= 2 + ? Promise.reject({ body: { error: { type: 'snapshot_in_progress_exception' } } }) + : Promise.resolve('success'); + }); + const retried = migrationsRetryCallCluster(callEsApi, mockLogger.get('mock log'), 1); + return expect(retried('endpoint')).resolves.toMatchInlineSnapshot(`"success"`); + }); + it('rejects when ES API calls reject with other errors', async () => { expect.assertions(3); const callEsApi = jest.fn(); diff --git a/src/core/server/elasticsearch/retry_call_cluster.ts b/src/core/server/elasticsearch/retry_call_cluster.ts index ea3cc0b90c077..901b801159cb6 100644 --- a/src/core/server/elasticsearch/retry_call_cluster.ts +++ b/src/core/server/elasticsearch/retry_call_cluster.ts @@ -64,7 +64,8 @@ export function migrationsRetryCallCluster( error instanceof esErrors.AuthenticationException || error instanceof esErrors.AuthorizationException || // @ts-ignore - error instanceof esErrors.Gone + error instanceof esErrors.Gone || + error?.body?.error?.type === 'snapshot_in_progress_exception' ); }, timer(delay), @@ -85,15 +86,7 @@ export function migrationsRetryCallCluster( * * @param apiCaller */ - -// TODO: Replace with APICaller from './scoped_cluster_client' once #46668 is merged -export function retryCallCluster( - apiCaller: ( - endpoint: string, - clientParams: Record, - options?: CallAPIOptions - ) => Promise -) { +export function retryCallCluster(apiCaller: APICaller) { return (endpoint: string, clientParams: Record = {}, options?: CallAPIOptions) => { return defer(() => apiCaller(endpoint, clientParams, options)) .pipe( diff --git a/src/core/server/index.ts b/src/core/server/index.ts index e45d4f28edcc3..de6cdb2d7acd7 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -245,6 +245,14 @@ export { StringValidationRegexString, } from './ui_settings'; +export { + OpsMetrics, + OpsOsMetrics, + OpsServerMetrics, + OpsProcessMetrics, + MetricsServiceSetup, +} from './metrics'; + export { RecursiveReadonly } from '../utils'; export { diff --git a/src/core/server/internal_types.ts b/src/core/server/internal_types.ts index ff68d1544d119..37d1061dc618d 100644 --- a/src/core/server/internal_types.ts +++ b/src/core/server/internal_types.ts @@ -30,6 +30,7 @@ import { } from './saved_objects'; import { InternalUiSettingsServiceSetup, InternalUiSettingsServiceStart } from './ui_settings'; import { UuidServiceSetup } from './uuid'; +import { InternalMetricsServiceSetup } from './metrics'; /** @internal */ export interface InternalCoreSetup { @@ -40,6 +41,7 @@ export interface InternalCoreSetup { uiSettings: InternalUiSettingsServiceSetup; savedObjects: InternalSavedObjectsServiceSetup; uuid: UuidServiceSetup; + metrics: InternalMetricsServiceSetup; } /** diff --git a/src/core/server/legacy/legacy_service.test.ts b/src/core/server/legacy/legacy_service.test.ts index 46436461505c0..50468db8a504d 100644 --- a/src/core/server/legacy/legacy_service.test.ts +++ b/src/core/server/legacy/legacy_service.test.ts @@ -43,6 +43,7 @@ import { savedObjectsServiceMock } from '../saved_objects/saved_objects_service. import { capabilitiesServiceMock } from '../capabilities/capabilities_service.mock'; import { setupMock as renderingServiceMock } from '../rendering/__mocks__/rendering_service'; import { uuidServiceMock } from '../uuid/uuid_service.mock'; +import { metricsServiceMock } from '../metrics/metrics_service.mock'; import { findLegacyPluginSpecs } from './plugins'; import { LegacyVars, LegacyServiceSetupDeps, LegacyServiceStartDeps } from './types'; import { LegacyService } from './legacy_service'; @@ -93,6 +94,7 @@ beforeEach(() => { }, }, rendering: renderingServiceMock, + metrics: metricsServiceMock.createInternalSetupContract(), uuid: uuidSetup, }, plugins: { 'plugin-id': 'plugin-value' }, diff --git a/src/core/server/metrics/collectors/index.ts b/src/core/server/metrics/collectors/index.ts new file mode 100644 index 0000000000000..f58ab02e63881 --- /dev/null +++ b/src/core/server/metrics/collectors/index.ts @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { OpsProcessMetrics, OpsOsMetrics, OpsServerMetrics, MetricsCollector } from './types'; +export { OsMetricsCollector } from './os'; +export { ProcessMetricsCollector } from './process'; +export { ServerMetricsCollector } from './server'; diff --git a/src/core/server/metrics/collectors/os.test.ts b/src/core/server/metrics/collectors/os.test.ts new file mode 100644 index 0000000000000..7d5a6da90b7d6 --- /dev/null +++ b/src/core/server/metrics/collectors/os.test.ts @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +jest.mock('getos', () => (cb: Function) => cb(null, { dist: 'distrib', release: 'release' })); + +import os from 'os'; +import { OsMetricsCollector } from './os'; + +describe('OsMetricsCollector', () => { + let collector: OsMetricsCollector; + + beforeEach(() => { + collector = new OsMetricsCollector(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('collects platform info from the os package', async () => { + const platform = 'darwin'; + const release = '10.14.1'; + + jest.spyOn(os, 'platform').mockImplementation(() => platform); + jest.spyOn(os, 'release').mockImplementation(() => release); + + const metrics = await collector.collect(); + + expect(metrics.platform).toBe(platform); + expect(metrics.platformRelease).toBe(`${platform}-${release}`); + }); + + it('collects distribution info when platform is linux', async () => { + const platform = 'linux'; + + jest.spyOn(os, 'platform').mockImplementation(() => platform); + + const metrics = await collector.collect(); + + expect(metrics.distro).toBe('distrib'); + expect(metrics.distroRelease).toBe('distrib-release'); + }); + + it('collects memory info from the os package', async () => { + const totalMemory = 1457886; + const freeMemory = 456786; + + jest.spyOn(os, 'totalmem').mockImplementation(() => totalMemory); + jest.spyOn(os, 'freemem').mockImplementation(() => freeMemory); + + const metrics = await collector.collect(); + + expect(metrics.memory.total_in_bytes).toBe(totalMemory); + expect(metrics.memory.free_in_bytes).toBe(freeMemory); + expect(metrics.memory.used_in_bytes).toBe(totalMemory - freeMemory); + }); + + it('collects uptime info from the os package', async () => { + const uptime = 325; + + jest.spyOn(os, 'uptime').mockImplementation(() => uptime); + + const metrics = await collector.collect(); + + expect(metrics.uptime_in_millis).toBe(uptime * 1000); + }); + + it('collects load info from the os package', async () => { + const oneMinLoad = 1; + const fiveMinLoad = 2; + const fifteenMinLoad = 3; + + jest.spyOn(os, 'loadavg').mockImplementation(() => [oneMinLoad, fiveMinLoad, fifteenMinLoad]); + + const metrics = await collector.collect(); + + expect(metrics.load).toEqual({ + '1m': oneMinLoad, + '5m': fiveMinLoad, + '15m': fifteenMinLoad, + }); + }); +}); diff --git a/src/core/server/metrics/collectors/os.ts b/src/core/server/metrics/collectors/os.ts new file mode 100644 index 0000000000000..d3d9bb0be86fa --- /dev/null +++ b/src/core/server/metrics/collectors/os.ts @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import os from 'os'; +import getosAsync, { LinuxOs } from 'getos'; +import { promisify } from 'util'; +import { OpsOsMetrics, MetricsCollector } from './types'; + +const getos = promisify(getosAsync); + +export class OsMetricsCollector implements MetricsCollector { + public async collect(): Promise { + const platform = os.platform(); + const load = os.loadavg(); + + const metrics: OpsOsMetrics = { + platform, + platformRelease: `${platform}-${os.release()}`, + load: { + '1m': load[0], + '5m': load[1], + '15m': load[2], + }, + memory: { + total_in_bytes: os.totalmem(), + free_in_bytes: os.freemem(), + used_in_bytes: os.totalmem() - os.freemem(), + }, + uptime_in_millis: os.uptime() * 1000, + }; + + if (platform === 'linux') { + try { + const distro = (await getos()) as LinuxOs; + metrics.distro = distro.dist; + metrics.distroRelease = `${distro.dist}-${distro.release}`; + } catch (e) { + // ignore errors + } + } + + return metrics; + } +} diff --git a/src/core/server/metrics/collectors/process.test.ts b/src/core/server/metrics/collectors/process.test.ts new file mode 100644 index 0000000000000..a437d799371f1 --- /dev/null +++ b/src/core/server/metrics/collectors/process.test.ts @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import v8, { HeapInfo } from 'v8'; +import { ProcessMetricsCollector } from './process'; + +describe('ProcessMetricsCollector', () => { + let collector: ProcessMetricsCollector; + + beforeEach(() => { + collector = new ProcessMetricsCollector(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('collects pid from the process', async () => { + const metrics = await collector.collect(); + + expect(metrics.pid).toEqual(process.pid); + }); + + it('collects event loop delay', async () => { + const metrics = await collector.collect(); + + expect(metrics.event_loop_delay).toBeGreaterThan(0); + }); + + it('collects uptime info from the process', async () => { + const uptime = 58986; + jest.spyOn(process, 'uptime').mockImplementation(() => uptime); + + const metrics = await collector.collect(); + + expect(metrics.uptime_in_millis).toEqual(uptime * 1000); + }); + + it('collects memory info from the process', async () => { + const heapTotal = 58986; + const heapUsed = 4688; + const heapSizeLimit = 5788; + const rss = 5865; + jest.spyOn(process, 'memoryUsage').mockImplementation(() => ({ + rss, + heapTotal, + heapUsed, + external: 0, + })); + + jest.spyOn(v8, 'getHeapStatistics').mockImplementation( + () => + ({ + heap_size_limit: heapSizeLimit, + } as HeapInfo) + ); + + const metrics = await collector.collect(); + + expect(metrics.memory.heap.total_in_bytes).toEqual(heapTotal); + expect(metrics.memory.heap.used_in_bytes).toEqual(heapUsed); + expect(metrics.memory.heap.size_limit).toEqual(heapSizeLimit); + expect(metrics.memory.resident_set_size_in_bytes).toEqual(rss); + }); +}); diff --git a/src/core/server/metrics/collectors/process.ts b/src/core/server/metrics/collectors/process.ts new file mode 100644 index 0000000000000..aa68abaf74e41 --- /dev/null +++ b/src/core/server/metrics/collectors/process.ts @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import v8 from 'v8'; +import { Bench } from 'hoek'; +import { OpsProcessMetrics, MetricsCollector } from './types'; + +export class ProcessMetricsCollector implements MetricsCollector { + public async collect(): Promise { + const heapStats = v8.getHeapStatistics(); + const memoryUsage = process.memoryUsage(); + const [eventLoopDelay] = await Promise.all([getEventLoopDelay()]); + return { + memory: { + heap: { + total_in_bytes: memoryUsage.heapTotal, + used_in_bytes: memoryUsage.heapUsed, + size_limit: heapStats.heap_size_limit, + }, + resident_set_size_in_bytes: memoryUsage.rss, + }, + pid: process.pid, + event_loop_delay: eventLoopDelay, + uptime_in_millis: process.uptime() * 1000, + }; + } +} + +const getEventLoopDelay = (): Promise => { + const bench = new Bench(); + return new Promise(resolve => { + setImmediate(() => { + return resolve(bench.elapsed()); + }); + }); +}; diff --git a/src/core/server/metrics/collectors/server.ts b/src/core/server/metrics/collectors/server.ts new file mode 100644 index 0000000000000..e46ac2f653df6 --- /dev/null +++ b/src/core/server/metrics/collectors/server.ts @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { ResponseObject, Server as HapiServer } from 'hapi'; +import { OpsServerMetrics, MetricsCollector } from './types'; + +interface ServerResponseTime { + count: number; + total: number; + max: number; +} + +export class ServerMetricsCollector implements MetricsCollector { + private readonly requests: OpsServerMetrics['requests'] = { + disconnects: 0, + total: 0, + statusCodes: {}, + }; + private readonly responseTimes: ServerResponseTime = { + count: 0, + total: 0, + max: 0, + }; + + constructor(private readonly server: HapiServer) { + this.server.ext('onRequest', (request, h) => { + this.requests.total++; + request.events.once('disconnect', () => { + this.requests.disconnects++; + }); + return h.continue; + }); + this.server.events.on('response', request => { + const statusCode = (request.response as ResponseObject)?.statusCode; + if (statusCode) { + if (!this.requests.statusCodes[statusCode]) { + this.requests.statusCodes[statusCode] = 0; + } + this.requests.statusCodes[statusCode]++; + } + + const duration = Date.now() - request.info.received; + this.responseTimes.count++; + this.responseTimes.total += duration; + this.responseTimes.max = Math.max(this.responseTimes.max, duration); + }); + } + + public async collect(): Promise { + const connections = await new Promise(resolve => { + this.server.listener.getConnections((_, count) => { + resolve(count); + }); + }); + + return { + requests: this.requests, + response_times: { + avg_in_millis: this.responseTimes.total / Math.max(this.responseTimes.count, 1), + max_in_millis: this.responseTimes.max, + }, + concurrent_connections: connections, + }; + } +} diff --git a/src/core/server/metrics/collectors/types.ts b/src/core/server/metrics/collectors/types.ts new file mode 100644 index 0000000000000..5a83bc70af3c1 --- /dev/null +++ b/src/core/server/metrics/collectors/types.ts @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** Base interface for all metrics gatherers */ +export interface MetricsCollector { + collect(): Promise; +} + +/** + * Process related metrics + * @public + */ +export interface OpsProcessMetrics { + /** process memory usage */ + memory: { + /** heap memory usage */ + heap: { + /** total heap available */ + total_in_bytes: number; + /** used heap */ + used_in_bytes: number; + /** v8 heap size limit */ + size_limit: number; + }; + /** node rss */ + resident_set_size_in_bytes: number; + }; + /** node event loop delay */ + event_loop_delay: number; + /** pid of the kibana process */ + pid: number; + /** uptime of the kibana process */ + uptime_in_millis: number; +} + +/** + * OS related metrics + * @public + */ +export interface OpsOsMetrics { + /** The os platform */ + platform: NodeJS.Platform; + /** The os platform release, prefixed by the platform name */ + platformRelease: string; + /** The os distrib. Only present for linux platforms */ + distro?: string; + /** The os distrib release, prefixed by the os distrib. Only present for linux platforms */ + distroRelease?: string; + /** cpu load metrics */ + load: { + /** load for last minute */ + '1m': number; + /** load for last 5 minutes */ + '5m': number; + /** load for last 15 minutes */ + '15m': number; + }; + /** system memory usage metrics */ + memory: { + /** total memory available */ + total_in_bytes: number; + /** current free memory */ + free_in_bytes: number; + /** current used memory */ + used_in_bytes: number; + }; + /** the OS uptime */ + uptime_in_millis: number; +} + +/** + * server related metrics + * @public + */ +export interface OpsServerMetrics { + /** server response time stats */ + response_times: { + /** average response time */ + avg_in_millis: number; + /** maximum response time */ + max_in_millis: number; + }; + /** server requests stats */ + requests: { + /** number of disconnected requests since startup */ + disconnects: number; + /** total number of requests handled since startup */ + total: number; + /** number of request handled per response status code */ + statusCodes: Record; + }; + /** number of current concurrent connections to the server */ + concurrent_connections: number; +} diff --git a/src/core/server/metrics/index.ts b/src/core/server/metrics/index.ts new file mode 100644 index 0000000000000..fdcf637c0cd7b --- /dev/null +++ b/src/core/server/metrics/index.ts @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { + InternalMetricsServiceStart, + InternalMetricsServiceSetup, + MetricsServiceSetup, + MetricsServiceStart, + OpsMetrics, +} from './types'; +export { OpsProcessMetrics, OpsServerMetrics, OpsOsMetrics } from './collectors'; +export { MetricsService } from './metrics_service'; +export { opsConfig } from './ops_config'; diff --git a/src/core/server/metrics/integration_tests/server_collector.test.ts b/src/core/server/metrics/integration_tests/server_collector.test.ts new file mode 100644 index 0000000000000..a387de80212d9 --- /dev/null +++ b/src/core/server/metrics/integration_tests/server_collector.test.ts @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Subject } from 'rxjs'; +import { take } from 'rxjs/operators'; +import supertest from 'supertest'; +import { Server as HapiServer } from 'hapi'; +import { createHttpServer } from '../../http/test_utils'; +import { HttpService, IRouter } from '../../http'; +import { contextServiceMock } from '../../context/context_service.mock'; +import { ServerMetricsCollector } from '../collectors/server'; + +describe('ServerMetricsCollector', () => { + let server: HttpService; + let collector: ServerMetricsCollector; + let hapiServer: HapiServer; + let router: IRouter; + + const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + const sendGet = (path: string) => supertest(hapiServer.listener).get(path); + + beforeEach(async () => { + server = createHttpServer(); + const contextSetup = contextServiceMock.createSetupContract(); + const httpSetup = await server.setup({ context: contextSetup }); + hapiServer = httpSetup.server; + router = httpSetup.createRouter('/'); + collector = new ServerMetricsCollector(hapiServer); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('collect requests infos', async () => { + router.get({ path: '/', validate: false }, async (ctx, req, res) => { + return res.ok({ body: '' }); + }); + await server.start(); + + let metrics = await collector.collect(); + + expect(metrics.requests).toEqual({ + total: 0, + disconnects: 0, + statusCodes: {}, + }); + + await sendGet('/'); + await sendGet('/'); + await sendGet('/not-found'); + + metrics = await collector.collect(); + + expect(metrics.requests).toEqual({ + total: 3, + disconnects: 0, + statusCodes: { + '200': 2, + '404': 1, + }, + }); + }); + + it('collect disconnects requests infos', async () => { + const never = new Promise(resolve => undefined); + + router.get({ path: '/', validate: false }, async (ctx, req, res) => { + return res.ok({ body: '' }); + }); + router.get({ path: '/disconnect', validate: false }, async (ctx, req, res) => { + await never; + return res.ok({ body: '' }); + }); + await server.start(); + + await sendGet('/'); + const discoReq1 = sendGet('/disconnect').end(); + const discoReq2 = sendGet('/disconnect').end(); + await delay(20); + + let metrics = await collector.collect(); + expect(metrics.requests).toEqual( + expect.objectContaining({ + total: 3, + disconnects: 0, + }) + ); + + discoReq1.abort(); + await delay(20); + + metrics = await collector.collect(); + expect(metrics.requests).toEqual( + expect.objectContaining({ + total: 3, + disconnects: 1, + }) + ); + + discoReq2.abort(); + await delay(20); + + metrics = await collector.collect(); + expect(metrics.requests).toEqual( + expect.objectContaining({ + total: 3, + disconnects: 2, + }) + ); + }); + + it('collect response times', async () => { + router.get({ path: '/no-delay', validate: false }, async (ctx, req, res) => { + return res.ok({ body: '' }); + }); + router.get({ path: '/500-ms', validate: false }, async (ctx, req, res) => { + await delay(500); + return res.ok({ body: '' }); + }); + router.get({ path: '/250-ms', validate: false }, async (ctx, req, res) => { + await delay(250); + return res.ok({ body: '' }); + }); + await server.start(); + + await Promise.all([sendGet('/no-delay'), sendGet('/250-ms')]); + let metrics = await collector.collect(); + + expect(metrics.response_times.avg_in_millis).toBeGreaterThanOrEqual(125); + expect(metrics.response_times.max_in_millis).toBeGreaterThanOrEqual(250); + + await Promise.all([sendGet('/500-ms'), sendGet('/500-ms')]); + metrics = await collector.collect(); + + expect(metrics.response_times.avg_in_millis).toBeGreaterThanOrEqual(250); + expect(metrics.response_times.max_in_millis).toBeGreaterThanOrEqual(500); + }); + + it('collect connection count', async () => { + const waitSubject = new Subject(); + + router.get({ path: '/', validate: false }, async (ctx, req, res) => { + await waitSubject.pipe(take(1)).toPromise(); + return res.ok({ body: '' }); + }); + await server.start(); + + let metrics = await collector.collect(); + expect(metrics.concurrent_connections).toEqual(0); + + sendGet('/').end(() => null); + await delay(20); + metrics = await collector.collect(); + expect(metrics.concurrent_connections).toEqual(1); + + sendGet('/').end(() => null); + await delay(20); + metrics = await collector.collect(); + expect(metrics.concurrent_connections).toEqual(2); + + waitSubject.next('go'); + await delay(20); + metrics = await collector.collect(); + expect(metrics.concurrent_connections).toEqual(0); + }); +}); diff --git a/src/core/server/metrics/metrics_service.mock.ts b/src/core/server/metrics/metrics_service.mock.ts new file mode 100644 index 0000000000000..cc53a4e27d571 --- /dev/null +++ b/src/core/server/metrics/metrics_service.mock.ts @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { MetricsService } from './metrics_service'; +import { + InternalMetricsServiceSetup, + InternalMetricsServiceStart, + MetricsServiceSetup, + MetricsServiceStart, +} from './types'; + +const createSetupContractMock = () => { + const setupContract: jest.Mocked = { + getOpsMetrics$: jest.fn(), + }; + return setupContract; +}; + +const createInternalSetupContractMock = () => { + const setupContract: jest.Mocked = createSetupContractMock(); + return setupContract; +}; + +const createStartContractMock = () => { + const startContract: jest.Mocked = {}; + return startContract; +}; + +const createInternalStartContractMock = () => { + const startContract: jest.Mocked = createStartContractMock(); + return startContract; +}; + +type MetricsServiceContract = PublicMethodsOf; + +const createMock = () => { + const mocked: jest.Mocked = { + setup: jest.fn().mockReturnValue(createInternalSetupContractMock()), + start: jest.fn().mockReturnValue(createInternalStartContractMock()), + stop: jest.fn(), + }; + return mocked; +}; + +export const metricsServiceMock = { + create: createMock, + createSetupContract: createSetupContractMock, + createStartContract: createStartContractMock, + createInternalSetupContract: createInternalSetupContractMock, + createInternalStartContract: createInternalStartContractMock, +}; diff --git a/typings/normalize_path/index.d.ts b/src/core/server/metrics/metrics_service.test.mocks.ts similarity index 80% rename from typings/normalize_path/index.d.ts rename to src/core/server/metrics/metrics_service.test.mocks.ts index 31e064ca63d90..8e91775283042 100644 --- a/typings/normalize_path/index.d.ts +++ b/src/core/server/metrics/metrics_service.test.mocks.ts @@ -17,8 +17,9 @@ * under the License. */ -declare function NormalizePath(path: string, stripTrailing?: boolean): string; - -declare module 'normalize-path' { - export = NormalizePath; -} +export const mockOpsCollector = { + collect: jest.fn(), +}; +jest.doMock('./ops_metrics_collector', () => ({ + OpsMetricsCollector: jest.fn().mockImplementation(() => mockOpsCollector), +})); diff --git a/src/core/server/metrics/metrics_service.test.ts b/src/core/server/metrics/metrics_service.test.ts new file mode 100644 index 0000000000000..10d6761adbe7d --- /dev/null +++ b/src/core/server/metrics/metrics_service.test.ts @@ -0,0 +1,134 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import moment from 'moment'; +import { mockOpsCollector } from './metrics_service.test.mocks'; +import { MetricsService } from './metrics_service'; +import { mockCoreContext } from '../core_context.mock'; +import { configServiceMock } from '../config/config_service.mock'; +import { httpServiceMock } from '../http/http_service.mock'; +import { take } from 'rxjs/operators'; + +const testInterval = 100; + +const dummyMetrics = { metricA: 'value', metricB: 'otherValue' }; + +describe('MetricsService', () => { + const httpMock = httpServiceMock.createSetupContract(); + let metricsService: MetricsService; + + beforeEach(() => { + jest.useFakeTimers(); + + const configService = configServiceMock.create({ + atPath: { interval: moment.duration(testInterval) }, + }); + const coreContext = mockCoreContext.create({ configService }); + metricsService = new MetricsService(coreContext); + }); + + afterEach(() => { + jest.clearAllMocks(); + jest.clearAllTimers(); + }); + + describe('#start', () => { + it('invokes setInterval with the configured interval', async () => { + await metricsService.setup({ http: httpMock }); + await metricsService.start(); + + expect(setInterval).toHaveBeenCalledTimes(1); + expect(setInterval).toHaveBeenCalledWith(expect.any(Function), testInterval); + }); + + it('emits the metrics at start', async () => { + mockOpsCollector.collect.mockResolvedValue(dummyMetrics); + + const { getOpsMetrics$ } = await metricsService.setup({ + http: httpMock, + }); + + await metricsService.start(); + + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(1); + expect( + await getOpsMetrics$() + .pipe(take(1)) + .toPromise() + ).toEqual(dummyMetrics); + }); + + it('collects the metrics at every interval', async () => { + mockOpsCollector.collect.mockResolvedValue(dummyMetrics); + + await metricsService.setup({ http: httpMock }); + + await metricsService.start(); + + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(testInterval); + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(2); + + jest.advanceTimersByTime(testInterval); + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(3); + }); + + it('throws when called before setup', async () => { + await expect(metricsService.start()).rejects.toThrowErrorMatchingInlineSnapshot( + `"#setup() needs to be run first"` + ); + }); + }); + + describe('#stop', () => { + it('stops the metrics interval', async () => { + const { getOpsMetrics$ } = await metricsService.setup({ http: httpMock }); + await metricsService.start(); + + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(testInterval); + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(2); + + await metricsService.stop(); + jest.advanceTimersByTime(10 * testInterval); + expect(mockOpsCollector.collect).toHaveBeenCalledTimes(2); + + getOpsMetrics$().subscribe({ complete: () => {} }); + }); + + it('completes the metrics observable', async () => { + const { getOpsMetrics$ } = await metricsService.setup({ http: httpMock }); + await metricsService.start(); + + let completed = false; + + getOpsMetrics$().subscribe({ + complete: () => { + completed = true; + }, + }); + + await metricsService.stop(); + + expect(completed).toEqual(true); + }); + }); +}); diff --git a/src/core/server/metrics/metrics_service.ts b/src/core/server/metrics/metrics_service.ts new file mode 100644 index 0000000000000..1aed89a4aad60 --- /dev/null +++ b/src/core/server/metrics/metrics_service.ts @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ReplaySubject } from 'rxjs'; +import { first, shareReplay } from 'rxjs/operators'; +import { CoreService } from '../../types'; +import { CoreContext } from '../core_context'; +import { Logger } from '../logging'; +import { InternalHttpServiceSetup } from '../http'; +import { InternalMetricsServiceSetup, InternalMetricsServiceStart, OpsMetrics } from './types'; +import { OpsMetricsCollector } from './ops_metrics_collector'; +import { opsConfig, OpsConfigType } from './ops_config'; + +interface MetricsServiceSetupDeps { + http: InternalHttpServiceSetup; +} + +/** @internal */ +export class MetricsService + implements CoreService { + private readonly logger: Logger; + private metricsCollector?: OpsMetricsCollector; + private collectInterval?: NodeJS.Timeout; + private metrics$ = new ReplaySubject(1); + + constructor(private readonly coreContext: CoreContext) { + this.logger = coreContext.logger.get('metrics'); + } + + public async setup({ http }: MetricsServiceSetupDeps): Promise { + this.metricsCollector = new OpsMetricsCollector(http.server); + + const metricsObservable = this.metrics$.pipe(shareReplay(1)); + + return { + getOpsMetrics$: () => metricsObservable, + }; + } + + public async start(): Promise { + if (!this.metricsCollector) { + throw new Error('#setup() needs to be run first'); + } + const config = await this.coreContext.configService + .atPath(opsConfig.path) + .pipe(first()) + .toPromise(); + + await this.refreshMetrics(); + + this.collectInterval = setInterval(() => { + this.refreshMetrics(); + }, config.interval.asMilliseconds()); + + return {}; + } + + private async refreshMetrics() { + this.logger.debug('Refreshing metrics'); + const metrics = await this.metricsCollector!.collect(); + this.metrics$.next(metrics); + } + + public async stop() { + if (this.collectInterval) { + clearInterval(this.collectInterval); + } + this.metrics$.complete(); + } +} diff --git a/src/core/server/metrics/ops_config.ts b/src/core/server/metrics/ops_config.ts new file mode 100644 index 0000000000000..bd6ae5cc5474d --- /dev/null +++ b/src/core/server/metrics/ops_config.ts @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const opsConfig = { + path: 'ops', + schema: schema.object({ + interval: schema.duration({ defaultValue: '5s' }), + }), +}; + +export type OpsConfigType = TypeOf; diff --git a/src/core/server/metrics/ops_metrics_collector.test.mocks.ts b/src/core/server/metrics/ops_metrics_collector.test.mocks.ts new file mode 100644 index 0000000000000..8265796d57970 --- /dev/null +++ b/src/core/server/metrics/ops_metrics_collector.test.mocks.ts @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const mockOsCollector = { + collect: jest.fn(), +}; +jest.doMock('./collectors/os', () => ({ + OsMetricsCollector: jest.fn().mockImplementation(() => mockOsCollector), +})); + +export const mockProcessCollector = { + collect: jest.fn(), +}; +jest.doMock('./collectors/process', () => ({ + ProcessMetricsCollector: jest.fn().mockImplementation(() => mockProcessCollector), +})); + +export const mockServerCollector = { + collect: jest.fn(), +}; +jest.doMock('./collectors/server', () => ({ + ServerMetricsCollector: jest.fn().mockImplementation(() => mockServerCollector), +})); diff --git a/src/core/server/metrics/ops_metrics_collector.test.ts b/src/core/server/metrics/ops_metrics_collector.test.ts new file mode 100644 index 0000000000000..04302a195fb6c --- /dev/null +++ b/src/core/server/metrics/ops_metrics_collector.test.ts @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + mockOsCollector, + mockProcessCollector, + mockServerCollector, +} from './ops_metrics_collector.test.mocks'; +import { httpServiceMock } from '../http/http_service.mock'; +import { OpsMetricsCollector } from './ops_metrics_collector'; + +describe('OpsMetricsCollector', () => { + let collector: OpsMetricsCollector; + + beforeEach(() => { + const hapiServer = httpServiceMock.createSetupContract().server; + collector = new OpsMetricsCollector(hapiServer); + + mockOsCollector.collect.mockResolvedValue('osMetrics'); + }); + + it('gathers metrics from the underlying collectors', async () => { + mockOsCollector.collect.mockResolvedValue('osMetrics'); + mockProcessCollector.collect.mockResolvedValue('processMetrics'); + mockServerCollector.collect.mockResolvedValue({ + requests: 'serverRequestsMetrics', + response_times: 'serverTimingMetrics', + }); + + const metrics = await collector.collect(); + + expect(mockOsCollector.collect).toHaveBeenCalledTimes(1); + expect(mockProcessCollector.collect).toHaveBeenCalledTimes(1); + expect(mockServerCollector.collect).toHaveBeenCalledTimes(1); + + expect(metrics).toEqual({ + process: 'processMetrics', + os: 'osMetrics', + requests: 'serverRequestsMetrics', + response_times: 'serverTimingMetrics', + }); + }); +}); diff --git a/src/core/server/metrics/ops_metrics_collector.ts b/src/core/server/metrics/ops_metrics_collector.ts new file mode 100644 index 0000000000000..04344f21f57f7 --- /dev/null +++ b/src/core/server/metrics/ops_metrics_collector.ts @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Server as HapiServer } from 'hapi'; +import { + ProcessMetricsCollector, + OsMetricsCollector, + ServerMetricsCollector, + MetricsCollector, +} from './collectors'; +import { OpsMetrics } from './types'; + +export class OpsMetricsCollector implements MetricsCollector { + private readonly processCollector: ProcessMetricsCollector; + private readonly osCollector: OsMetricsCollector; + private readonly serverCollector: ServerMetricsCollector; + + constructor(server: HapiServer) { + this.processCollector = new ProcessMetricsCollector(); + this.osCollector = new OsMetricsCollector(); + this.serverCollector = new ServerMetricsCollector(server); + } + + public async collect(): Promise { + const [process, os, server] = await Promise.all([ + this.processCollector.collect(), + this.osCollector.collect(), + this.serverCollector.collect(), + ]); + return { + process, + os, + ...server, + }; + } +} diff --git a/src/core/server/metrics/types.ts b/src/core/server/metrics/types.ts new file mode 100644 index 0000000000000..5c8f18fff380d --- /dev/null +++ b/src/core/server/metrics/types.ts @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Observable } from 'rxjs'; +import { OpsProcessMetrics, OpsOsMetrics, OpsServerMetrics } from './collectors'; + +/** + * APIs to retrieves metrics gathered and exposed by the core platform. + * + * @public + */ +export interface MetricsServiceSetup { + /** + * Retrieve an observable emitting the {@link OpsMetrics} gathered. + * The observable will emit an initial value during core's `start` phase, and a new value every fixed interval of time, + * based on the `opts.interval` configuration property. + * + * @example + * ```ts + * core.metrics.getOpsMetrics$().subscribe(metrics => { + * // do something with the metrics + * }) + * ``` + */ + getOpsMetrics$: () => Observable; +} +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface MetricsServiceStart {} + +export type InternalMetricsServiceSetup = MetricsServiceSetup; +export type InternalMetricsServiceStart = MetricsServiceStart; + +/** + * Regroups metrics gathered by all the collectors. + * This contains metrics about the os/runtime, the kibana process and the http server. + * + * @public + */ +export interface OpsMetrics { + /** Process related metrics */ + process: OpsProcessMetrics; + /** OS related metrics */ + os: OpsOsMetrics; + /** server response time stats */ + response_times: OpsServerMetrics['response_times']; + /** server requests stats */ + requests: OpsServerMetrics['requests']; + /** number of current concurrent connections to the server */ + concurrent_connections: OpsServerMetrics['concurrent_connections']; +} diff --git a/src/core/server/mocks.ts b/src/core/server/mocks.ts index 96b28ab5827e1..037f3bbed67e0 100644 --- a/src/core/server/mocks.ts +++ b/src/core/server/mocks.ts @@ -30,6 +30,8 @@ import { uiSettingsServiceMock } from './ui_settings/ui_settings_service.mock'; import { SharedGlobalConfig } from './plugins'; import { InternalCoreSetup, InternalCoreStart } from './internal_types'; import { capabilitiesServiceMock } from './capabilities/capabilities_service.mock'; +import { metricsServiceMock } from './metrics/metrics_service.mock'; +import { uuidServiceMock } from './uuid/uuid_service.mock'; export { httpServerMock } from './http/http_server.mocks'; export { sessionStorageMock } from './http/cookie_session_storage.mocks'; @@ -40,7 +42,7 @@ export { loggingServiceMock } from './logging/logging_service.mock'; export { savedObjectsRepositoryMock } from './saved_objects/service/lib/repository.mock'; export { typeRegistryMock as savedObjectsTypeRegistryMock } from './saved_objects/saved_objects_type_registry.mock'; export { uiSettingsServiceMock } from './ui_settings/ui_settings_service.mock'; -import { uuidServiceMock } from './uuid/uuid_service.mock'; +export { metricsServiceMock } from './metrics/metrics_service.mock'; export function pluginInitializerContextConfigMock(config: T) { const globalConfig: SharedGlobalConfig = { @@ -153,6 +155,7 @@ function createInternalCoreSetupMock() { uiSettings: uiSettingsServiceMock.createSetupContract(), savedObjects: savedObjectsServiceMock.createInternalSetupContract(), uuid: uuidServiceMock.createSetupContract(), + metrics: metricsServiceMock.createInternalSetupContract(), }; return setupDeps; } diff --git a/src/core/server/plugins/plugin.ts b/src/core/server/plugins/plugin.ts index d6c774f6fc41c..b372874264eb5 100644 --- a/src/core/server/plugins/plugin.ts +++ b/src/core/server/plugins/plugin.ts @@ -95,7 +95,7 @@ export class PluginWrapper< public async setup(setupContext: CoreSetup, plugins: TPluginsSetup) { this.instance = this.createPluginInstance(); - this.log.info('Setting up plugin'); + this.log.debug('Setting up plugin'); return this.instance.setup(setupContext, plugins); } @@ -112,6 +112,8 @@ export class PluginWrapper< throw new Error(`Plugin "${this.name}" can't be started since it isn't set up.`); } + this.log.debug('Starting plugin'); + const startContract = await this.instance.start(startContext, plugins); this.startDependencies$.next([startContext, plugins]); return startContract; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 42bc1ce214b19..445ed16ec7829 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1176,6 +1176,11 @@ export interface LogRecord { timestamp: Date; } +// @public +export interface MetricsServiceSetup { + getOpsMetrics$: () => Observable; +} + // @public (undocumented) export type MIGRATION_ASSISTANCE_INDEX_ACTION = 'upgrade' | 'reindex'; @@ -1227,6 +1232,63 @@ export interface OnPreResponseToolkit { next: (responseExtensions?: OnPreResponseExtensions) => OnPreResponseResult; } +// @public +export interface OpsMetrics { + concurrent_connections: OpsServerMetrics['concurrent_connections']; + os: OpsOsMetrics; + process: OpsProcessMetrics; + requests: OpsServerMetrics['requests']; + response_times: OpsServerMetrics['response_times']; +} + +// @public +export interface OpsOsMetrics { + distro?: string; + distroRelease?: string; + load: { + '1m': number; + '5m': number; + '15m': number; + }; + memory: { + total_in_bytes: number; + free_in_bytes: number; + used_in_bytes: number; + }; + platform: NodeJS.Platform; + platformRelease: string; + uptime_in_millis: number; +} + +// @public +export interface OpsProcessMetrics { + event_loop_delay: number; + memory: { + heap: { + total_in_bytes: number; + used_in_bytes: number; + size_limit: number; + }; + resident_set_size_in_bytes: number; + }; + pid: number; + uptime_in_millis: number; +} + +// @public +export interface OpsServerMetrics { + concurrent_connections: number; + requests: { + disconnects: number; + total: number; + statusCodes: Record; + }; + response_times: { + avg_in_millis: number; + max_in_millis: number; + }; +} + // @public (undocumented) export interface PackageInfo { // (undocumented) diff --git a/src/core/server/server.test.mocks.ts b/src/core/server/server.test.mocks.ts index 038c4651ff5a7..53d1b742a6494 100644 --- a/src/core/server/server.test.mocks.ts +++ b/src/core/server/server.test.mocks.ts @@ -79,3 +79,9 @@ export const mockUuidService = uuidServiceMock.create(); jest.doMock('./uuid/uuid_service', () => ({ UuidService: jest.fn(() => mockUuidService), })); + +import { metricsServiceMock } from './metrics/metrics_service.mock'; +export const mockMetricsService = metricsServiceMock.create(); +jest.doMock('./metrics/metrics_service', () => ({ + MetricsService: jest.fn(() => mockMetricsService), +})); diff --git a/src/core/server/server.test.ts b/src/core/server/server.test.ts index 161dd3759a218..a4b5a9d81df20 100644 --- a/src/core/server/server.test.ts +++ b/src/core/server/server.test.ts @@ -28,6 +28,7 @@ import { mockEnsureValidConfiguration, mockUiSettingsService, mockRenderingService, + mockMetricsService, } from './server.test.mocks'; import { BehaviorSubject } from 'rxjs'; @@ -61,6 +62,7 @@ test('sets up services on "setup"', async () => { expect(mockSavedObjectsService.setup).not.toHaveBeenCalled(); expect(mockUiSettingsService.setup).not.toHaveBeenCalled(); expect(mockRenderingService.setup).not.toHaveBeenCalled(); + expect(mockMetricsService.setup).not.toHaveBeenCalled(); await server.setup(); @@ -71,6 +73,7 @@ test('sets up services on "setup"', async () => { expect(mockSavedObjectsService.setup).toHaveBeenCalledTimes(1); expect(mockUiSettingsService.setup).toHaveBeenCalledTimes(1); expect(mockRenderingService.setup).toHaveBeenCalledTimes(1); + expect(mockMetricsService.setup).toHaveBeenCalledTimes(1); }); test('injects legacy dependency to context#setup()', async () => { @@ -107,6 +110,7 @@ test('runs services on "start"', async () => { expect(mockLegacyService.start).not.toHaveBeenCalled(); expect(mockSavedObjectsService.start).not.toHaveBeenCalled(); expect(mockUiSettingsService.start).not.toHaveBeenCalled(); + expect(mockMetricsService.start).not.toHaveBeenCalled(); await server.start(); @@ -114,6 +118,7 @@ test('runs services on "start"', async () => { expect(mockLegacyService.start).toHaveBeenCalledTimes(1); expect(mockSavedObjectsService.start).toHaveBeenCalledTimes(1); expect(mockUiSettingsService.start).toHaveBeenCalledTimes(1); + expect(mockMetricsService.start).toHaveBeenCalledTimes(1); }); test('does not fail on "setup" if there are unused paths detected', async () => { @@ -135,6 +140,7 @@ test('stops services on "stop"', async () => { expect(mockLegacyService.stop).not.toHaveBeenCalled(); expect(mockSavedObjectsService.stop).not.toHaveBeenCalled(); expect(mockUiSettingsService.stop).not.toHaveBeenCalled(); + expect(mockMetricsService.stop).not.toHaveBeenCalled(); await server.stop(); @@ -144,6 +150,7 @@ test('stops services on "stop"', async () => { expect(mockLegacyService.stop).toHaveBeenCalledTimes(1); expect(mockSavedObjectsService.stop).toHaveBeenCalledTimes(1); expect(mockUiSettingsService.stop).toHaveBeenCalledTimes(1); + expect(mockMetricsService.stop).toHaveBeenCalledTimes(1); }); test(`doesn't setup core services if config validation fails`, async () => { @@ -159,6 +166,7 @@ test(`doesn't setup core services if config validation fails`, async () => { expect(mockLegacyService.setup).not.toHaveBeenCalled(); expect(mockUiSettingsService.setup).not.toHaveBeenCalled(); expect(mockRenderingService.setup).not.toHaveBeenCalled(); + expect(mockMetricsService.setup).not.toHaveBeenCalled(); }); test(`doesn't setup core services if legacy config validation fails`, async () => { @@ -178,4 +186,5 @@ test(`doesn't setup core services if legacy config validation fails`, async () = expect(mockLegacyService.setup).not.toHaveBeenCalled(); expect(mockSavedObjectsService.stop).not.toHaveBeenCalled(); expect(mockUiSettingsService.setup).not.toHaveBeenCalled(); + expect(mockMetricsService.setup).not.toHaveBeenCalled(); }); diff --git a/src/core/server/server.ts b/src/core/server/server.ts index db2493b38d6e0..8603f5fba1da8 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -34,6 +34,7 @@ import { Logger, LoggerFactory } from './logging'; import { UiSettingsService } from './ui_settings'; import { PluginsService, config as pluginsConfig } from './plugins'; import { SavedObjectsService } from '../server/saved_objects'; +import { MetricsService, opsConfig } from './metrics'; import { config as cspConfig } from './csp'; import { config as elasticsearchConfig } from './elasticsearch'; @@ -67,6 +68,7 @@ export class Server { private readonly savedObjects: SavedObjectsService; private readonly uiSettings: UiSettingsService; private readonly uuid: UuidService; + private readonly metrics: MetricsService; private coreStart?: InternalCoreStart; @@ -89,6 +91,7 @@ export class Server { this.uiSettings = new UiSettingsService(core); this.capabilities = new CapabilitiesService(core); this.uuid = new UuidService(core); + this.metrics = new MetricsService(core); } public async setup() { @@ -137,6 +140,8 @@ export class Server { legacyPlugins, }); + const metricsSetup = await this.metrics.setup({ http: httpSetup }); + const coreSetup: InternalCoreSetup = { capabilities: capabilitiesSetup, context: contextServiceSetup, @@ -145,6 +150,7 @@ export class Server { uiSettings: uiSettingsSetup, savedObjects: savedObjectsSetup, uuid: uuidSetup, + metrics: metricsSetup, }; const pluginsSetup = await this.plugins.setup(coreSetup); @@ -193,6 +199,7 @@ export class Server { await this.http.start(); await this.rendering.start(); + await this.metrics.start(); return this.coreStart; } @@ -207,6 +214,7 @@ export class Server { await this.http.stop(); await this.uiSettings.stop(); await this.rendering.stop(); + await this.metrics.stop(); } private registerDefaultRoute(httpSetup: InternalHttpServiceSetup) { @@ -260,6 +268,7 @@ export class Server { [savedObjectsConfig.path, savedObjectsConfig.schema], [savedObjectsMigrationConfig.path, savedObjectsMigrationConfig.schema], [uiSettingsConfig.path, uiSettingsConfig.schema], + [opsConfig.path, opsConfig.schema], ]; this.configService.addDeprecationProvider(rootConfigPath, coreDeprecationProvider); diff --git a/src/core/utils/merge.test.ts b/src/core/utils/merge.test.ts index c857e980dec21..7ef07a83399ac 100644 --- a/src/core/utils/merge.test.ts +++ b/src/core/utils/merge.test.ts @@ -17,6 +17,7 @@ * under the License. */ +// eslint-disable-next-line max-classes-per-file import { merge } from './merge'; describe('merge', () => { @@ -62,6 +63,29 @@ describe('merge', () => { expect(merge({ a: 0 }, { a: 1 }, {})).toEqual({ a: 1 }); }); + test('does not merge class instances', () => { + class Folder { + constructor(public readonly path: string) {} + getPath() { + return this.path; + } + } + class File { + constructor(public readonly content: string) {} + getContent() { + return this.content; + } + } + const folder = new Folder('/etc'); + const file = new File('yolo'); + + const result = merge({}, { content: folder }, { content: file }); + expect(result).toStrictEqual({ + content: file, + }); + expect(result.content.getContent()).toBe('yolo'); + }); + test(`doesn't pollute prototypes`, () => { merge({}, JSON.parse('{ "__proto__": { "foo": "bar" } }')); merge({}, JSON.parse('{ "constructor": { "prototype": { "foo": "bar" } } }')); diff --git a/src/core/utils/merge.ts b/src/core/utils/merge.ts index 8e5d9f4860d95..43878c27b1e19 100644 --- a/src/core/utils/merge.ts +++ b/src/core/utils/merge.ts @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ - +import { isPlainObject } from 'lodash'; /** * Deeply merges two objects, omitting undefined values, and not deeply merging Arrays. * @@ -60,7 +60,7 @@ export function merge>( ) as TReturn; } -const isMergable = (obj: any) => typeof obj === 'object' && obj !== null && !Array.isArray(obj); +const isMergable = (obj: any) => isPlainObject(obj); const mergeObjects = , U extends Record>( baseObj: T, diff --git a/src/legacy/core_plugins/data/public/search/aggs/__tests__/buckets/_terms_other_bucket_helper.js b/src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts similarity index 54% rename from src/legacy/core_plugins/data/public/search/aggs/__tests__/buckets/_terms_other_bucket_helper.js rename to src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts index 749dad377f2e2..976ab57c00b63 100644 --- a/src/legacy/core_plugins/data/public/search/aggs/__tests__/buckets/_terms_other_bucket_helper.js +++ b/src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.test.ts @@ -17,39 +17,73 @@ * under the License. */ -import expect from '@kbn/expect'; -import ngMock from 'ng_mock'; import { buildOtherBucketAgg, mergeOtherBucketAggResponse, updateMissingBucket, -} from '../../buckets/_terms_other_bucket_helper'; -import { start as visualizationsStart } from '../../../../../../../core_plugins/visualizations/public/np_ready/public/legacy'; -import FixturesStubbedLogstashIndexPatternProvider from 'fixtures/stubbed_logstash_index_pattern'; +} from './_terms_other_bucket_helper'; +import { AggConfigs, CreateAggConfigParams } from '../agg_configs'; +import { BUCKET_TYPES } from './bucket_agg_types'; +import { IBucketAggConfig } from './_bucket_agg_type'; +import { mockDataServices, mockAggTypesRegistry } from '../test_helpers'; -const visConfigSingleTerm = { - type: 'pie', +const indexPattern = { + id: '1234', + title: 'logstash-*', + fields: [ + { + name: 'field', + }, + ], +} as any; + +const singleTerm = { aggs: [ { - type: 'terms', - schema: 'segment', - params: { field: 'machine.os.raw', otherBucket: true, missingBucket: true }, + id: '1', + type: BUCKET_TYPES.TERMS, + params: { + field: { + name: 'machine.os.raw', + indexPattern, + filterable: true, + }, + otherBucket: true, + missingBucket: true, + }, }, ], }; -const visConfigNestedTerm = { - type: 'pie', +const nestedTerm = { aggs: [ { - type: 'terms', - schema: 'segment', - params: { field: 'geo.src', size: 2, otherBucket: false, missingBucket: false }, + id: '1', + type: BUCKET_TYPES.TERMS, + params: { + field: { + name: 'geo.src', + indexPattern, + filterable: true, + }, + size: 2, + otherBucket: false, + missingBucket: false, + }, }, { - type: 'terms', - schema: 'segment', - params: { field: 'machine.os.raw', size: 2, otherBucket: true, missingBucket: true }, + id: '2', + type: BUCKET_TYPES.TERMS, + params: { + field: { + name: 'machine.os.raw', + indexPattern, + filterable: true, + }, + size: 2, + otherBucket: true, + missingBucket: true, + }, }, ], }; @@ -183,28 +217,36 @@ const nestedOtherResponse = { status: 200, }; -describe('Terms Agg Other bucket helper', () => { - let vis; +jest.mock('ui/new_platform'); - function init(aggConfig) { - ngMock.module('kibana'); - ngMock.inject(Private => { - const indexPattern = Private(FixturesStubbedLogstashIndexPatternProvider); +describe('Terms Agg Other bucket helper', () => { + const typesRegistry = mockAggTypesRegistry(); + const getAggConfigs = (aggs: CreateAggConfigParams[] = []) => { + return new AggConfigs(indexPattern, [...aggs], { typesRegistry }); + }; - vis = new visualizationsStart.Vis(indexPattern, aggConfig); - }); - } + beforeEach(() => { + mockDataServices(); + }); describe('buildOtherBucketAgg', () => { - it('returns a function', () => { - init(visConfigSingleTerm); - const agg = buildOtherBucketAgg(vis.aggs, vis.aggs.aggs[0], singleTermResponse); - expect(agg).to.be.a('function'); + test('returns a function', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + singleTermResponse + ); + expect(typeof agg).toBe('function'); }); - it('correctly builds query with single terms agg', () => { - init(visConfigSingleTerm); - const agg = buildOtherBucketAgg(vis.aggs, vis.aggs.aggs[0], singleTermResponse)(); + test('correctly builds query with single terms agg', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + singleTermResponse + ); const expectedResponse = { aggs: undefined, filters: { @@ -223,13 +265,19 @@ describe('Terms Agg Other bucket helper', () => { }, }, }; - - expect(agg['other-filter']).to.eql(expectedResponse); + expect(agg).toBeDefined(); + if (agg) { + expect(agg()['other-filter']).toEqual(expectedResponse); + } }); - it('correctly builds query for nested terms agg', () => { - init(visConfigNestedTerm); - const agg = buildOtherBucketAgg(vis.aggs, vis.aggs.aggs[1], nestedTermResponse)(); + test('correctly builds query for nested terms agg', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + nestedTermResponse + ); const expectedResponse = { 'other-filter': { aggs: undefined, @@ -267,54 +315,84 @@ describe('Terms Agg Other bucket helper', () => { }, }, }; - - expect(agg).to.eql(expectedResponse); + expect(agg).toBeDefined(); + if (agg) { + expect(agg()).toEqual(expectedResponse); + } }); - it('returns false when nested terms agg has no buckets', () => { - init(visConfigNestedTerm); - const agg = buildOtherBucketAgg(vis.aggs, vis.aggs.aggs[1], nestedTermResponseNoResults); - expect(agg).to.eql(false); + test('returns false when nested terms agg has no buckets', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const agg = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + nestedTermResponseNoResults + ); + + expect(agg).toEqual(false); }); }); describe('mergeOtherBucketAggResponse', () => { - it('correctly merges other bucket with single terms agg', () => { - init(visConfigSingleTerm); - const otherAggConfig = buildOtherBucketAgg(vis.aggs, vis.aggs.aggs[0], singleTermResponse)(); - const mergedResponse = mergeOtherBucketAggResponse( - vis.aggs, - singleTermResponse, - singleOtherResponse, - vis.aggs.aggs[0], - otherAggConfig + test('correctly merges other bucket with single terms agg', () => { + const aggConfigs = getAggConfigs(singleTerm.aggs); + const otherAggConfig = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig, + singleTermResponse ); - expect(mergedResponse.aggregations['1'].buckets[3].key).to.equal('__other__'); + expect(otherAggConfig).toBeDefined(); + if (otherAggConfig) { + const mergedResponse = mergeOtherBucketAggResponse( + aggConfigs, + singleTermResponse, + singleOtherResponse, + aggConfigs.aggs[0] as IBucketAggConfig, + otherAggConfig() + ); + expect(mergedResponse.aggregations['1'].buckets[3].key).toEqual('__other__'); + } }); - it('correctly merges other bucket with nested terms agg', () => { - init(visConfigNestedTerm); - const otherAggConfig = buildOtherBucketAgg(vis.aggs, vis.aggs.aggs[1], nestedTermResponse)(); - const mergedResponse = mergeOtherBucketAggResponse( - vis.aggs, - nestedTermResponse, - nestedOtherResponse, - vis.aggs.aggs[1], - otherAggConfig + test('correctly merges other bucket with nested terms agg', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const otherAggConfig = buildOtherBucketAgg( + aggConfigs, + aggConfigs.aggs[1] as IBucketAggConfig, + nestedTermResponse ); - expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].key).to.equal('__other__'); + expect(otherAggConfig).toBeDefined(); + if (otherAggConfig) { + const mergedResponse = mergeOtherBucketAggResponse( + aggConfigs, + nestedTermResponse, + nestedOtherResponse, + aggConfigs.aggs[1] as IBucketAggConfig, + otherAggConfig() + ); + + expect(mergedResponse.aggregations['1'].buckets[1]['2'].buckets[3].key).toEqual( + '__other__' + ); + } }); }); describe('updateMissingBucket', () => { - it('correctly updates missing bucket key', () => { - init(visConfigNestedTerm); - const updatedResponse = updateMissingBucket(singleTermResponse, vis.aggs, vis.aggs.aggs[0]); + test('correctly updates missing bucket key', () => { + const aggConfigs = getAggConfigs(nestedTerm.aggs); + const updatedResponse = updateMissingBucket( + singleTermResponse, + aggConfigs, + aggConfigs.aggs[0] as IBucketAggConfig + ); expect( - updatedResponse.aggregations['1'].buckets.find(bucket => bucket.key === '__missing__') - ).to.not.be('undefined'); + updatedResponse.aggregations['1'].buckets.find( + (bucket: Record) => bucket.key === '__missing__' + ) + ).toBeDefined(); }); }); }); diff --git a/src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.js b/src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts similarity index 65% rename from src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.js rename to src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts index ddab360161744..42db37c81eadd 100644 --- a/src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.js +++ b/src/legacy/core_plugins/data/public/search/aggs/buckets/_terms_other_bucket_helper.ts @@ -17,21 +17,24 @@ * under the License. */ -import _ from 'lodash'; +import { isNumber, keys, values, find, each, cloneDeep, flatten } from 'lodash'; import { esFilters, esQuery } from '../../../../../../../plugins/data/public'; import { AggGroupNames } from '../agg_groups'; +import { IAggConfigs } from '../agg_configs'; +import { IBucketAggConfig } from './_bucket_agg_type'; /** * walks the aggregation DSL and returns DSL starting at aggregation with id of startFromAggId * @param aggNestedDsl: aggregation config DSL (top level) * @param startFromId: id of an aggregation from where we want to get the nested DSL */ -const getNestedAggDSL = (aggNestedDsl, startFromAggId) => { +const getNestedAggDSL = (aggNestedDsl: Record, startFromAggId: string): any => { if (aggNestedDsl[startFromAggId]) { return aggNestedDsl[startFromAggId]; } - const nestedAggs = _.values(aggNestedDsl); + const nestedAggs: Array> = values(aggNestedDsl); let aggs; + for (let i = 0; i < nestedAggs.length; i++) { if (nestedAggs[i].aggs && (aggs = getNestedAggDSL(nestedAggs[i].aggs, startFromAggId))) { return aggs; @@ -46,27 +49,34 @@ const getNestedAggDSL = (aggNestedDsl, startFromAggId) => { * @param aggWithOtherBucket: AggConfig of the aggregation with other bucket enabled * @param key: key from the other bucket request for a specific other bucket */ -const getAggResultBuckets = (aggConfigs, response, aggWithOtherBucket, key) => { +const getAggResultBuckets = ( + aggConfigs: IAggConfigs, + response: any, + aggWithOtherBucket: IBucketAggConfig, + key: string +) => { const keyParts = key.split('-'); let responseAgg = response; for (const i in keyParts) { if (keyParts[i]) { - const responseAggs = _.values(responseAgg); + const responseAggs: Array> = values(responseAgg); // If you have multi aggs, we cannot just assume the first one is the `other` bucket, // so we need to loop over each agg until we find it. for (let aggId = 0; aggId < responseAggs.length; aggId++) { - const agg = responseAggs[aggId]; - const aggKey = _.keys(responseAgg)[aggId]; - const aggConfig = _.find(aggConfigs.aggs, agg => agg.id === aggKey); - const bucket = _.find(agg.buckets, (bucket, bucketObjKey) => { - const bucketKey = aggConfig - .getKey(bucket, Number.isInteger(bucketObjKey) ? null : bucketObjKey) - .toString(); - return bucketKey === keyParts[i]; - }); - if (bucket) { - responseAgg = bucket; - break; + const aggById = responseAggs[aggId]; + const aggKey = keys(responseAgg)[aggId]; + const aggConfig = find(aggConfigs.aggs, agg => agg.id === aggKey); + if (aggConfig) { + const aggResultBucket = find(aggById.buckets, (bucket, bucketObjKey) => { + const bucketKey = aggConfig + .getKey(bucket, isNumber(bucketObjKey) ? undefined : bucketObjKey) + .toString(); + return bucketKey === keyParts[i]; + }); + if (aggResultBucket) { + responseAgg = aggResultBucket; + break; + } } } } @@ -82,21 +92,20 @@ const getAggResultBuckets = (aggConfigs, response, aggWithOtherBucket, key) => { * @param responseAggs: array of aggregations from response * @param aggId: id of the aggregation with missing bucket */ -const getAggConfigResultMissingBuckets = (responseAggs, aggId) => { +const getAggConfigResultMissingBuckets = (responseAggs: any, aggId: string) => { const missingKey = '__missing__'; - let resultBuckets = []; + let resultBuckets: Array> = []; if (responseAggs[aggId]) { - const matchingBucket = responseAggs[aggId].buckets.find(bucket => bucket.key === missingKey); + const matchingBucket = responseAggs[aggId].buckets.find( + (bucket: Record) => bucket.key === missingKey + ); if (matchingBucket) resultBuckets.push(matchingBucket); return resultBuckets; } - _.each(responseAggs, agg => { + each(responseAggs, agg => { if (agg.buckets) { - _.each(agg.buckets, bucket => { - resultBuckets = [ - ...resultBuckets, - ...getAggConfigResultMissingBuckets(bucket, aggId, missingKey), - ]; + each(agg.buckets, bucket => { + resultBuckets = [...resultBuckets, ...getAggConfigResultMissingBuckets(bucket, aggId)]; }); } }); @@ -110,13 +119,24 @@ const getAggConfigResultMissingBuckets = (responseAggs, aggId) => { * @param key: the key for this specific other bucket * @param otherAgg: AggConfig of the aggregation with other bucket */ -const getOtherAggTerms = (requestAgg, key, otherAgg) => { +const getOtherAggTerms = ( + requestAgg: Record, + key: string, + otherAgg: IBucketAggConfig +) => { return requestAgg['other-filter'].filters.filters[key].bool.must_not - .filter(filter => filter.match_phrase && filter.match_phrase[otherAgg.params.field.name]) - .map(filter => filter.match_phrase[otherAgg.params.field.name]); + .filter( + (filter: Record) => + filter.match_phrase && filter.match_phrase[otherAgg.params.field.name] + ) + .map((filter: Record) => filter.match_phrase[otherAgg.params.field.name]); }; -export const buildOtherBucketAgg = (aggConfigs, aggWithOtherBucket, response) => { +export const buildOtherBucketAgg = ( + aggConfigs: IAggConfigs, + aggWithOtherBucket: IBucketAggConfig, + response: any +) => { const bucketAggs = aggConfigs.aggs.filter(agg => agg.type.type === AggGroupNames.Buckets); const index = bucketAggs.findIndex(agg => agg.id === aggWithOtherBucket.id); const aggs = aggConfigs.toDsl(); @@ -130,6 +150,7 @@ export const buildOtherBucketAgg = (aggConfigs, aggWithOtherBucket, response) => params: { filters: [], }, + enabled: false, }, { addToAggConfigs: false, @@ -145,25 +166,31 @@ export const buildOtherBucketAgg = (aggConfigs, aggWithOtherBucket, response) => let noAggBucketResults = false; // recursively create filters for all parent aggregation buckets - const walkBucketTree = (aggIndex, aggs, aggId, filters, key) => { + const walkBucketTree = ( + aggIndex: number, + aggregations: any, + aggId: string, + filters: any[], + key: string + ) => { // make sure there are actually results for the buckets - if (aggs[aggId].buckets.length < 1) { + if (aggregations[aggId].buckets.length < 1) { noAggBucketResults = true; return; } - const agg = aggs[aggId]; + const agg = aggregations[aggId]; const newAggIndex = aggIndex + 1; const newAgg = bucketAggs[newAggIndex]; const currentAgg = bucketAggs[aggIndex]; if (aggIndex < index) { - _.each(agg.buckets, (bucket, bucketObjKey) => { + each(agg.buckets, (bucket: any, bucketObjKey) => { const bucketKey = currentAgg.getKey( bucket, - Number.isInteger(bucketObjKey) ? null : bucketObjKey + isNumber(bucketObjKey) ? undefined : bucketObjKey ); - const filter = _.cloneDeep(bucket.filters) || currentAgg.createFilter(bucketKey); - const newFilters = _.flatten([...filters, filter]); + const filter = cloneDeep(bucket.filters) || currentAgg.createFilter(bucketKey); + const newFilters = flatten([...filters, filter]); walkBucketTree( newAggIndex, bucket, @@ -177,7 +204,7 @@ export const buildOtherBucketAgg = (aggConfigs, aggWithOtherBucket, response) => if ( !aggWithOtherBucket.params.missingBucket || - agg.buckets.some(bucket => bucket.key === '__missing__') + agg.buckets.some((bucket: { key: string }) => bucket.key === '__missing__') ) { filters.push( esFilters.buildExistsFilter( @@ -188,7 +215,7 @@ export const buildOtherBucketAgg = (aggConfigs, aggWithOtherBucket, response) => } // create not filters for all the buckets - _.each(agg.buckets, bucket => { + each(agg.buckets, bucket => { if (bucket.key === '__missing__') return; const filter = currentAgg.createFilter(bucket.key); filter.meta.negate = true; @@ -214,15 +241,15 @@ export const buildOtherBucketAgg = (aggConfigs, aggWithOtherBucket, response) => }; export const mergeOtherBucketAggResponse = ( - aggsConfig, - response, - otherResponse, - otherAgg, - requestAgg + aggsConfig: IAggConfigs, + response: any, + otherResponse: any, + otherAgg: IBucketAggConfig, + requestAgg: Record ) => { - const updatedResponse = _.cloneDeep(response); - _.each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => { - if (!bucket.doc_count) return; + const updatedResponse = cloneDeep(response); + each(otherResponse.aggregations['other-filter'].buckets, (bucket, key) => { + if (!bucket.doc_count || key === undefined) return; const bucketKey = key.replace(/^-/, ''); const aggResultBuckets = getAggResultBuckets( aggsConfig, @@ -241,7 +268,11 @@ export const mergeOtherBucketAggResponse = ( bucket.filters = [phraseFilter]; bucket.key = '__other__'; - if (aggResultBuckets.some(bucket => bucket.key === '__missing__')) { + if ( + aggResultBuckets.some( + (aggResultBucket: Record) => aggResultBucket.key === '__missing__' + ) + ) { bucket.filters.push( esFilters.buildExistsFilter(otherAgg.params.field, otherAgg.params.field.indexPattern) ); @@ -251,8 +282,12 @@ export const mergeOtherBucketAggResponse = ( return updatedResponse; }; -export const updateMissingBucket = (response, aggConfigs, agg) => { - const updatedResponse = _.cloneDeep(response); +export const updateMissingBucket = ( + response: any, + aggConfigs: IAggConfigs, + agg: IBucketAggConfig +) => { + const updatedResponse = cloneDeep(response); const aggResultBuckets = getAggConfigResultMissingBuckets(updatedResponse.aggregations, agg.id); aggResultBuckets.forEach(bucket => { bucket.key = '__missing__'; diff --git a/src/legacy/core_plugins/data/public/search/aggs/buckets/terms.ts b/src/legacy/core_plugins/data/public/search/aggs/buckets/terms.ts index 0ed44aa876744..8fd95c86d8476 100644 --- a/src/legacy/core_plugins/data/public/search/aggs/buckets/terms.ts +++ b/src/legacy/core_plugins/data/public/search/aggs/buckets/terms.ts @@ -39,7 +39,6 @@ import { buildOtherBucketAgg, mergeOtherBucketAggResponse, updateMissingBucket, - // @ts-ignore } from './_terms_other_bucket_helper'; import { Schemas } from '../schemas'; import { AggGroupNames } from '../agg_groups'; diff --git a/src/legacy/core_plugins/input_control_vis/public/components/vis/__snapshots__/list_control.test.tsx.snap b/src/legacy/core_plugins/input_control_vis/public/components/vis/__snapshots__/list_control.test.tsx.snap index 99482a4be2d7b..59ae99260cecd 100644 --- a/src/legacy/core_plugins/input_control_vis/public/components/vis/__snapshots__/list_control.test.tsx.snap +++ b/src/legacy/core_plugins/input_control_vis/public/components/vis/__snapshots__/list_control.test.tsx.snap @@ -25,6 +25,7 @@ exports[`renders ListControl 1`] = ` compressed={false} data-test-subj="listControlSelect0" fullWidth={false} + inputRef={[Function]} isClearable={true} isLoading={false} onChange={[Function]} diff --git a/src/legacy/core_plugins/input_control_vis/public/components/vis/list_control.tsx b/src/legacy/core_plugins/input_control_vis/public/components/vis/list_control.tsx index d62adfdce56b4..d01cef15ea41b 100644 --- a/src/legacy/core_plugins/input_control_vis/public/components/vis/list_control.tsx +++ b/src/legacy/core_plugins/input_control_vis/public/components/vis/list_control.tsx @@ -58,8 +58,17 @@ class ListControlUi extends PureComponent { + if (this.textInput) { + this.textInput.setAttribute('focusable', 'false'); // remove when #59039 is fixed + } this.isMounted = true; }; @@ -67,6 +76,10 @@ class ListControlUi extends PureComponent { + this.textInput = ref; + }; + handleOnChange = (selectedOptions: any[]) => { const selectedValues = selectedOptions.map(({ value }) => { return value; @@ -143,6 +156,7 @@ class ListControlUi extends PureComponent ); } diff --git a/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/__snapshots__/clone_modal.test.js.snap b/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/__snapshots__/clone_modal.test.js.snap index f5a00e5435ed6..771d53b73d960 100644 --- a/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/__snapshots__/clone_modal.test.js.snap +++ b/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/__snapshots__/clone_modal.test.js.snap @@ -28,6 +28,7 @@ exports[`renders DashboardCloneModal 1`] = ` } showCopyOnSave={true} + showDescription={false} title="dash title" /> `; diff --git a/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/clone_modal.tsx b/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/clone_modal.tsx index e5e75e4b7d277..08e2b98d1c73d 100644 --- a/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/clone_modal.tsx +++ b/src/legacy/core_plugins/kibana/public/dashboard/np_ready/top_nav/clone_modal.tsx @@ -178,6 +178,9 @@ export class DashboardCloneModal extends React.Component { { showCopyOnSave={this.props.showCopyOnSave} objectType="dashboard" options={this.renderDashboardSaveOptions()} + showDescription={false} /> ); } diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/context_state.ts b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/context_state.ts index 8fb6140d55e31..bf185f78941de 100644 --- a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/context_state.ts +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/context_state.ts @@ -24,9 +24,9 @@ import { syncStates, BaseStateContainer, } from '../../../../../../../plugins/kibana_utils/public'; -import { esFilters, FilterManager, Filter } from '../../../../../../../plugins/data/public'; +import { esFilters, FilterManager, Filter, Query } from '../../../../../../../plugins/data/public'; -interface AppState { +export interface AppState { /** * Columns displayed in the table, cannot be changed by UI, just in discover's main app */ @@ -47,6 +47,7 @@ interface AppState { * Number of records to be fetched after the anchor records (older records) */ successorCount: number; + query?: Query; } interface GlobalState { diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js index 1ac54ad5dabee..bb693ab860221 100644 --- a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.js @@ -305,6 +305,7 @@ function discoverController( defaultMessage: 'Save your Discover search so you can use it in visualizations and dashboards', })} + showDescription={false} /> ); showSaveModal(saveModal, core.i18n.Context); diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/doc_table/components/pager/__snapshots__/tool_bar_pager_buttons.test.tsx.snap b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/doc_table/components/pager/__snapshots__/tool_bar_pager_buttons.test.tsx.snap index a6aab8f74a674..20e503fd5ff91 100644 --- a/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/doc_table/components/pager/__snapshots__/tool_bar_pager_buttons.test.tsx.snap +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/angular/doc_table/components/pager/__snapshots__/tool_bar_pager_buttons.test.tsx.snap @@ -5,6 +5,7 @@ exports[`it renders ToolBarPagerButtons 1`] = ` className="kuiButtonGroup" > @@ -41,6 +48,12 @@ export function ToolBarPagerButtons(props: Props) { onClick={() => props.onPageNext()} disabled={!props.hasNextPage} data-test-subj="btnNextPage" + aria-label={i18n.translate( + 'kbn.ddiscover.docTable.pager.toolbarPagerButtons.nextButtonAriaLabel', + { + defaultMessage: 'Next page in table', + } + )} > diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/field_chooser.js b/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/field_chooser.js index a175a1aebebdf..df970ab5f2584 100644 --- a/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/field_chooser.js +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/field_chooser.js @@ -24,7 +24,11 @@ import './discover_field'; import './discover_field_search_directive'; import './discover_index_pattern_directive'; import fieldChooserTemplate from './field_chooser.html'; -import { IndexPatternFieldList } from '../../../../../../../../plugins/data/public'; +import { + IndexPatternFieldList, + KBN_FIELD_TYPES, +} from '../../../../../../../../plugins/data/public'; +import { getMapsAppUrl, isFieldVisualizable, isMapsAppRegistered } from './lib/visualize_url_utils'; export function createFieldChooserDirective($location, config, $route) { return { @@ -186,8 +190,15 @@ export function createFieldChooserDirective($location, config, $route) { return ''; } + if ( + (field.type === KBN_FIELD_TYPES.GEO_POINT || field.type === KBN_FIELD_TYPES.GEO_SHAPE) && + isMapsAppRegistered() + ) { + return getMapsAppUrl(field, $scope.indexPattern, $scope.state, $scope.columns); + } + let agg = {}; - const isGeoPoint = field.type === 'geo_point'; + const isGeoPoint = field.type === KBN_FIELD_TYPES.GEO_POINT; const type = isGeoPoint ? 'tile_map' : 'histogram'; // If we're visualizing a date field, and our index is time based (and thus has a time filter), // then run a date histogram @@ -243,7 +254,7 @@ export function createFieldChooserDirective($location, config, $route) { $scope.computeDetails = function(field, recompute) { if (_.isUndefined(field.details) || recompute) { field.details = { - visualizeUrl: field.visualizable ? getVisualizeUrl(field) : null, + visualizeUrl: isFieldVisualizable(field) ? getVisualizeUrl(field) : null, ...fieldCalculator.getFieldValueCounts({ hits: $scope.hits, field: field, diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/detail_views/string.html b/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/detail_views/string.html index 5d134911fc91b..333dc472e956d 100644 --- a/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/detail_views/string.html +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/detail_views/string.html @@ -79,7 +79,7 @@ @@ -87,7 +87,7 @@ diff --git a/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/visualize_url_utils.ts b/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/visualize_url_utils.ts new file mode 100644 index 0000000000000..8dbf3cd79ccb1 --- /dev/null +++ b/src/legacy/core_plugins/kibana/public/discover/np_ready/components/field_chooser/lib/visualize_url_utils.ts @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import uuid from 'uuid/v4'; +// @ts-ignore +import rison from 'rison-node'; +import { + IFieldType, + IIndexPattern, + KBN_FIELD_TYPES, +} from '../../../../../../../../../plugins/data/public'; +import { AppState } from '../../../angular/context_state'; +import { getServices } from '../../../../kibana_services'; + +function getMapsAppBaseUrl() { + const mapsAppVisAlias = getServices() + .visualizations.types.getAliases() + .find(({ name }) => { + return name === 'maps'; + }); + return mapsAppVisAlias ? mapsAppVisAlias.aliasUrl : null; +} + +export function isMapsAppRegistered() { + return getServices() + .visualizations.types.getAliases() + .some(({ name }) => { + return name === 'maps'; + }); +} + +export function isFieldVisualizable(field: IFieldType) { + if ( + (field.type === KBN_FIELD_TYPES.GEO_POINT || field.type === KBN_FIELD_TYPES.GEO_SHAPE) && + isMapsAppRegistered() + ) { + return true; + } + return field.visualizable; +} + +export function getMapsAppUrl( + field: IFieldType, + indexPattern: IIndexPattern, + appState: AppState, + columns: string[] +) { + const mapAppParams = new URLSearchParams(); + + // Copy global state + const locationSplit = window.location.href.split('discover?'); + if (locationSplit.length > 1) { + const discoverParams = new URLSearchParams(locationSplit[1]); + const globalStateUrlValue = discoverParams.get('_g'); + if (globalStateUrlValue) { + mapAppParams.set('_g', globalStateUrlValue); + } + } + + // Copy filters and query in app state + const mapsAppState: any = { + filters: appState.filters || [], + }; + if (appState.query) { + mapsAppState.query = appState.query; + } + // @ts-ignore + mapAppParams.set('_a', rison.encode(mapsAppState)); + + // create initial layer descriptor + const hasColumns = columns && columns.length && columns[0] !== '_source'; + mapAppParams.set( + 'initialLayers', + // @ts-ignore + rison.encode_array([ + { + id: uuid(), + label: indexPattern.title, + sourceDescriptor: { + id: uuid(), + type: 'ES_SEARCH', + geoField: field.name, + tooltipProperties: hasColumns ? columns : [], + indexPatternId: indexPattern.id, + }, + visible: true, + type: 'VECTOR', + }, + ]) + ); + + return getServices().addBasePath(`${getMapsAppBaseUrl()}?${mapAppParams.toString()}`); +} diff --git a/src/legacy/core_plugins/kibana/public/kibana.js b/src/legacy/core_plugins/kibana/public/kibana.js index a83d1176a7197..a9f32949628e9 100644 --- a/src/legacy/core_plugins/kibana/public/kibana.js +++ b/src/legacy/core_plugins/kibana/public/kibana.js @@ -26,8 +26,6 @@ import { npSetup } from 'ui/new_platform'; // import the uiExports that we want to "use" import 'uiExports/home'; -import 'uiExports/visTypes'; - import 'uiExports/visualize'; import 'uiExports/savedObjectTypes'; import 'uiExports/fieldFormatEditors'; diff --git a/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/_editor.scss b/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/_editor.scss index 2f48ecc322fea..3a542cacc44be 100644 --- a/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/_editor.scss +++ b/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/_editor.scss @@ -22,10 +22,6 @@ a tilemap in an iframe: https://github.com/elastic/kibana/issues/16457 */ } } -.visEditor__linkedMessage { - padding: $euiSizeS; -} - .visEditor__content { @include flex-parent(); width: 100%; diff --git a/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/editor.html b/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/editor.html index 4979d9dc89a0c..9dbb05ea95b48 100644 --- a/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/editor.html +++ b/src/legacy/core_plugins/kibana/public/visualize/np_ready/editor/editor.html @@ -1,28 +1,4 @@ - -