From 44b88d2b00c7a0f280ce118c6ddeb9d6e8554b92 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 15:19:17 -0700 Subject: [PATCH 1/8] Update CI to build elastic 8.0.0 and 8.8.0 --- .github/workflows/_integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_integration_tests.yml b/.github/workflows/_integration_tests.yml index dfcbf621..c4f0e458 100644 --- a/.github/workflows/_integration_tests.yml +++ b/.github/workflows/_integration_tests.yml @@ -8,7 +8,7 @@ jobs: os: - ${{ vars.UBUNTU_VERSION }} node-version: [16.x, 18.x] - es-version: [7.6.1] + es-version: [7.6.1, 8.0.0, 8.8.0] jdk-version: [oraclejdk11] steps: - uses: actions/checkout@v2 From da99b80217b1176ee032b36581169f9e76288ed3 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 15:35:12 -0700 Subject: [PATCH 2/8] Fix CI: elastic8 startup times out elastic7 seems to start up faster than 8, or maybe it was just random jitter, but in any case we probably want to avoid spurious timeouts like this. e.g. see elastic7 which started up: ://github.com/michaelkirk-pelias/schema/actions/runs/5272730400/jobs/9535345934 vs. elastic8 which timed out: https://github.com/michaelkirk-pelias/schema/actions/runs/5272730400/jobs/9535346043 --- scripts/elastic_wait.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/elastic_wait.sh b/scripts/elastic_wait.sh index c8df72cf..2f741d5a 100644 --- a/scripts/elastic_wait.sh +++ b/scripts/elastic_wait.sh @@ -10,7 +10,7 @@ function elastic_status(){ function elastic_wait(){ echo 'waiting for elasticsearch service to come up'; - retry_count=30 + retry_count=60 i=1 while [[ "$i" -le "$retry_count" ]]; do From 8dcabb58e32e3ebcaeb2eac1fdb2085268746d03 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 15:45:35 -0700 Subject: [PATCH 3/8] Fix CI: disable xpack.security for CI With xpack security, elastic8 starts with TLS and user/password auth, which is unnecessary configuration for CI. --- scripts/setup_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/setup_ci.sh b/scripts/setup_ci.sh index b2e055f3..2c90203d 100755 --- a/scripts/setup_ci.sh +++ b/scripts/setup_ci.sh @@ -24,7 +24,8 @@ wget -O - "https://artifacts.elastic.co/downloads/elasticsearch/${FILENAME}" \ /tmp/elasticsearch/bin/elasticsearch-plugin install analysis-icu # start elasticsearch server -/tmp/elasticsearch/bin/elasticsearch --daemonize -Epath.data=/tmp/elasticsearch -Ediscovery.type=single-node +# Settings are NOT suitable for production! +/tmp/elasticsearch/bin/elasticsearch --daemonize -Epath.data=/tmp/elasticsearch -Ediscovery.type=single-node -Expack.security.enabled=false # wait for server to boot up # logs show that on travis-ci it can take ~17s to boot an ES6 server From 72b5cc97c4bb05e1597f4f2b4916c79ee5a98979 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 15:59:35 -0700 Subject: [PATCH 4/8] Update elasticsearch package to be compatible with elasticsearch8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous package was producing an error[^1]. Given that the old npm package is deprecated[^2], I've switched to the new recommended client. The new npm package has a peculiar (though well reasoned) versioning scheme. The major version of the new npm package denotes the minimum supported elasticsearch server version. The package is (apparently) given forward compatibility with newer version. So in this case, using version 7, means we can talk to elasticsearch7 and elasticsearch8 (but not elasticsearch6). This does mean that breaking changes, like dropping node versions, can occur in a minor release, hence the more restrictive versioning constraint (~7.17.0 instead of ^7.17.0). [1] fixes error: ``` -------------- create index -------------- /home/runner/work/schema/schema/node_modules/elasticsearch/src/lib/utils.js:303 throw new TypeError(err); ^ TypeError: Invalid apiVersion "8.0", expected a function or one of _default, 7.6, 7.5, 7.4, 7.3, 7.2, 7.1, 7.0, 6.8, 5.6, 7.7, 7.x, master at Object.utils.funcEnum (/home/runner/work/schema/schema/node_modules/elasticsearch/src/lib/utils.js:303:13) at new Client (/home/runner/work/schema/schema/node_modules/elasticsearch/src/lib/client.js:74:33) at Object. (/home/runner/work/schema/schema/scripts/create_index.js:11:16) at Module._compile (node:internal/modules/cjs/loader:1196:14) at Object.Module._extensions..js (node:internal/modules/cjs/loader:1250:10) at Module.load (node:internal/modules/cjs/loader:1074:32) at Function.Module._load (node:internal/modules/cjs/loader:909:12) at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:81:12) at node:internal/main/run_main_module:22:47 Error: Process completed with exit code 1. ``` [2] old package is deprecated From https://www.npmjs.com/package/elasticsearch > ⚠️ This client is no longer maintained. We strongly advise you to migrate to the [new Elasticsearch client](https://www.elastic.co/blog/new-elasticsearch-javascript-client-released). --- package.json | 4 ++-- scripts/check_plugins.js | 2 +- scripts/check_version.js | 2 +- scripts/create_index.js | 2 +- scripts/drop_index.js | 2 +- scripts/info.js | 2 +- scripts/update_settings.js | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index bae002c5..af2c5a41 100644 --- a/package.json +++ b/package.json @@ -30,10 +30,10 @@ "dependencies": { "@hapi/joi": "^16.1.8", "colors": "^1.1.2", - "elasticsearch": "^16.0.0", + "@elastic/elasticsearch": "~7.17.0", "glob": "^7.1.6", "lodash": "^4.17.15", - "pelias-config": "^6.0.0", + "pelias-config": "https://github.com/michaelkirk-pelias/config#mkirk\/elastic8", "pelias-logger": "^1.3.0", "semver": "^7.1.1" }, diff --git a/scripts/check_plugins.js b/scripts/check_plugins.js index 175594a6..fa5e2edb 100644 --- a/scripts/check_plugins.js +++ b/scripts/check_plugins.js @@ -1,6 +1,6 @@ const colors = require('colors/safe'); const config = require('pelias-config').generate(); -const es = require('elasticsearch'); +const es = require('@elastic/elasticsearch'); const client = new es.Client(config.esclient); const cli = require('./cli'); diff --git a/scripts/check_version.js b/scripts/check_version.js index 2b7d4155..bae503bb 100644 --- a/scripts/check_version.js +++ b/scripts/check_version.js @@ -1,6 +1,6 @@ const _ = require('lodash'); const semver = require('semver'); -const es = require('elasticsearch'); +const es = require('@elastic/elasticsearch'); const config = require('pelias-config').generate(); const client = new es.Client(config.esclient); const cli = require('./cli'); diff --git a/scripts/create_index.js b/scripts/create_index.js index 9072cd1d..93b68b57 100644 --- a/scripts/create_index.js +++ b/scripts/create_index.js @@ -1,6 +1,6 @@ const child_process = require('child_process'); const config = require('pelias-config').generate(); -const es = require('elasticsearch'); +const es = require('@elastic/elasticsearch'); const SUPPORTED_ES_VERSIONS = '>=7.4.2'; const cli = require('./cli'); diff --git a/scripts/drop_index.js b/scripts/drop_index.js index 100ad2ec..a47be7c7 100644 --- a/scripts/drop_index.js +++ b/scripts/drop_index.js @@ -1,6 +1,6 @@ const colors = require('colors/safe'); const config = require('pelias-config').generate(); -const es = require('elasticsearch'); +const es = require('@elastic/elasticsearch'); const client = new es.Client(config.esclient); const readline = require('readline'); const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); diff --git a/scripts/info.js b/scripts/info.js index 373dbccf..62504c6e 100644 --- a/scripts/info.js +++ b/scripts/info.js @@ -1,5 +1,5 @@ var config = require('pelias-config').generate().esclient; -var es = require('elasticsearch'); +var es = require('@elastic/elasticsearch'); var client = new es.Client(config); client.info( {}, console.log.bind(console) ); diff --git a/scripts/update_settings.js b/scripts/update_settings.js index 22d7b9c5..b2ae69b8 100644 --- a/scripts/update_settings.js +++ b/scripts/update_settings.js @@ -1,5 +1,5 @@ var config = require('pelias-config').generate(); -var es = require('elasticsearch'); +var es = require('@elastic/elasticsearch'); var client = new es.Client(config.esclient); var schema = require('../schema'); From 9a46aef5d032187f23aa3560a6d9c134b595319c Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 18:11:21 -0700 Subject: [PATCH 5/8] adapt to new elasticsearch package - response body moved from `res` to `res.body` - nodes.info takes *no* arg rather than null before callback - config.hosts became config.nodes (and changed to string format) --- scripts/check_plugins.js | 8 ++++---- scripts/check_version.js | 5 ++--- scripts/drop_index.js | 9 ++++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/check_plugins.js b/scripts/check_plugins.js index fa5e2edb..ac7a2aa2 100644 --- a/scripts/check_plugins.js +++ b/scripts/check_plugins.js @@ -11,22 +11,22 @@ const required = ['analysis-icu']; let failures = []; cli.header("checking elasticsearch plugins"); -client.nodes.info(null, (err, res) => { +client.nodes.info((err, { body }) => { if( err ){ console.error(err); process.exit(1); } - if( !res || !res.nodes ){ + if( !body || !body.nodes ){ console.error("no nodes found"); process.exit(1); } // iterate over all nodes in cluster - for( const uid in res.nodes ){ + for( const uid in body.nodes ){ - const node = res.nodes[uid]; + const node = body.nodes[uid]; // Amazon's hosted Elasticsearch does not have the plugins property // but has the plugins we need diff --git a/scripts/check_version.js b/scripts/check_version.js index bae503bb..9e0d76b6 100644 --- a/scripts/check_version.js +++ b/scripts/check_version.js @@ -13,14 +13,13 @@ if(!targetVersion){ } cli.header(`checking elasticsearch server version matches "${targetVersion}"`); -client.info(null, (err, res) => { - +client.info((err, { body }) => { if (err) { console.error(err); process.exit(1); } - const version = _.get(res, 'version.number', '0.0.0'); + const version = _.get(body, 'version.number', '0.0.0'); // pretty print error message if (!semver.satisfies(version, targetVersion)) { diff --git a/scripts/drop_index.js b/scripts/drop_index.js index a47be7c7..0c41f56e 100644 --- a/scripts/drop_index.js +++ b/scripts/drop_index.js @@ -19,8 +19,11 @@ function drop() { // check all hosts to see if any is not localhost function warnIfNotLocal() { - if (config.esclient.hosts.some((env) => { return env.host !== 'localhost'; })) { - console.log(colors.red(`WARNING: DROPPING SCHEMA NOT ON LOCALHOST: ${config.esclient.hosts[0].host}`)); + for (const idx in config.esclient.nodes) { + const node = config.esclient.nodes[idx]; + if (!node.includes('://localhost')) { + console.log(colors.red(`WARNING: DROPPING SCHEMA NOT ON LOCALHOST: ${node}`)); + } } } @@ -39,4 +42,4 @@ function fail() { function isForced() { return process.argv.length > 2 && ['--force-yes', '-f'].indexOf(process.argv[2]) > -1; -} \ No newline at end of file +} From 87019dc8c191e35a83b7e06712fdbeb24533b145 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 18:31:28 -0700 Subject: [PATCH 6/8] adapt integration tests to new config We can just use the default here. --- integration/run.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/integration/run.js b/integration/run.js index 4c906792..00686dec 100644 --- a/integration/run.js +++ b/integration/run.js @@ -1,15 +1,9 @@ const _ = require('lodash'); const tape = require('tape'); -const config = require('pelias-config').generate(); const schema = require('../schema'); const common = { - clientOpts: { - host: 'localhost:9200', - keepAlive: true, - apiVersion: config.esclient.apiVersion - }, create: { schema: schema, create: { } From 9090ebab467f6119b2919ace6182c4ca54c258bb Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 18:56:41 -0700 Subject: [PATCH 7/8] Adapt integration suite to new elasticsearch package - On success, elasticsearch errors are now *null* rather than *undefined*, which broke some tests. Now we just check that `err` is falsy rather than strictly equal to `undefined`. - Payload moved from res to res.body - `statusCode` moved from top level callback arg to `res.statuCode` - normalized function formatting for tests to `(err, { body })` there were a few different styles/indentations. --- integration/address_matching.js | 62 ++++++++-------- integration/admin_abbreviations.js | 72 +++++++++---------- integration/admin_matching.js | 18 ++--- integration/analyzer_peliasPhrase.js | 14 ++-- .../autocomplete_abbreviated_street_names.js | 18 ++--- ...ocomplete_directional_synonym_expansion.js | 36 +++++----- .../autocomplete_street_synonym_expansion.js | 36 +++++----- integration/bounding_box.js | 6 +- integration/dynamic_templates.js | 17 +++-- integration/run.js | 6 +- .../source_layer_sourceid_filtering.js | 28 ++++---- integration/validate.js | 4 +- 12 files changed, 158 insertions(+), 159 deletions(-) diff --git a/integration/address_matching.js b/integration/address_matching.js index 4ea5e23d..59d508e6 100644 --- a/integration/address_matching.js +++ b/integration/address_matching.js @@ -68,9 +68,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.number': 30 } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match street number' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'match street number' ); done(); }); }); @@ -82,9 +82,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match_phrase: { 'address_parts.street': 'west 26th street' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 2, 'match street name' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 2, 'match street name' ); done(); }); }); @@ -96,9 +96,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match_phrase: { 'address_parts.street': 'W 26th ST' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 2, 'match street name - abbr' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 2, 'match street name - abbr' ); done(); }); }); @@ -110,9 +110,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.zip': '10010' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 3, 'match zip - numeric' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 3, 'match zip - numeric' ); done(); }); }); @@ -124,9 +124,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.zip': 'e24dn' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match zip - string' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'match zip - string' ); done(); }); }); @@ -138,9 +138,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.zip': '100-10' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 3, 'match zip - numeric - punct' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 3, 'match zip - numeric - punct' ); done(); }); }); @@ -152,9 +152,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.zip': '10 0 10' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 3, 'match zip - numeric - whitespace' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 3, 'match zip - numeric - whitespace' ); done(); }); }); @@ -166,9 +166,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.zip': 'E2-4DN' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match zip - string - punct' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'match zip - string - punct' ); done(); }); }); @@ -180,9 +180,9 @@ module.exports.tests.functional = function(test, common){ body: { query: { bool: { must: [ { match: { 'address_parts.zip': 'E2 4DN' } } ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match zip - string - whitespace' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'match zip - string - whitespace' ); done(); }); }); @@ -289,10 +289,10 @@ module.exports.tests.venue_vs_address = function(test, common){ } } } - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), TOTAL_ADDRESS_DOCS+1, 'matched all docs' ); - t.equal( res.hits.hits[TOTAL_ADDRESS_DOCS]._id, '1', 'exact name match first' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), TOTAL_ADDRESS_DOCS+1, 'matched all docs' ); + t.equal( body.hits.hits[TOTAL_ADDRESS_DOCS]._id, '1', 'exact name match first' ); done(); }); }); diff --git a/integration/admin_abbreviations.js b/integration/admin_abbreviations.js index 20369180..666404c8 100644 --- a/integration/admin_abbreviations.js +++ b/integration/admin_abbreviations.js @@ -55,10 +55,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -77,10 +77,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -99,10 +99,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -121,10 +121,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -177,10 +177,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -199,10 +199,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -221,10 +221,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -243,10 +243,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); @@ -311,10 +311,10 @@ module.exports.tests.synonyms = function (test, common) { } } } - }, (err, res) => { - t.equal(err, undefined); - t.equal(getTotalHits(res.hits), 2, 'matches both documents'); - t.equal(res.hits.hits[0]._score, res.hits.hits[1]._score, 'scores match'); + }, (err, { body }) => { + t.false(err); + t.equal(getTotalHits(body.hits), 2, 'matches both documents'); + t.equal(body.hits.hits[0]._score, body.hits.hits[1]._score, 'scores match'); done(); }); }); diff --git a/integration/admin_matching.js b/integration/admin_matching.js index 7d1de3b0..ead3dc45 100644 --- a/integration/admin_matching.js +++ b/integration/admin_matching.js @@ -46,9 +46,9 @@ module.exports.tests.functional = function(test, common){ suite.client.search({ index: suite.props.index, body: { query: { match: { 'parent.country': 'Test Country' } } } - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -58,9 +58,9 @@ module.exports.tests.functional = function(test, common){ suite.client.search({ index: suite.props.index, body: { query: { match: { 'parent.country_a': 'TestCountry' } } } - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -70,9 +70,9 @@ module.exports.tests.functional = function(test, common){ suite.client.search({ index: suite.props.index, body: { query: { match: { 'parent.country_id': '100' } } } - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); diff --git a/integration/analyzer_peliasPhrase.js b/integration/analyzer_peliasPhrase.js index 11828cd6..d207ee04 100644 --- a/integration/analyzer_peliasPhrase.js +++ b/integration/analyzer_peliasPhrase.js @@ -215,12 +215,12 @@ module.exports.tests.slop_query = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: buildQuery('Lake Cayuga') - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 3 ); - var hits = res.hits.hits; + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 3 ); + var hits = body.hits.hits; t.equal( hits[0]._source.name.default, 'Lake Cayuga' ); @@ -273,9 +273,9 @@ module.exports.tests.slop = function(test, common){ 'slop': 3, } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); diff --git a/integration/autocomplete_abbreviated_street_names.js b/integration/autocomplete_abbreviated_street_names.js index fde4add4..678842c0 100644 --- a/integration/autocomplete_abbreviated_street_names.js +++ b/integration/autocomplete_abbreviated_street_names.js @@ -36,9 +36,9 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm 'query': 'Grolmanstr.' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -78,9 +78,9 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm // 'query': 'Grolmanstraße' // } // }}} -// }, function( err, res ){ -// t.equal( err, undefined ); -// t.equal( getTotalHits(res.hits), 1, 'document found' ); +// }, (err, { body }) => { +// t.false(err); +// t.equal( getTotalHits(body.hits), 1, 'document found' ); // done(); // }); // }); @@ -98,9 +98,9 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm // 'query': 'Grolmanstraße' // } // }}} -// }, function( err, res ){ -// t.equal( err, undefined ); -// t.equal( getTotalHits(res.hits), 1, 'document found' ); +// }, (err, { body }) => { +// t.false(err); +// t.equal( getTotalHits(body.hits), 1, 'document found' ); // done(); // }); // }); diff --git a/integration/autocomplete_directional_synonym_expansion.js b/integration/autocomplete_directional_synonym_expansion.js index 4f409ae6..2496376a 100644 --- a/integration/autocomplete_directional_synonym_expansion.js +++ b/integration/autocomplete_directional_synonym_expansion.js @@ -36,9 +36,9 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'query': 'nor' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -53,9 +53,9 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'query': 'north' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -90,9 +90,9 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) 'query': 'n' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -127,9 +127,9 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'query': 'nor' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -144,9 +144,9 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'query': 'north' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -181,9 +181,9 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ 'query': 'n' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); diff --git a/integration/autocomplete_street_synonym_expansion.js b/integration/autocomplete_street_synonym_expansion.js index c2c21e6f..ff789eb2 100644 --- a/integration/autocomplete_street_synonym_expansion.js +++ b/integration/autocomplete_street_synonym_expansion.js @@ -37,9 +37,9 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'query': 'cent' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -54,9 +54,9 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'query': 'center' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -91,9 +91,9 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) 'query': 'ctr' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -128,9 +128,9 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'query': 'cent' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -145,9 +145,9 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'query': 'center' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); @@ -182,9 +182,9 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ 'query': 'ctr' } }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); + }, (err, { body }) => { + t.false(err); + t.equal( getTotalHits(body.hits), 1, 'document found' ); done(); }); }); diff --git a/integration/bounding_box.js b/integration/bounding_box.js index 21e5c864..e7413f85 100644 --- a/integration/bounding_box.js +++ b/integration/bounding_box.js @@ -29,9 +29,9 @@ module.exports.tests.index_and_retrieve = function(test, common){ index: suite.props.index, id: '1' }, - function (err, res) { - t.equal(err, undefined); - t.deepEqual(res._source.bounding_box, '{"min_lat":-47.75,"max_lat":-33.9,"min_lon":163.82,"max_lon":179.42}'); + function (err, { body }) { + t.false(err); + t.deepEqual(body._source.bounding_box, '{"min_lat":-47.75,"max_lat":-33.9,"min_lon":163.82,"max_lon":179.42}'); done(); } ); diff --git a/integration/dynamic_templates.js b/integration/dynamic_templates.js index 6e64a09c..f0804e8c 100644 --- a/integration/dynamic_templates.js +++ b/integration/dynamic_templates.js @@ -48,9 +48,8 @@ function nameAssertion( analyzer, common ){ suite.client.indices.getMapping({ index: suite.props.index, - }, (err, res) => { - - const properties = res[suite.props.index].mappings.properties; + }, (err, { body }) => { + const properties = body[suite.props.index].mappings.properties; t.equal( properties.name.dynamic, 'true' ); const nameProperties = properties.name.properties; @@ -84,9 +83,9 @@ function phraseAssertion( analyzer, common ){ suite.client.indices.getMapping({ index: suite.props.index, - }, ( err, res ) => { + }, ( err, { body }) => { - const properties = res[suite.props.index].mappings.properties; + const properties = body[suite.props.index].mappings.properties; t.equal( properties.phrase.dynamic, 'true' ); const phraseProperties = properties.phrase.properties; @@ -119,9 +118,9 @@ function addendumAssertion( namespace, value, common ){ suite.assert( done => { suite.client.indices.getMapping({ index: suite.props.index, - }, ( err, res ) => { + }, ( err, { body }) => { - const properties = res[suite.props.index].mappings.properties; + const properties = body[suite.props.index].mappings.properties; t.equal( properties.addendum.dynamic, 'true' ); const addendumProperties = properties.addendum.properties; @@ -148,9 +147,9 @@ function addendumAssertion( namespace, value, common ){ suite.client.get({ index: suite.props.index, id: 1 - }, ( err, res ) => { + }, ( err, { body }) => { t.false( err ); - t.equal( res._source.addendum[namespace], value ); + t.equal( body._source.addendum[namespace], value ); done(); }); }); diff --git a/integration/run.js b/integration/run.js index 00686dec..26a1c396 100644 --- a/integration/run.js +++ b/integration/run.js @@ -9,7 +9,7 @@ const common = { create: { } }, summaryMap: (res) => { - return res.hits.hits.map(h => { + return res.body.hits.hits.map(h => { return { _id: h._id, _score: h._score, @@ -63,10 +63,10 @@ const common = { analyzer: analyzer, text: text.toString() } - }, (err, res) => { + }, (err, { body }) => { if (err) { console.error(err); } t.deepEqual({}, removeIndexTokensFromExpectedTokens( - common.bucketTokens(res.tokens), + common.bucketTokens(body.tokens), common.bucketTokens(expected) ), comment); done(); diff --git a/integration/source_layer_sourceid_filtering.js b/integration/source_layer_sourceid_filtering.js index 5b3e5cb5..731a2692 100644 --- a/integration/source_layer_sourceid_filtering.js +++ b/integration/source_layer_sourceid_filtering.js @@ -50,8 +50,8 @@ module.exports.tests.source_filter = function(test, common){ source: 'osm' } }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 2 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 2 ); done(); }); }); @@ -65,8 +65,8 @@ module.exports.tests.source_filter = function(test, common){ layer: 'address' } }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 2 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 2 ); done(); }); }); @@ -80,8 +80,8 @@ module.exports.tests.source_filter = function(test, common){ source_id: 'dataset/1' } }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 2 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 2 ); done(); }); }); @@ -94,8 +94,8 @@ module.exports.tests.source_filter = function(test, common){ { term: { source: 'osm' } }, { term: { source_id: 'dataset/1' } } ]}}} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 1 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 1 ); done(); }); }); @@ -109,8 +109,8 @@ module.exports.tests.source_filter = function(test, common){ source: 'OSM' } }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 0 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 0 ); done(); }); }); @@ -124,8 +124,8 @@ module.exports.tests.source_filter = function(test, common){ source: 'foo' } }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 0 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 0 ); done(); }); }); @@ -139,8 +139,8 @@ module.exports.tests.source_filter = function(test, common){ source: 'foo bar baz' } }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 1 ); + }, (err, { body }) => { + t.equal( getTotalHits(body.hits), 1 ); done(); }); }); diff --git a/integration/validate.js b/integration/validate.js index 31622e98..a850cadd 100644 --- a/integration/validate.js +++ b/integration/validate.js @@ -11,8 +11,8 @@ module.exports.tests.validate = function(test, common){ var suite = new Suite( common.clientOpts, common.create ); suite.assert( done => { - suite.client.info({}, ( err, res, status ) => { - t.equal( status, 200 ); + suite.client.info(( err, { statusCode }) => { + t.equal( statusCode, 200 ); done(); }); }); From 811b45fdcb8877edfa90c4e55b84410e516f7158 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 29 Jun 2023 13:19:59 -0700 Subject: [PATCH 8/8] stdout logger with pelias-elasticsearch The legacy elasticsearch client had an integrated stdout logger. The new elasticsearch client does not, so instead we use the pelias-elasticsearch which integrates the modern elasticsearch client with pelias-logger. --- package.json | 2 +- scripts/check_plugins.js | 4 ++-- scripts/check_version.js | 4 ++-- scripts/create_index.js | 4 ++-- scripts/drop_index.js | 4 ++-- scripts/info.js | 4 ++-- scripts/update_settings.js | 4 ++-- test/elastictest/Suite.js | 4 ++-- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index af2c5a41..79d64fd1 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "dependencies": { "@hapi/joi": "^16.1.8", "colors": "^1.1.2", - "@elastic/elasticsearch": "~7.17.0", + "pelias-elasticsearch": "https://github.com/michaelkirk-pelias/pelias-elasticsearch#mkirk\/elastic8", "glob": "^7.1.6", "lodash": "^4.17.15", "pelias-config": "https://github.com/michaelkirk-pelias/config#mkirk\/elastic8", diff --git a/scripts/check_plugins.js b/scripts/check_plugins.js index ac7a2aa2..8b23db65 100644 --- a/scripts/check_plugins.js +++ b/scripts/check_plugins.js @@ -1,7 +1,7 @@ const colors = require('colors/safe'); const config = require('pelias-config').generate(); -const es = require('@elastic/elasticsearch'); -const client = new es.Client(config.esclient); +const buildClient = require('pelias-elasticsearch'); +const client = buildClient(config.esclient); const cli = require('./cli'); // mandatory plugins diff --git a/scripts/check_version.js b/scripts/check_version.js index 9e0d76b6..77e0894d 100644 --- a/scripts/check_version.js +++ b/scripts/check_version.js @@ -1,8 +1,8 @@ const _ = require('lodash'); const semver = require('semver'); -const es = require('@elastic/elasticsearch'); +const buildClient = require('pelias-elasticsearch'); const config = require('pelias-config').generate(); -const client = new es.Client(config.esclient); +const client = buildClient(config.esclient); const cli = require('./cli'); // pass target elastic version semver as the first CLI arg diff --git a/scripts/create_index.js b/scripts/create_index.js index 93b68b57..9cca702f 100644 --- a/scripts/create_index.js +++ b/scripts/create_index.js @@ -1,6 +1,6 @@ const child_process = require('child_process'); const config = require('pelias-config').generate(); -const es = require('@elastic/elasticsearch'); +const buildClient = require('pelias-elasticsearch'); const SUPPORTED_ES_VERSIONS = '>=7.4.2'; const cli = require('./cli'); @@ -8,7 +8,7 @@ const schema = require('../schema'); cli.header("create index"); -const client = new es.Client(config.esclient); +const client = buildClient(config.esclient); // check minimum elasticsearch versions before continuing try { diff --git a/scripts/drop_index.js b/scripts/drop_index.js index 0c41f56e..91e1547f 100644 --- a/scripts/drop_index.js +++ b/scripts/drop_index.js @@ -1,7 +1,7 @@ const colors = require('colors/safe'); const config = require('pelias-config').generate(); -const es = require('@elastic/elasticsearch'); -const client = new es.Client(config.esclient); +const buildClient = require('pelias-elasticsearch'); +const client = buildClient(config.esclient); const readline = require('readline'); const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); diff --git a/scripts/info.js b/scripts/info.js index 62504c6e..26d0788b 100644 --- a/scripts/info.js +++ b/scripts/info.js @@ -1,5 +1,5 @@ var config = require('pelias-config').generate().esclient; -var es = require('@elastic/elasticsearch'); -var client = new es.Client(config); +var buildClient = require('pelias-elasticsearch'); +var client = buildClient(config); client.info( {}, console.log.bind(console) ); diff --git a/scripts/update_settings.js b/scripts/update_settings.js index b2ae69b8..1dbdd633 100644 --- a/scripts/update_settings.js +++ b/scripts/update_settings.js @@ -1,6 +1,6 @@ var config = require('pelias-config').generate(); -var es = require('@elastic/elasticsearch'); -var client = new es.Client(config.esclient); +var buildClient = require('pelias-elasticsearch'); +var client = buildClient(config.esclient); var schema = require('../schema'); var _index = config.schema.indexName; diff --git a/test/elastictest/Suite.js b/test/elastictest/Suite.js index 6a52ba68..8349629a 100644 --- a/test/elastictest/Suite.js +++ b/test/elastictest/Suite.js @@ -1,6 +1,6 @@ const _ = require('lodash') const randomstring = require('randomstring') -const elasticsearch = require('elasticsearch') +const buildClient = require('pelias-elasticsearch'); const async = require('async') function Suite (clientOpts, props) { @@ -26,7 +26,7 @@ Suite.prototype.assert = function (assert) { } Suite.prototype.start = function (cb) { - this.client = new elasticsearch.Client(this.clientOpts) + this.client = buildClient(this.clientOpts) cb() }