Merge pull request #295 from pelias/staging

Merge staging into production

.jshintrc

@@ -0,0 +1,20 @@
+{
+  "node": true,
+  "curly": true,
+  "eqeqeq": true,
+  "esversion": 6,
+  "freeze": true,
+  "immed": true,
+  "indent": 2,
+  "latedef": false,
+  "newcap": true,
+  "noarg": true,
+  "noempty": true,
+  "nonbsp": true,
+  "nonew": true,
+  "plusplus": false,
+  "undef": true,
+  "unused": false,
+  "maxparams": 4,
+  "maxdepth": 4
+}

.travis.yml

@@ -3,15 +3,14 @@ language: node_js
 notifications:
   email: false
 node_js:
-  - 4
   - 6
+  - 8
+  - 10
 matrix:
   fast_finish: true
 env:
-  global:
-    - BUILD_LEADER_ID=2
   matrix:
-    - ES_VERSION=2.3.4
+    - ES_VERSION=2.4.6
 jdk:
   - oraclejdk8
 install:
@@ -27,12 +26,13 @@ addons:
     packages:
       - oracle-java8-installer
 before_install:
-  - npm i -g npm@^3.0.0
+  - npm i -g npm
 before_script:
-  - npm prune
   - sleep 10
   - curl http://127.0.0.1:9200/
   - node scripts/create_index.js -y
 after_success:
-  - npm install -g npx
-  - npx -p node@8 npm run semantic-release
+  - npx semantic-release
+branches:
+  except:
+    - /^v\d+\.\d+\.\d+$/

Dockerfile

@@ -5,11 +5,9 @@ FROM pelias/baseimage
 ENV WORKDIR /code/pelias/schema
 WORKDIR ${WORKDIR}
 
-# add code from local checkout to image
-ADD . ${WORKDIR}
-
-# install npm dependencies
+# copy package.json first to prevent npm install being rerun when only code changes
+COPY ./package.json ${WORK}
 RUN npm install
 
-# run tests
-RUN npm test
+# add code from local checkout to image
+ADD . ${WORKDIR}

README.md

@@ -1,12 +1,21 @@
-## Installation
+>This repository is part of the [Pelias](https://github.com/pelias/pelias)
+>project. Pelias is an open-source, open-data geocoder originally sponsored by
+>[Mapzen](https://www.mapzen.com/). Our official user documentation is
+>[here](https://github.com/pelias/documentation).
+
+# Pelias Elasticsearch Schema Definition
+
+This package defines the Elasticsearch schema used by Pelias. Pelias requires quite a few settings for performance and accuracy. This repository contains those settings as well as useful tools to ensure they are applied correctly.
 
 [![Greenkeeper badge](https://badges.greenkeeper.io/pelias/schema.svg)](https://greenkeeper.io/)
+[![NPM](https://nodei.co/npm/pelias-schema.png?downloads=true&stars=true)](https://nodei.co/npm/pelias-schema)
+[![Build Status](https://travis-ci.org/pelias/schema.png?branch=master)](https://travis-ci.org/pelias/schema)
+## Installation
 
 ```bash
 $ npm install pelias-schema
 ```
 
-[![NPM](https://nodei.co/npm/pelias-schema.png?downloads=true&stars=true)](https://nodei.co/npm/pelias-schema)
 
 ## Usage
 
@@ -58,6 +67,25 @@ Print a list of which plugins are installed and how to install any that are miss
 node scripts/check_plugins.js;
 ```
 
+#### user customizable synonyms files
+
+You may provide your own custom synonyms by editing files in the `./synonyms/` directory.
+
+```bash
+$ ls -1 synonyms/custom_*
+synonyms/custom_admin.txt
+synonyms/custom_name.txt
+synonyms/custom_street.txt
+```
+
+You must edit the files **before** running `create_index.js`, any changes made to the files will require you to drop and recreate the index before those synonyms are available.
+
+Synonyms are only used at index-time. The filename contains the name of the elasticsearch field which the synonyms will apply. ie. `custom_name` will apply to the `name.*` fields, `custom_street` will apply to the `address_parts.name` field and `custom_admin` will apply to the `parent.*` fields.
+
+see: https://github.com/pelias/schema/pull/273 for more info.
+
+With great power comes great responsibility. Synonyms files are often used as a hammer when a scalpel is required. Please take care with their use and make maintainers aware that you are using custom synonyms when you open support tickets.
+
 ## NPM Module
 
 The `pelias-schema` npm module can be found here:
@@ -145,6 +173,4 @@ The restarted server should now pass the `node scripts/check_plugins.js` check,
 
 ### Continuous Integration
 
-Travis tests every release against Node.js versions 4` and `6`.
-
-[![Build Status](https://travis-ci.org/pelias/schema.png?branch=master)](https://travis-ci.org/pelias/schema)
+Travis tests every release against all supported Node.js versions.

integration/analyzer_peliasAdmin.js

@@ -65,9 +65,32 @@ module.exports.tests.functional = function(test, common){
       'paskalomatunturi'
     ]);
 
+    suite.run( t.end );
+  });
+};
+
+module.exports.tests.synonyms = function(test, common){
+  test( 'synonyms', function(t){
+
+    var suite = new elastictest.Suite( null, { schema: schema } );
+    var assertAnalysis = analyze.bind( null, suite, t, 'peliasAdmin' );
+    suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
+
     assertAnalysis( 'place', 'Saint-Louis-du-Ha! Ha!', [
-      'saint', 'louis', 'du', 'ha', 'ha'
-    ]);
+      '0:saint', '0:st', '1:louis', '2:du', '3:ha', '4:ha'
+    ], true);
+
+    assertAnalysis( 'place', 'Sainte-Chapelle', [
+      '0:sainte', '0:ste', '1:chapelle'
+    ], true);
+
+    assertAnalysis( 'place', 'Mount Everest', [
+      '0:mount', '0:mt', '1:everest'
+    ], true);
+
+    assertAnalysis( 'place', 'Mont Blanc', [
+      '0:mont', '0:mt', '1:blanc'
+    ], true);
 
     suite.run( t.end );
   });
@@ -114,14 +137,14 @@ module.exports.tests.unicode = function(test, common){
 
     // Chambéry (both forms appear the same)
     var composed = "Chamb" + latin_small_letter_e_with_acute + "ry";
-    var decomposed = "Chamb" + combining_acute_accent + latin_small_letter_e + "ry"
+    var decomposed = "Chamb" + combining_acute_accent + latin_small_letter_e + "ry";
 
     assertAnalysis( 'composed', composed, ['chambery'] );
     assertAnalysis( 'decomposed', decomposed, ['chambery'] );
 
     // Één (both forms appear the same)
-    var composed = latin_large_letter_e_with_acute + latin_small_letter_e_with_acute + "n";
-    var decomposed = combining_acute_accent + latin_large_letter_e + combining_acute_accent + latin_small_letter_e + "n"
+    composed = latin_large_letter_e_with_acute + latin_small_letter_e_with_acute + "n";
+    decomposed = combining_acute_accent + latin_large_letter_e + combining_acute_accent + latin_small_letter_e + "n";
 
     assertAnalysis( 'composed', composed, ['een'] );
     assertAnalysis( 'decomposed', decomposed, ['een'] );
@@ -141,22 +164,22 @@ module.exports.all = function (tape, common) {
   }
 };
 
-function analyze( suite, t, analyzer, comment, text, expected ){
+function analyze( suite, t, analyzer, comment, text, expected, includePosition ){
   suite.assert( function( done ){
     suite.client.indices.analyze({
       index: suite.props.index,
       analyzer: analyzer,
       text: text
     }, function( err, res ){
-      if( err ) console.error( err );
-      t.deepEqual( simpleTokens( res.tokens ), expected, comment );
+      if( err ){ console.error( err ); }
+      t.deepEqual( simpleTokens( res.tokens, includePosition ), expected, comment );
       done();
     });
   });
 }
 
-function simpleTokens( tokens ){
+function simpleTokens( tokens, includePosition ){
   return tokens.map( function( t ){
-    return t.token;
+    return (!!includePosition ? t.position + ':' : '') + t.token;
   });
 }

mappings/document.js

@@ -51,6 +51,15 @@ var schema = {
       type: 'object',
       dynamic: true,
       properties: {
+        // https://github.com/whosonfirst/whosonfirst-placetypes#continent
+        continent: admin,
+        continent_a: admin,
+        continent_id: literal,
+
+        // https://github.com/whosonfirst/whosonfirst-placetypes#empire
+        empire: admin,
+        empire_a: admin,
+        empire_id: literal,
 
         // https://github.com/whosonfirst/whosonfirst-placetypes#country
         country: admin,

package.json

@@ -1,7 +1,7 @@
 {
   "name": "pelias-schema",
-  "version": "0.0.0-semantic-release",
-  "author": "mapzen",
+  "version": "0.0.0-development",
+  "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",
   "license": "MIT",
@@ -13,9 +13,7 @@
     "drop_index": "node scripts/drop_index",
     "reset_type": "node scripts/reset_type",
     "update_settings": "node scripts/update_settings",
-    "travis": "npm run check-dependencies && npm run test",
-    "semantic-release": "semantic-release pre && npm publish && semantic-release post",
-    "check-dependencies": "node_modules/.bin/npm-check --production"
+    "travis": "npm run test"
   },
   "repository": {
     "type": "git",
@@ -29,27 +27,29 @@
     "url": "https://github.com/pelias/schema/issues"
   },
   "engines": {
-    "node": ">=4.0.0",
+    "node": ">=6.0.0",
     "npm": ">=1.4.3",
     "elasticsearch": ">=1.1.1"
   },
   "dependencies": {
     "colors": "^1.1.2",
-    "elasticsearch": "^13.3.1",
-    "joi": "^12.0.0",
+    "elasticsearch": "^14.0.0",
+    "joi": "^13.1.0",
     "lodash.merge": "^4.6.0",
-    "pelias-config": "2.13.0"
+    "pelias-config": "2.14.0"
   },
   "devDependencies": {
     "difflet": "^1.0.1",
     "elastictest": "^1.2.0",
-    "npm-check": "git://github.com/orangejulius/npm-check.git#disable-update-check",
-    "proxyquire": "^1.7.10",
-    "semantic-release": "^8.0.0",
+    "proxyquire": "^2.0.0",
+    "semantic-release": "^15.0.0",
     "tap-spec": "^4.1.1",
     "tape": "^4.5.0"
   },
   "pre-commit": [
-    "check-dependencies"
-  ]
+  ],
+  "release": {
+    "branch": "production",
+    "success": []
+  }
 }

scripts/drop_index.js

@@ -20,7 +20,7 @@ function drop(){
 // check all hosts to see if any is not localhost
 function warnIfNotLocal() {
   if (config.esclient.hosts.some((env) => { return env.host !== 'localhost'; } )) {
-    console.log(colors.red("WARNING: DROPPING SCHEMA NOT ON LOCALHOST"));
+    console.log(colors.red(`WARNING: DROPPING SCHEMA NOT ON LOCALHOST: ${config.esclient.hosts[0].host}`));
   }
 
 }