Skip to content

Commit

Permalink
Merge pull request #28 from ibi-group/rebuild
Browse files Browse the repository at this point in the history 
Support Infinite Geocoders, Support Offline Geocoder
  • Loading branch information
@miles-grant-ibigroup
miles-grant-ibigroup authored Apr 2, 2024
2 parents 942ff03 + 8bff9d9 commit 4623ba9
Show file tree
Hide file tree
Showing 6 changed files with 113 additions and 386 deletions.
16 changes: 2 additions & 14 deletions env.example.yml
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,5 @@
LAMBDA_EXEC_SG: Insert AWS Security Group ID Here (it must be in the same VPC as the subnet)
LAMBDA_EXEC_SUBNET: Insert AWS Subnet ID Here (it must be in the same VPC as the security group)
BUGSNAG_NOTIFIER_KEY: INSERT BUGSNAG NOTIFIER KEY HERE
GEOCODER_API_KEY: INSERT API KEY HERE
GEOCODE_EARTH_URL: https://api.geocode.earth/v1 # Not needed if geocode.earth is not used
CSV_ENABLED: true
CUSTOM_PELIAS_URL: http://<insert your Pelias endpoint here>/v1
GEOCODER: HERE # Options: HERE, PELIAS
TRANSIT_GEOCODER: (optional) OTP/PELIAS
TRANSIT_BASE_URL: (conditionally required) OTP instance when TRANSIT_GEOCODER=OTP (/otp/routers/{routerId})

SECONDARY_GEOCODER: (optional) HERE/PELIAS
SECONDARY_GEOCODER_API_KEY: (optional) INSERT SECONDARY API KEY HERE
SECONDARY_GEOCODE_EARTH_URL: (optional) https://api.geocode.earth/v1 # Not needed if geocode.earth is not used

REDIS_HOST: (optional) <insert IP of redis host here>
REDIS_KEY: (optional) <insert redis password here>
GEOCODERS: <Stringified JSON Array of OTP-UI `GeocoderConfig`s>
BACKUP_GEOCODERS: <Stringified JSON Array of OTP-UI `GeocoderConfig`'s. Same length and order as GEOCODERS>
228 changes: 76 additions & 152 deletions handler.ts
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
/**
* This script contains the AWS Lambda handler code for merging two Pelias instances
* together.
* This script contains the AWS Lambda handler code for merging some number of geocoder instances
* together
* Dependencies are listed in package.json in the same folder.
* Notes:
* - Most of the folder contents is uploaded to AWS Lambda (see README.md for deploying).
*/
import { URLSearchParams } from 'url'

import Bugsnag from '@bugsnag/js'
import getGeocoder from '@opentripplanner/geocoder'
import { createCluster } from 'redis'
import type { FeatureCollection } from 'geojson'
import { Geometry, FeatureCollection, GeoJsonProperties } from 'geojson'
import { OfflineResponse } from '@opentripplanner/geocoder/lib/apis/offline'

import {
cachedGeocoderRequest,
checkIfResultsAreSatisfactory,
convertQSPToGeocoderArgs,
fetchPelias,
makeQueryPeliasCompatible,
mergeResponses,
ServerlessCallbackFunction,
Expand All @@ -26,56 +23,37 @@ import {

// This plugin must be imported via cjs to ensure its existence (typescript recommendation)
const BugsnagPluginAwsLambda = require('@bugsnag/plugin-aws-lambda')
const {
BUGSNAG_NOTIFIER_KEY,
CSV_ENABLED,
GEOCODE_EARTH_URL,
GEOCODER,
GEOCODER_API_KEY,
REDIS_HOST,
REDIS_KEY,
SECONDARY_GEOCODE_EARTH_URL,
SECONDARY_GEOCODER,
SECONDARY_GEOCODER_API_KEY,
TRANSIT_BASE_URL,
TRANSIT_GEOCODER
} = process.env

// Severless... why!
const redis =
!!REDIS_HOST && REDIS_HOST !== 'null'
? createCluster({
rootNodes: [
{
password: REDIS_KEY,
url: 'redis://' + REDIS_HOST
}
],
useReplicas: true
})
: null
if (redis) redis.on('error', (err) => console.log('Redis Client Error', err))
const { BACKUP_GEOCODERS, BUGSNAG_NOTIFIER_KEY, GEOCODERS, POIS } = process.env

// Ensure env variables have been set
if (
typeof TRANSIT_BASE_URL !== 'string' ||
typeof GEOCODER_API_KEY !== 'string' ||
typeof BUGSNAG_NOTIFIER_KEY !== 'string' ||
typeof GEOCODER !== 'string'
) {
if (!GEOCODERS) {
throw new Error(
'Error: required configuration variables not found! Ensure env.yml has been decrypted.'
'Error: required configuration variable GEOCODERS not found! Ensure env.yml has been decrypted.'
)
}
const geocoders = JSON.parse(GEOCODERS)
const backupGeocoders = BACKUP_GEOCODERS && JSON.parse(BACKUP_GEOCODERS)
// Serverless is not great about null
const pois =
POIS && POIS !== 'null'
? (JSON.parse(POIS) as OfflineResponse).map((poi) => {
if (typeof poi.lat === 'string') {
poi.lat = parseFloat(poi.lat)
}
if (typeof poi.lon === 'string') {
poi.lon = parseFloat(poi.lon)
}
return poi
})
: []

if (CSV_ENABLED === 'true' && TRANSIT_GEOCODER === 'OTP') {
if (geocoders.length !== backupGeocoders.length) {
throw new Error(
'Error: Invalid configuration. OTP Geocoder does not support CSV_ENABLED.'
'Error: BACKUP_GEOCODERS is not set to the same length as GEOCODERS'
)
}

Bugsnag.start({
apiKey: BUGSNAG_NOTIFIER_KEY,
apiKey: BUGSNAG_NOTIFIER_KEY || '',
appType: 'pelias-stitcher-lambda-function',
appVersion: require('./package.json').version,
plugins: [BugsnagPluginAwsLambda],
Expand All @@ -84,52 +62,7 @@ Bugsnag.start({
// This handler will wrap around the handler code
// and will report exceptions to Bugsnag automatically.
// For reference, see https://docs.bugsnag.com/platforms/javascript/aws-lambda/#usage
const bugsnagHandler = Bugsnag.getPlugin('awsLambda').createHandler()

const getPrimaryGeocoder = () => {
if (GEOCODER === 'PELIAS' && typeof GEOCODE_EARTH_URL !== 'string') {
throw new Error('Error: Geocode earth URL not set.')
}
return getGeocoder({
apiKey: GEOCODER_API_KEY,
baseUrl: GEOCODE_EARTH_URL,
reverseUseFeatureCollection: true,
type: GEOCODER
})
}

const getTransitGeocoder = () => {
if (TRANSIT_GEOCODER === 'OTP') {
return getGeocoder({
baseUrl: TRANSIT_BASE_URL,
type: TRANSIT_GEOCODER
})
}

return null
}

const getSecondaryGeocoder = () => {
if (!SECONDARY_GEOCODER || !SECONDARY_GEOCODER_API_KEY) {
console.warn('Not using secondary Geocoder')
return false
}

if (
SECONDARY_GEOCODER === 'PELIAS' &&
typeof SECONDARY_GEOCODE_EARTH_URL !== 'string'
) {
throw new Error(
'Secondary geocoder configured incorrectly: Geocode.earth URL not set.'
)
}
return getGeocoder({
apiKey: SECONDARY_GEOCODER_API_KEY,
baseUrl: SECONDARY_GEOCODE_EARTH_URL,
reverseUseFeatureCollection: true,
type: SECONDARY_GEOCODER
})
}
const bugsnagHandler = Bugsnag?.getPlugin('awsLambda')?.createHandler()

/**
* Makes a call to a Pelias Instance using secrets from the config file.
Expand All @@ -156,57 +89,51 @@ export const makeGeocoderRequests = async (
const peliasQSP = { ...event.queryStringParameters }
delete peliasQSP.layers

// Run both requests in parallel
let [primaryResponse, customResponse]: [
FeatureCollection,
FeatureCollection
] = await Promise.all([
cachedGeocoderRequest(
getPrimaryGeocoder(),
apiMethod,
convertQSPToGeocoderArgs(event.queryStringParameters),
// @ts-expect-error Redis Typescript types are not friendly
redis
),
// Custom request is either through geocoder package or "old" pelias method
getTransitGeocoder()
? cachedGeocoderRequest(
getTransitGeocoder(),
'autocomplete',
convertQSPToGeocoderArgs(event.queryStringParameters),
null
// Run all requests in parallel
const uncheckedResponses: FeatureCollection[] = await Promise.all(
geocoders.map((geocoder) =>
cachedGeocoderRequest(getGeocoder(geocoder), apiMethod, {
...convertQSPToGeocoderArgs(event.queryStringParameters),
items: pois
})
)
)

// Check if responses are satisfactory, and re-do them if needed
const responses = await Promise.all(
uncheckedResponses.map(async (response, index) => {
// If backup geocoder is present, and the returned results are garbage, use the backup geocoder
// if one is configured. This request will not be cached
if (
backupGeocoders[index] &&
!checkIfResultsAreSatisfactory(
response,
event.queryStringParameters.text
)
: fetchPelias(
TRANSIT_BASE_URL,
apiMethod,
`${new URLSearchParams(peliasQSP).toString()}&sources=transit${
CSV_ENABLED && CSV_ENABLED === 'true' ? ',pelias' : ''
}`
) {
const backupGeocoder = getGeocoder(backupGeocoders[index])
return await backupGeocoder[apiMethod](
convertQSPToGeocoderArgs(event.queryStringParameters)
)
])
}

// If the primary response doesn't contain responses or the responses are not satisfactory,
// run a second (non-cached) request with the secondary geocoder, but only if one is configured.
const secondaryGeocoder = getSecondaryGeocoder()
if (
secondaryGeocoder &&
!checkIfResultsAreSatisfactory(
primaryResponse,
event.queryStringParameters.text
)
) {
console.log('Results not satisfactory, falling back on secondary geocoder')
primaryResponse = await secondaryGeocoder[apiMethod](
convertQSPToGeocoderArgs(event.queryStringParameters)
)
}
return response
})
)

const merged = responses.reduce<
FeatureCollection<Geometry, GeoJsonProperties>
>(
(prev, cur, idx) => {
if (idx === 0) return cur
return mergeResponses({ customResponse: cur, primaryResponse: prev })
},
// TODO: clean this reducer up. See https://github.com/ibi-group/pelias-stitch/pull/28#discussion_r1547582739
{ features: [], type: 'FeatureCollection' }
)

const mergedResponse = mergeResponses({
customResponse,
primaryResponse
})
return {
body: JSON.stringify(mergedResponse),
body: JSON.stringify(merged),
/*
The third "standard" CORS header, Access-Control-Allow-Methods is not included here
following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/
Expand All @@ -232,18 +159,7 @@ module.exports.autocomplete = bugsnagHandler(
context: null,
callback: ServerlessCallbackFunction
): Promise<void> => {
if (redis) {
// Only autocomplete needs redis
try {
await redis.connect()
} catch (e) {
console.warn('Redis connection failed. Likely already connected')
console.log(e)
}
}

const response = await makeGeocoderRequests(event, 'autocomplete')

callback(null, response)
}
)
Expand Down Expand Up @@ -274,12 +190,20 @@ module.exports.reverse = bugsnagHandler(
context: null,
callback: ServerlessCallbackFunction
): Promise<void> => {
const geocoderResponse = await getPrimaryGeocoder().reverse(
let geocoderResponse = await getGeocoder(geocoders[0]).reverse(
convertQSPToGeocoderArgs(event.queryStringParameters)
)

if (!geocoderResponse && backupGeocoders[0]) {
geocoderResponse = await getGeocoder(backupGeocoders[0]).reverse(
convertQSPToGeocoderArgs(event.queryStringParameters)
)
}

geocoderResponse.label = geocoderResponse.name

callback(null, {
body: JSON.stringify(geocoderResponse),
body: JSON.stringify([geocoderResponse]),
/*
The third "standard" CORS header, Access-Control-Allow-Methods is not included here
following reccomendations in https://www.serverless.com/blog/cors-api-gateway-survival-guide/
Expand Down
3 changes: 1 addition & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,9 @@
"@bugsnag/js": "^7.11.0",
"@bugsnag/plugin-aws-lambda": "^7.11.0",
"@conveyal/lonlat": "^1.4.1",
"@opentripplanner/geocoder": "^2.0.1",
"@opentripplanner/geocoder": "^2.2.0",
"geolib": "^3.3.1",
"node-fetch": "^2.6.1",
"redis": "^4.1.0",
"serverless-api-gateway-caching": "^1.8.1",
"serverless-plugin-typescript": "^1.1.9"
},
Expand Down
18 changes: 3 additions & 15 deletions serverless.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,10 @@ provider:
subnetIds:
- ${self:custom.secrets.LAMBDA_EXEC_SUBNET}
environment:
GEOCODER: ${self:custom.secrets.GEOCODER}
TRANSIT_GEOCODER: ${self:custom.secrets.TRANSIT_GEOCODER, null}
TRANSIT_BASE_URL: ${self:custom.secrets.TRANSIT_BASE_URL, null}
# Pelias instance of Geocode.Earth, with street and landmarks
GEOCODE_EARTH_URL: ${self:custom.secrets.GEOCODE_EARTH_URL, null}
GEOCODER_API_KEY: ${self:custom.secrets.GEOCODER_API_KEY, null}
# Used to logging to Bugsnag
GEOCODERS: ${self:custom.secrets.GEOCODERS}
BACKUP_GEOCODERS: ${self:custom.secrets.BACKUP_GEOCODERS}
POIS: ${self:custom.secrets.POIS, null}
BUGSNAG_NOTIFIER_KEY: ${self:custom.secrets.BUGSNAG_NOTIFIER_KEY}
REDIS_HOST: ${self:custom.secrets.REDIS_HOST, null}
REDIS_KEY: ${self:custom.secrets.REDIS_KEY, null}
# Used to enable CSV source
CSV_ENABLED: ${self:custom.secrets.CSV_ENABLED, false}
# Secondary Geocoder config
SECONDARY_GEOCODER: ${self:custom.secrets.SECONDARY_GEOCODER, null}
SECONDARY_GEOCODER_API_KEY: ${self:custom.secrets.SECONDARY_GEOCODER_API_KEY, null}
SECONDARY_GEOCODE_EARTH_URL: ${self:custom.secrets.SECONDARY_GEOCODE_EARTH_URL, null}
custom:
secrets: ${file(env.yml)}
functions:
Expand Down
Loading

0 comments on commit 4623ba9

Please sign in to comment.