Update lambdas for linting and include in test run (#490)

docker-compose-dev.yml

@@ -109,6 +109,17 @@ services:
     volumes:
       - './services/client:/usr/src/app'
 
+  lambdas-test:
+    container_name: lambdas-test
+    build:
+      context: ./lambdas
+      dockerfile: Dockerfile-test
+    environment:
+      - NODE_ENV=development
+      - CI=${CI:-false}
+    volumes:
+      - './lambdas:/usr/src/app'
+
   swagger:
     container_name: swagger
     build:

docker-compose-prod.yml

@@ -103,6 +103,17 @@ services:
       - NODE_ENV=production
       - CI=${CI:-false}
 
+  lambdas-test:
+    container_name: lambdas-test
+    build:
+      context: ./lambdas
+      dockerfile: Dockerfile-test
+    environment:
+      - NODE_ENV=production
+      - CI=${CI:-false}
+    volumes:
+      - './lambdas:/usr/src/app'
+
   swagger:
     container_name: swagger
     build:

docker-compose-stage.yml

@@ -103,6 +103,17 @@ services:
       - NODE_ENV=staging
       - CI=${CI:-false}
 
+  lambdas-test:
+    container_name: lambdas-test
+    build:
+      context: ./lambdas
+      dockerfile: Dockerfile-test
+    environment:
+      - NODE_ENV=staging
+      - CI=${CI:-false}
+    volumes:
+      - './lambdas:/usr/src/app'
+
   swagger:
     container_name: swagger
     build:

lambdas/Dockerfile-test

@@ -0,0 +1,9 @@
+FROM node:8.10-alpine
+
+ENV NPM_CONFIG_LOGLEVEL warn
+
+COPY package.json package-lock.json /lambdas-test/
+WORKDIR /lambdas-test/
+RUN npm install
+
+COPY . /lambdas-test/

lambdas/eventbrite/config.js

@@ -10,7 +10,17 @@ const eventsParams = page =>
     "location.longitude": -6.762739,
     "location.latitude": 54.6425126, // Cookstown
     "location.within": "60mi", // 60 mile radius (all of Northern Ireland)
-    expand: "logo,venue,organizer,format,category,subcategory,bookmark_info,refund_policy,ticket_availability",
+    expand: [
+      "logo",
+      "venue",
+      "organizer",
+      "format",
+      "category",
+      "subcategory",
+      "bookmark_info",
+      "refund_policy",
+      "ticket_availability"
+    ].join(","),
     categories: EVENTBRITE_TECH_CATEGORY,
     token: process.env.EVENTBRITE_API_TOKEN
   });

lambdas/eventbrite/handlers/producer.js

@@ -48,9 +48,11 @@ module.exports.produce = async (event, context, callback) => {
 
     // Write captured data to S3
     const { producerBucket } = buckets();
-    const filePaths = await Promise.all(await eventsPages.map(async function (eventsPage, index) {
-      return (await uploadData(producerBucket, eventsPage, { index })).key;
-    }));
+    const filePaths = await Promise.all(
+      await eventsPages.map(async function(eventsPage, index) {
+        return (await uploadData(producerBucket, eventsPage, { index })).key;
+      })
+    );
 
     callback(null, { message: filePaths });
   } catch (err) {

lambdas/eventbrite/handlers/transformer.js

@@ -6,23 +6,26 @@ const eventSchema = require("./schemas/event-schema");
 const { buckets } = require("../config");
 const { uploadTo } = require("../utils");
 
-const getLogoFrom = function ({ logo }) {
+const getLogoFrom = function({ logo }) {
   if (!logo) return undefined;
 
   return {
     high: logo.original ? logo.original.url.trim() : undefined,
     regular: logo.url.trim()
-  }
+  };
 };
 
-const getTopicsFrom = function ({ category, subcategory }) {
+const getTopicsFrom = function({ category, subcategory }) {
   return [category, subcategory].reduce(
-    (topics, source) => !source ? topics : topics.concat(source.name.trim()),
+    (topics, source) => (!source ? topics : topics.concat(source.name.trim())),
     []
   );
 };
 
-const getChargeFrom = function ({ is_free: isFree, ticket_availability: tickets }) {
+const getChargeFrom = function({
+  is_free: isFree,
+  ticket_availability: tickets
+}) {
   if (isFree) return { is_free: true };
 
   return {
@@ -31,10 +34,10 @@ const getChargeFrom = function ({ is_free: isFree, ticket_availability: tickets
       currency: tickets.minimum_ticket_price.currency.trim(),
       value: tickets.minimum_ticket_price.value
     }
-  }
+  };
 };
 
-const transformEvent = function (defaults, event) {
+const transformEvent = function(defaults, event) {
   const {
     id,
     name,
@@ -45,7 +48,7 @@ const transformEvent = function (defaults, event) {
     venue,
     organizer,
     created,
-    changed,
+    changed
   } = event;
   const startDate = new Date(start.utc);
   const endDate = new Date(end.utc);
@@ -85,16 +88,15 @@ const transformEvent = function (defaults, event) {
       name: "eventbrite",
       id
     }
-  }
+  };
 };
 
-const isValidEvent = (event) => validate(event, eventSchema).errors.length === 0;
+const isValidEvent = event => validate(event, eventSchema).errors.length === 0;
 
 const uploadData = function(bucketName, eventsPage) {
   return uploadTo(
     bucketName,
-    (today, hash) =>
-      `eventbrite-events__${today.valueOf()}__${hash}.json`,
+    (today, hash) => `eventbrite-events__${today.valueOf()}__${hash}.json`,
     eventsPage
   );
 };
@@ -103,29 +105,31 @@ module.exports.transform = async (event, context, callback) => {
   try {
     const records = event.Records;
 
-    const transformedFiles = await Promise.all(await records.map(async function ({
-      s3: { bucket, object: file }
-    }) {
-      const data = await getFromS3(bucket.name, file.key);
-      const response = JSON.parse(data.Body.toString());
+    const transformedFiles = await Promise.all(
+      await records.map(async function({ s3: { bucket, object: file } }) {
+        const data = await getFromS3(bucket.name, file.key);
+        const response = JSON.parse(data.Body.toString());
 
-      const transformedEvents = response.events.map((event) => (
-        transformEvent({ country: "GB" }, event)
-      ));
+        const transformedEvents = response.events.map(rawEvent =>
+          transformEvent({ country: "GB" }, rawEvent)
+        );
 
-      const validEvents = transformedEvents.filter(isValidEvent)
+        const validEvents = transformedEvents.filter(isValidEvent);
 
-      if (validEvents.length !== transformedEvents.length) {
-        console.log('WARNING: some events generated were not valid!')
-      }
+        if (validEvents.length !== transformedEvents.length) {
+          console.log("WARNING: some events generated were not valid!");
+        }
 
-      return validEvents;
-    }));
+        return validEvents;
+      })
+    );
 
     const { eventsBucket } = buckets();
-    const filePaths = await Promise.all(await transformedFiles.map(async function (transformedFile) {
-      return (await uploadData(eventsBucket, transformedFile)).key;
-    }));
+    const filePaths = await Promise.all(
+      await transformedFiles.map(async function(transformedFile) {
+        return (await uploadData(eventsBucket, transformedFile)).key;
+      })
+    );
 
     callback(null, { message: filePaths });
   } catch (err) {

lambdas/eventbrite/package.json

@@ -1,7 +1,12 @@
 {
   "name": "muxer-pull-events-eventbrite",
-  "version": "1.0.0",
   "description": "Pull eventbrite.com events data",
+  "license": "MIT",
+  "repository": {
+    "type" : "git",
+    "url" : "https://github.com/apoclyps/my-dev-space.git"
+  },
+  "private": true,
   "dependencies": {
     "aws-lambda-data-utils": "^1.0.0",
     "jsonschema": "^1.2.4"

lambdas/eventbrite/utils.js

@@ -23,4 +23,4 @@ const convert = params =>
 module.exports = {
   uploadTo,
   convert
-}
+};

lambdas/farsetlabs/config.js

@@ -1,13 +1,14 @@
 const { addYear, convert } = require("./utils");
 
-const CALENDAR_ID = 'farsetlabs.org.uk_srmqnkn373auq51u00s2nijrq8%40group.calendar.google.com'
+const CALENDAR_ID =
+  "farsetlabs.org.uk_srmqnkn373auq51u00s2nijrq8%40group.calendar.google.com";
 
 const eventsApi = `https://www.googleapis.com/calendar/v3/calendars/${CALENDAR_ID}/events`;
 const eventsParams = () =>
   convert({
     maxResults: 2500,
     singleEvents: true,
-    orderBy: 'startTime',
+    orderBy: "startTime",
     timeMin: new Date().toISOString(),
     timeMax: addYear(new Date()).toISOString(),
     key: process.env.GOOGLE_CALENDAR_API_TOKEN

lambdas/farsetlabs/handlers/producer.js

@@ -7,8 +7,7 @@ const { uploadTo } = require("../utils");
 const uploadData = function(bucketName, calendarData) {
   return uploadTo(
     bucketName,
-    (today, hash) =>
-      `farset-labs-calendar__${today.valueOf()}__${hash}.json`,
+    (today, hash) => `farset-labs-calendar__${today.valueOf()}__${hash}.json`,
     calendarData
   );
 };

lambdas/farsetlabs/handlers/transformer.js

@@ -6,9 +6,10 @@ const eventSchema = require("./schemas/event-schema");
 const { buckets } = require("../config");
 const { uploadTo } = require("../utils");
 
-const transformEvent = function (defaults, {
-  id, summary, description, start, end, created, updated
-}) {
+const transformEvent = function(
+  defaults,
+  { id, summary, description, start, end, created, updated }
+) {
   const startDate = new Date(start.dateTime);
   const endDate = new Date(end.dateTime);
 
@@ -45,13 +46,12 @@ const transformEvent = function (defaults, {
   };
 };
 
-const isValidEvent = (event) => validate(event, eventSchema).errors.length === 0;
+const isValidEvent = event => validate(event, eventSchema).errors.length === 0;
 
 const uploadData = function(bucketName, calendarData) {
   return uploadTo(
     bucketName,
-    (today, hash) =>
-      `farset-labs-calendar__${today.valueOf()}__${hash}.json`,
+    (today, hash) => `farset-labs-calendar__${today.valueOf()}__${hash}.json`,
     calendarData
   );
 };
@@ -60,29 +60,31 @@ module.exports.transform = async (event, context, callback) => {
   try {
     const records = event.Records;
 
-    const transformedCalendars = await Promise.all(await records.map(async function ({
-      s3: { bucket, object: file }
-    }) {
-      const calendarData = await getFromS3(bucket.name, file.key);
-      const calendar = JSON.parse(calendarData.Body.toString());
+    const transformedCalendars = await Promise.all(
+      await records.map(async function({ s3: { bucket, object: file } }) {
+        const calendarData = await getFromS3(bucket.name, file.key);
+        const calendar = JSON.parse(calendarData.Body.toString());
 
-      const transformedEvents = calendar.items.map((event) => (
-        transformEvent({ timeZone: calendar.timeZone }, event)
-      ));
+        const transformedEvents = calendar.items.map(rawEvent =>
+          transformEvent({ timeZone: calendar.timeZone }, rawEvent)
+        );
 
-      const validEvents = transformedEvents.filter(isValidEvent)
+        const validEvents = transformedEvents.filter(isValidEvent);
 
-      if (validEvents.length !== transformedEvents.length) {
-        console.log('WARNING: some events generated were not valid!')
-      }
+        if (validEvents.length !== transformedEvents.length) {
+          console.log("WARNING: some events generated were not valid!");
+        }
 
-      return validEvents;
-    }));
+        return validEvents;
+      })
+    );
 
     const { eventsBucket } = buckets();
-    const filePaths = await Promise.all(await transformedCalendars.map(async function (transformedCalendar) {
-      return (await uploadData(eventsBucket, transformedCalendar)).key;
-    }));
+    const filePaths = await Promise.all(
+      await transformedCalendars.map(async function(transformedCalendar) {
+        return (await uploadData(eventsBucket, transformedCalendar)).key;
+      })
+    );
 
     callback(null, { message: filePaths });
   } catch (err) {

lambdas/farsetlabs/package.json

@@ -1,7 +1,12 @@
 {
   "name": "muxer-pull-events-farsetlabs",
-  "version": "1.0.0",
   "description": "Pull Farset Labs events data",
+  "license": "MIT",
+  "repository": {
+    "type" : "git",
+    "url" : "https://github.com/apoclyps/my-dev-space.git"
+  },
+  "private": true,
   "dependencies": {
     "aws-lambda-data-utils": "^1.0.0",
     "jsonschema": "^1.2.4"

lambdas/farsetlabs/utils.js

@@ -15,7 +15,7 @@ const uploadTo = function(bucketName, createFilename, data) {
   return setInS3(prefix, bucketName, filePath, fileContents);
 };
 
-const addYear = (date) => new Date(date.setFullYear(date.getFullYear() + 1));
+const addYear = date => new Date(date.setFullYear(date.getFullYear() + 1));
 
 const convert = params =>
   Object.entries(params)
@@ -26,4 +26,4 @@ module.exports = {
   uploadTo,
   addYear,
   convert
-}
+};

lambdas/meetupcom/config.js

@@ -18,7 +18,7 @@ const groupsParams = convert({
     "plain_text_description",
     "plain_text_no_images_description",
     "topics"
-  ].join(','),
+  ].join(","),
   fallback_suggestions: false,
   category: MEETUPCOM_TECH_CATEGORY,
   page: 200,
@@ -29,13 +29,7 @@ const groupsParams = convert({
 const eventsApi = slug => `https://api.meetup.com/${slug}/events`;
 const eventsParams = convert({
   "photo-host": "public",
-  status: [
-    "cancelled",
-    "past",
-    "proposed",
-    "suggested",
-    "upcoming"
-  ].join(','),
+  status: ["cancelled", "past", "proposed", "suggested", "upcoming"].join(","),
   page: 200
 });