Skip to content
This repository has been archived by the owner on Apr 17, 2021. It is now read-only.

Commit

Permalink
Merge pull request #489 from apoclyps/implement-meetupcom-transformer
Browse files Browse the repository at this point in the history
Implement Meetup.com transformer
  • Loading branch information
alistairjcbrown authored Oct 21, 2018
2 parents 8f12bc0 + 719a0a2 commit 920cfb9
Show file tree
Hide file tree
Showing 11 changed files with 563 additions and 9 deletions.
38 changes: 38 additions & 0 deletions lambdas/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,44 @@ will receive an "Access Denied" error. Instead you may want to comment out the

Pulls the logs from cloudwatch of the last lambda run. Useful for debugging.


### Transformer

This lambda takes the Meetup.com JSON data which has been saved to S3,
transforms it into a standardised format and saves it back to S3.

This lambda is triggered by the creation of the source file by the producer
lambda.

#### `meetupcom:transform:update`

Update just the handler functionality. Do then whenever you change the
functionality of the lambda.

#### `meetupcom:transform:invoke`

Invoke the lambda on AWS. As this lambda is triggered by the creation of files
in the producer bucket, this may not run correctly. It can be invoked by calling
the producer invoke command instead.

#### `meetupcom:transform:invoke-local`

Invoke the lambda locally. Use this for development.

The local lambda will need to be provided with an event object which contains
mock values for the newly created file. Samples of the AWS event objects can be
found at:
https://docs.aws.amazon.com/lambda/latest/dg/eventsources.html#eventsources-s3-put

The local lambda will not have permissions to read / write files to / from S3
and you will receive an "Access Denied" error. Instead you may want to comment
out the `getListFromS3`, `getFromS3` and `uploadTo` calls whilst in development.

#### `meetupcom:transform:logs`

Pulls the logs from cloudwatch of the last lambda run. Useful for debugging.


---

## Eventbrite
Expand Down
1 change: 0 additions & 1 deletion lambdas/eventbrite/handlers/schemas/event-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@
"attendee_numbers": {
"type": "object",
"required": [
"capacity",
"responses",
"waitlist"
],
Expand Down
1 change: 0 additions & 1 deletion lambdas/farsetlabs/handlers/schemas/event-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,6 @@
"attendee_numbers": {
"type": "object",
"required": [
"capacity",
"responses",
"waitlist"
],
Expand Down
20 changes: 17 additions & 3 deletions lambdas/meetupcom/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,16 @@ const groupsParams = convert({
lon: -6.762739,
lat: 54.6425126, // Cookstown
radius: 60, // 60 mile radius (all of Northern Ireland)
fields: "approved,best_topics,past_event_count,plain_text_description,topics",
upcoming_events: true,
fields: [
"approved",
"best_topics",
"description_images",
"past_event_count_inclusive",
"group_past_event_count",
"plain_text_description",
"plain_text_no_images_description",
"topics"
].join(','),
fallback_suggestions: false,
category: MEETUPCOM_TECH_CATEGORY,
page: 200,
Expand All @@ -21,7 +29,13 @@ const groupsParams = convert({
const eventsApi = slug => `https://api.meetup.com/${slug}/events`;
const eventsParams = convert({
"photo-host": "public",
status: "cancelled,past,proposed,suggested,upcoming",
status: [
"cancelled",
"past",
"proposed",
"suggested",
"upcoming"
].join(','),
page: 200
});

Expand Down
5 changes: 3 additions & 2 deletions lambdas/meetupcom/handlers/producer.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const uploadData = function(bucketName, groups, groupsEvents) {
const eventsUploads = groupsEvents.map(function(groupEvents, index) {
const { urlname } = groups[index];
return uploadTo(
bucketName,
(today, hash) =>
`groups-events/meetupcom-group-${urlname.toLowerCase()}__${today.valueOf()}__${hash}.json`,
groupEvents
Expand Down Expand Up @@ -55,9 +56,9 @@ module.exports.produce = async (event, context, callback) => {
// Write captured data to S3
const { producerBucket } = buckets();
const uploads = uploadData(producerBucket, groups, groupsEvents);
const message = (await Promise.all(uploads)).map(({ key }) => key);
const filePaths = (await Promise.all(uploads)).map(({ key }) => key);

callback(null, { message });
callback(null, { message: filePaths });
} catch (err) {
callback(err, null);
}
Expand Down
259 changes: 259 additions & 0 deletions lambdas/meetupcom/handlers/schemas/event-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"time": {
"type": "object",
"required": [
"utc",
"timezone"
],
"properties": {
"utc": {
"type": "string",
"format": "date-time",
"title": "ISO 8601 to second precision in UTC",
"pattern": "(:?\\d{4})-(:?\\d{2})-(:?\\d{2})T(:?\\d{2})\\:(:?\\d{2})\\:(:?\\d{2}).(:?\\d{3})Z",
"examples": ["2018-10-20T18:00:00.000Z"]
},
"timezone": {
"type": "string",
"title": "Timezone in TZ value",
"pattern": "\\w+\\/[\\w\\/]+",
"examples": ["Europe/Belfast"]
}
}
},
"image-set": {
"type": "object",
"required": [
"regular"
],
"properties": {
"high": {
"type": "string",
"format": "uri",
"title": "URL to high resolution image"
},
"regular": {
"type": "string",
"format": "uri",
"title": "URL to regular resolution image"
},
"thumnail": {
"type": "string",
"format": "uri",
"title": "URL to thumbnail image"
}
}
}
},
"type": "object",
"required": [
"name",
"times",
"venue",
"created_at",
"last_updated",
"source_data"
],
"properties": {
"name": {
"type": "string",
"title": "Name of event",
"examples": ["Code Co-Op Challenge"]
},
"description": {
"type": "string",
"title": "Description of event",
"examples": ["Try your hand at this month's coding challenge and learn how your peers tackle the same task."]
},
"url": {
"type": "string",
"title": "URL to source webpage for event",
"examples": ["https://www.meetup.com/CodeCoop-NI/events/ggxkhqyxpbcb/"]
},
"times": {
"type": "object",
"title": "Timing information for the event",
"required": [
"start",
"end",
"duration"
],
"properties": {
"start": {
"$ref": "#/definitions/time"
},
"end": {
"$ref": "#/definitions/time"
},
"duration": {
"type": "integer",
"title": "Number of milliseconds between start and end times",
"examples": [14400000]
}
}
},
"logo": {
"$ref": "#/definitions/image-set"
},
"topics": {
"type": "array",
"title": "Topics describing the event",
"items": {
"type": "string",
"examples": ["Science & Technology"]
}
},
"venue": {
"type": "object",
"required": [
"name",
"address",
"country",
"latitude",
"longitude"
],
"properties": {
"name": {
"type": "string",
"title": "Name of venue",
"examples": ["Farset Labs"]
},
"address": {
"type": "string",
"title": "Address of venue",
"examples": ["Weavers Court, Linfield Road, BT12 5GH"]
},
"city": {
"type": "string",
"title": "City the venue is in",
"examples": ["Belfast"]
},
"country": {
"type": "string",
"title": "Country the venue is in, must be ISO 3166-1 alpha-2 upper cased two-letter country code",
"pattern": "[A-Z]{2}",
"examples": ["GB"]
},
"latitude": {
"type": "string",
"title": "Latitude of venue location, no precision requirement",
"examples": ["54.592826"]
},
"longitude": {
"type": "string",
"title": "Longitude of venue location, no precision requirement",
"examples": ["-5.940666"]
}
}
},
"organiser": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"title": "Name of organiser or organising group",
"examples": ["Code Co-Op"]
},
"logo": {
"$ref": "#/definitions/image-set"
}
}
},
"attendee_numbers": {
"type": "object",
"required": [
"responses",
"waitlist"
],
"properties": {
"capacity": {
"type": "integer",
"title": "The maximum number of attendees",
"examples": [50]
},
"responses": {
"type": "integer",
"title": "The number of people who are going / have indicated they are going",
"examples": [38]
},
"waitlist": {
"type": "integer",
"title": "The number of people who are on the wait list (when at capacity)",
"examples": [0]
}
}
},
"charge": {
"type": "object",
"required": [
"is_free"
],
"properties": {
"is_free": {
"type": "boolean",
"title": "Whether the event is free to attend",
"examples": [false]
},
"cost": {
"type": "object",
"required": [
"currency",
"value"
],
"properties": {
"currency": {
"type": "string",
"title": "Currency of the cost, must be ISO 4217 three-letter currency code",
"pattern": "[A-Z]{2}",
"examples": ["GBP"]
},
"value": {
"type": "integer",
"title": "Number representing the value of the cost in the lowest denomination (eg. pence if in GBP)",
"examples": [1200]
}
}
}
}
},
"created_at": {
"type": "string",
"format": "date-time",
"title": "ISO 8601 to second precision in UTC",
"pattern": "(:?\\d{4})-(:?\\d{2})-(:?\\d{2})T(:?\\d{2})\\:(:?\\d{2})\\:(:?\\d{2}).(:?\\d{3})Z",
"examples": ["2018-06-20T09:26:59Z"]
},
"last_updated": {
"type": "string",
"format": "date-time",
"title": "ISO 8601 to second precision in UTC",
"pattern": "(:?\\d{4})-(:?\\d{2})-(:?\\d{2})T(:?\\d{2})\\:(:?\\d{2})\\:(:?\\d{2}).(:?\\d{3})Z",
"examples": ["2018-10-07T14:51:21Z"]
},
"source_data": {
"type": "object",
"required": [
"name",
"id"
],
"properties": {
"name": {
"type": "string",
"title": "Name of the source, eg. farsetlabs-calendar, meetupcom, eventbrite, etc.",
"enum": ["meetupcom", "eventbrite", "farsetlabs-calendar"],
"examples": ["farsetlabs-calendar"]
},
"id": {
"type": "string",
"title": "ID of the event from the source",
"examples": ["[email protected]"]
}
}
}
}
}
Loading

0 comments on commit 920cfb9

Please sign in to comment.