Skip to content
This repository has been archived by the owner on Apr 17, 2021. It is now read-only.

Commit

Permalink
Merge pull request #464 from apoclyps/implement-eventbrite-transformer
Browse files Browse the repository at this point in the history
Implement Eventbrite transformer
  • Loading branch information
alistairjcbrown authored Oct 14, 2018
2 parents 6330021 + 371cdd6 commit 5df534c
Show file tree
Hide file tree
Showing 9 changed files with 486 additions and 15 deletions.
40 changes: 39 additions & 1 deletion lambdas/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,44 @@ will receive an "Access Denied" error. Instead you may want to comment out the

Pulls the logs from cloudwatch of the last lambda run. Useful for debugging.


### Transformer

This lambda takes the Eventbrite JSON data which has been saved to S3,
transforms it into a standardised format and saves it back to S3.

This lambda is triggered by the creation of the source file by the producer
lambda.

#### `eventbrite:transform:update`

Update just the handler functionality. Do then whenever you change the
functionality of the lambda.

#### `eventbrite:transform:invoke`

Invoke the lambda on AWS. As this lambda is triggered by the creation of files
in the producer bucket, this may not run correctly. It can be invoked by calling
the producer invoke command instead.

#### `eventbrite:transform:invoke-local`

Invoke the lambda locally. Use this for development.

The local lambda will need to be provided with an event object which contains
mock values for the newly created file. Samples of the AWS event objects can be
found at:
https://docs.aws.amazon.com/lambda/latest/dg/eventsources.html#eventsources-s3-put

The local lambda will not have permissions to read / write files to / from S3
and you will receive an "Access Denied" error. Instead you may want to comment
out the `getFromS3` and `uploadTo` calls whilst in development.

#### `eventbrite:transform:logs`

Pulls the logs from cloudwatch of the last lambda run. Useful for debugging.


---

## Farset Labs Calendar
Expand Down Expand Up @@ -204,7 +242,7 @@ functionality of the lambda.
#### `farsetlabs:transform:invoke`

Invoke the lambda on AWS. As this lambda is triggered by the creation of files
in the events bucket, this may not run correctly. It can be invoked by calling
in the producer bucket, this may not run correctly. It can be invoked by calling
the producer invoke command instead.

#### `farsetlabs:transform:invoke-local`
Expand Down
23 changes: 11 additions & 12 deletions lambdas/eventbrite/handlers/producer.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,13 @@ const getFromApi = async function() {
return Promise.all([initialResponse].concat(requests));
};

const uploadData = function(bucketName, eventsPages) {
return eventsPages.map(function(eventsPage, index) {
return uploadTo(
bucketName,
(today, hash) =>
`eventbrite-events-page-${index + 1}__${today.valueOf()}__${hash}.json`,
eventsPage
);
});
const uploadData = function(bucketName, eventsPage, { index }) {
return uploadTo(
bucketName,
(today, hash) =>
`eventbrite-events-page-${index + 1}__${today.valueOf()}__${hash}.json`,
eventsPage
);
};

module.exports.produce = async (event, context, callback) => {
Expand All @@ -50,10 +48,11 @@ module.exports.produce = async (event, context, callback) => {

// Write captured data to S3
const { producerBucket } = buckets();
const uploads = uploadData(producerBucket, eventsPages);
const message = (await Promise.all(uploads)).map(({ key }) => key);
const filePaths = await Promise.all(await eventsPages.map(async function (eventsPage, index) {
return (await uploadData(producerBucket, eventsPage, { index })).key;
}));

callback(null, { message });
callback(null, { message: filePaths });
} catch (err) {
callback(err, null);
}
Expand Down
260 changes: 260 additions & 0 deletions lambdas/eventbrite/handlers/schemas/event-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"time": {
"type": "object",
"required": [
"utc",
"timezone"
],
"properties": {
"utc": {
"type": "string",
"format": "date-time",
"title": "ISO 8601 to second precision in UTC",
"pattern": "(:?\\d{4})-(:?\\d{2})-(:?\\d{2})T(:?\\d{2})\\:(:?\\d{2})\\:(:?\\d{2}).(:?\\d{3})Z",
"examples": ["2018-10-20T18:00:00.000Z"]
},
"timezone": {
"type": "string",
"title": "Timezone in TZ value",
"pattern": "\\w+\\/[\\w\\/]+",
"examples": ["Europe/Belfast"]
}
}
},
"image-set": {
"type": "object",
"required": [
"regular"
],
"properties": {
"high": {
"type": "string",
"format": "uri",
"title": "URL to high resolution image"
},
"regular": {
"type": "string",
"format": "uri",
"title": "URL to regular resolution image"
},
"thumnail": {
"type": "string",
"format": "uri",
"title": "URL to thumbnail image"
}
}
}
},
"type": "object",
"required": [
"name",
"times",
"venue",
"created_at",
"last_updated",
"source_data"
],
"properties": {
"name": {
"type": "string",
"title": "Name of event",
"examples": ["Code Co-Op Challenge"]
},
"description": {
"type": "string",
"title": "Description of event",
"examples": ["Try your hand at this month's coding challenge and learn how your peers tackle the same task."]
},
"url": {
"type": "string",
"title": "URL to source webpage for event",
"examples": ["https://www.meetup.com/CodeCoop-NI/events/ggxkhqyxpbcb/"]
},
"times": {
"type": "object",
"title": "Timing information for the event",
"required": [
"start",
"end",
"duration"
],
"properties": {
"start": {
"$ref": "#/definitions/time"
},
"end": {
"$ref": "#/definitions/time"
},
"duration": {
"type": "integer",
"title": "Number of milliseconds between start and end times",
"examples": [14400000]
}
}
},
"logo": {
"$ref": "#/definitions/image-set"
},
"topics": {
"type": "array",
"title": "Topics describing the event",
"items": {
"type": "string",
"examples": ["Science & Technology"]
}
},
"venue": {
"type": "object",
"required": [
"name",
"address",
"country",
"latitude",
"longitude"
],
"properties": {
"name": {
"type": "string",
"title": "Name of venue",
"examples": ["Farset Labs"]
},
"address": {
"type": "string",
"title": "Address of venue",
"examples": ["Weavers Court, Linfield Road, BT12 5GH"]
},
"city": {
"type": "string",
"title": "City the venue is in",
"examples": ["Belfast"]
},
"country": {
"type": "string",
"title": "Country the venue is in, must be ISO 3166-1 alpha-2 upper cased two-letter country code",
"pattern": "[A-Z]{2}",
"examples": ["GB"]
},
"latitude": {
"type": "string",
"title": "Latitude of venue location, no precision requirement",
"examples": ["54.592826"]
},
"longitude": {
"type": "string",
"title": "Longitude of venue location, no precision requirement",
"examples": ["-5.940666"]
}
}
},
"organiser": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string",
"title": "Name of organiser or organising group",
"examples": ["Code Co-Op"]
},
"logo": {
"$ref": "#/definitions/image-set"
}
}
},
"attendee_numbers": {
"type": "object",
"required": [
"capacity",
"responses",
"waitlist"
],
"properties": {
"capacity": {
"type": "integer",
"title": "The maximum number of attendees",
"examples": [50]
},
"responses": {
"type": "integer",
"title": "The number of people who are going / have indicated they are going",
"examples": [38]
},
"waitlist": {
"type": "integer",
"title": "The number of people who are on the wait list (when at capacity)",
"examples": [0]
}
}
},
"charge": {
"type": "object",
"required": [
"is_free"
],
"properties": {
"is_free": {
"type": "boolean",
"title": "Whether the event is free to attend",
"examples": [false]
},
"cost": {
"type": "object",
"required": [
"currency",
"value"
],
"properties": {
"currency": {
"type": "string",
"title": "Currency of the cost, must be ISO 4217 three-letter currency code",
"pattern": "[A-Z]{2}",
"examples": ["GBP"]
},
"value": {
"type": "integer",
"title": "Number representing the value of the cost in the lowest denomination (eg. pence if in GBP)",
"examples": [1200]
}
}
}
}
},
"created_at": {
"type": "string",
"format": "date-time",
"title": "ISO 8601 to second precision in UTC",
"pattern": "(:?\\d{4})-(:?\\d{2})-(:?\\d{2})T(:?\\d{2})\\:(:?\\d{2})\\:(:?\\d{2}).(:?\\d{3})Z",
"examples": ["2018-06-20T09:26:59Z"]
},
"last_updated": {
"type": "string",
"format": "date-time",
"title": "ISO 8601 to second precision in UTC",
"pattern": "(:?\\d{4})-(:?\\d{2})-(:?\\d{2})T(:?\\d{2})\\:(:?\\d{2})\\:(:?\\d{2}).(:?\\d{3})Z",
"examples": ["2018-10-07T14:51:21Z"]
},
"source_data": {
"type": "object",
"required": [
"name",
"id"
],
"properties": {
"name": {
"type": "string",
"title": "Name of the source, eg. farsetlabs-calendar, meetupcom, eventbrite, etc.",
"enum": ["meetupcom", "eventbrite", "farsetlabs-calendar"],
"examples": ["farsetlabs-calendar"]
},
"id": {
"type": "string",
"title": "ID of the event from the source",
"examples": ["[email protected]"]
}
}
}
}
}
Loading

0 comments on commit 5df534c

Please sign in to comment.