diff --git a/.npmignore b/.npmignore index 1d7b701..789a58c 100644 --- a/.npmignore +++ b/.npmignore @@ -14,14 +14,15 @@ results node_modules/ test_output/ -.vagrant/ env/ src/ test/ utils/ -Vagrantfile +docker-compose.yml +wercker.yml +.vim.custom .gitignore .npmignore .DS_Store diff --git a/README.md b/README.md index 08a8638..5b49dea 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ AWS S3 Image Uploader ===================== +**Documentation for `s3-uploader@0.9` can be found [here](https://github.com/Turistforeningen/node-s3-uploader/blob/stable/0.x/README.md).** + [![Build status](https://img.shields.io/wercker/ci/54f18246d9b14636634ff908.svg "Build status")](https://app.wercker.com/project/bykey/50fbdf51cf64b01a738379a028b8a885) [![NPM downloads](https://img.shields.io/npm/dm/s3-uploader.svg "NPM downloads")](https://www.npmjs.com/package/s3-uploader) [![NPM version](https://img.shields.io/npm/v/s3-uploader.svg "NPM version")](https://www.npmjs.com/package/s3-uploader) @@ -13,6 +15,8 @@ and [im-resize](https://github.com/Turistforeningen/node-im-resize) and [im-metadata](https://github.com/Turistforeningen/node-im-metadata) for image processing. +![Overview of image upload to AWS S3](https://docs.google.com/drawings/d/1EZaE8LaQ6FRSg4R-2QQiT1af-y2AgDknBGrx6SPIKy0/pub?w=766&h=216) + [![Join the chat at https://gitter.im/Turistforeningen/node-s3-uploader](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Turistforeningen/node-s3-uploader) ## Changelog @@ -29,7 +33,7 @@ npm install s3-uploader --save ## Requirements * Node.JS >= v0.10 -* imagemagic +* ImageMagic >= v6.8 * AWS credentials environment variables * `AWS_ACCESS_KEY_ID` * `AWS_SECRET_ACCESS_KEY` @@ -55,7 +59,7 @@ var Upload = require('s3-uploader'); * **object** `aws` - see [note](#aws-note) * **string** `region` - region for you bucket (**default** `us-east-1`) * **string** `path` - path within your bucket (**default** `""`) - * **string** `acl` - default ACL for uploaded images (**default** `privat`) + * **string** `acl` - default ACL for uploaded images (**default** `private`) * **string** `accessKeyId` - AWS access key ID override * **string** `secretAccessKey` - AWS secret access key override @@ -67,15 +71,22 @@ var Upload = require('s3-uploader'); * **object[]** `versions` * **string** `suffix` - image file name suffix (**default** `""`) * **number** `quality` - image resize quality + * **string** `format` - force output image file format (**default** `format of original image`) * **number** `maxWidth` - max width for resized image * **number** `maxHeight` - max height for resized image * **string** `aspect` - force aspect ratio for resized image (**example:** `4:3` * **string** `background` - set background for transparent images (**example:** `red`) * **boolean** `flatten` - flatten backgrund for transparent images * **string** `awsImageAcl` - access control for AWS S3 upload (**example:** `private`) + * **number** `awsImageExpires` - add `Expires` header to image version + * **number** `awsImageCacheControl` - add `Cache-Control` header to image version * **object** `original` * **string** `awsImageAcl` - access control for AWS S3 upload (**example:** `private`) + * **number** `awsImageExpires` - add `Expires` header to image version + * **number** `awsImageCacheControl` - add `Cache-Control` header to image version + + * **function** `randomPath` - custom random path function #### AWS note > The `aws` object is passed directly to `aws-sdk`. You can add any of [these @@ -104,22 +115,21 @@ var client = new Upload('my_s3_bucket', { versions: [{ maxHeight: 1040, maxWidth: 1040, + format: 'jpg', suffix: '-large', quality: 80 },{ - maxHeight: 780, maxWidth: 780, - aspect: '4:3', + aspect: '3:2!h', suffix: '-medium' },{ - maxHeight: 320, maxWidth: 320, - aspect: '4:3', + aspect: '16:9!h', suffix: '-small' },{ maxHeight: 100, - maxWidth: 100, aspect: '1:1', + format: 'png', suffix: '-thumb1' },{ maxHeight: 250, @@ -136,6 +146,7 @@ var client = new Upload('my_s3_bucket', { * **object** `opts` * **string** `awsPath` - override the path on AWS set through `opts.aws.path` + * **string** `path` - set absolute path for uploaded image (disables random path) * **function** `cb` - callback function (**Error** `err`, **object[]** `versions`, **object** `meta`) * **Error** `err` - `null` if everything went fine @@ -145,14 +156,13 @@ var client = new Upload('my_s3_bucket', { #### Example ```javascript -client.upload('/some/file/path.jpg', {}, function(err, images, meta) { - if (err) { - console.error(err); - } else { - for (var i = 0; i < images.length; i++) { - console.log('Thumbnail with width %i, height %i, at %s', images[i].width, images[i].height, images[i].url); - } - } +client.upload('/some/image.jpg', {}, function(err, versions, meta) { + if (err) { throw err; } + + versions.forEach(function(image) { + console.log(image.width, image.height, image.url); + // 1234 4567 https://my-bucket.s3.amazonaws.com/path/ab/cd/ef.jpg + }); }); ``` diff --git a/docker-compose.yml b/docker-compose.yml index 5d82016..44fac00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,12 @@ dev: - image: starefossen/iojs-imagemagick:1-6 + image: starefossen/iojs-imagemagick:2-6 working_dir: /usr/src/app volumes: - ".:/usr/src/app" command: "npm run watch" env_file: .env environment: - NODE_ENV: development + - NODE_ENV=development + - NPM_CONFIG_LOGLEVEL=info + - NPM_PACKAGE_CONFIG_UNSAFE_PERM=true + - NPM_CONFIG_UNSAFE_PERM=true diff --git a/package.json b/package.json index d7e31e9..45b6209 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,18 @@ { "name": "s3-uploader", - "version": "1.0.0-rc.2", + "version": "1.1.0-rc.2", "description": "Resize, rename, and upload images to AWS S3", "main": "lib/index.js", "directories": { "test": "test" }, "scripts": { - "build": "coffee --bare --compile --output lib/ src/*.coffee", - "prepublish": "coffee --bare --compile --output lib/ src/*.coffee", + "build": "coffee -c -b -o lib/ src/*.coffee", + "prepublish": "coffee -c -b -o lib/ src/*.coffee", "postpublish": "rm -rf lib/*", - "test": "mocha test/suite.coffee -R spec --compilers coffee:coffee-script/register,litcoffee:coffee-script/register", - "watch": "./node_modules/.bin/mocha -w -b -c --check-leaks test/suite.coffee -R progress --compilers coffee:coffee-script/register,litcoffee:coffee-script/register" + "hint": "coffeelint src test", + "test": "mocha --compilers coffee:coffee-script/register test/suite.coffee", + "watch": "mocha -w -b -c --check-leaks -R progress --compilers coffee:coffee-script/register test/suite.coffee" }, "repository": { "type": "git", @@ -24,7 +25,8 @@ "image", "resize", "rename", - "upload" + "upload", + "versions" ], "contributors": [ "Håvard Ranum ", @@ -36,16 +38,20 @@ }, "homepage": "https://github.com/Turistforeningen/node-s3-uploader", "devDependencies": { - "coffee-script": "~1.9.1", - "mocha": "~2.2.1" + "coffee-script": "~1", + "coffeelint": "~1", + "mocha": "~2", + "uuid": "^2" }, "dependencies": { - "async": "~1.2", - "aws-sdk": "~2.1", - "im-resize": "~2.0", - "im-metadata": "~2.1" + "@starefossen/rand-path": "^1.0.1", + "async": "~1.4", + "aws-sdk": "^2.2.9", + "im-metadata": "~2.2", + "im-resize": "~2.3" }, "engines": { - "node": ">=0.10" + "node": ">=0.10", + "iojs": ">=1.0.0" } } diff --git a/src/index.coffee b/src/index.coffee index 22c076a..073e103 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -11,17 +11,17 @@ retry = require('async').retry resize = require 'im-resize' metadata = require 'im-metadata' -Upload = module.exports = (awsBucketName, @opts = {}) -> - throw new TypeError 'Bucket name can not be undefined' if not awsBucketName +Upload = module.exports = (bucketName, @opts = {}) -> + throw new TypeError 'Bucket name can not be undefined' if not bucketName @opts.aws ?= {} #@opts.aws.accessKeyId - @opts.aws.acl ?= 'privat' + @opts.aws.acl ?= 'private' @opts.aws.httpOptions ?= {} @opts.aws.httpOptions.timeout ?= 10000 @opts.aws.maxRetries ?= 3 @opts.aws.params ?= {} - @opts.aws.params.Bucket = awsBucketName + @opts.aws.params.Bucket = bucketName @opts.aws.path ?= '' @opts.aws.region ?= 'us-east-1' #@opts.aws.secretAccessKey @@ -37,28 +37,16 @@ Upload = module.exports = (awsBucketName, @opts = {}) -> @opts.versions ?= [] if not @opts.url and @opts.aws.region is 'us-east-1' - @opts.url ?= "https://s3.amazonaws.com/#{@opts.aws.params.Bucket}/" + @opts.url ?= "https://s3.amazonaws.com/#{bucketName}/" else if not @opts.url - @opts.url ?= "https://s3-#{@opts.aws.region}.amazonaws.com/#{@opts.aws.params.Bucket}/" + @opts.url ?= "https://s3-#{@opts.aws.region}.amazonaws.com/#{bucketName}/" + + @._getRandomPath = @opts.randomPath or require('@starefossen/rand-path') @s3 = new S3 @opts.aws @ -## -# Generate a random path on the form /xx/yy/zz -## -Upload.prototype._getRandomPath = -> - input = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' - res = [] - - for i in [1..3] - x = input[Math.floor((Math.random() * input.length))] - y = input[Math.floor((Math.random() * input.length))] - res.push x + y - - return res.join '/' - ## # Generate a random avaiable path on the S3 bucket ## @@ -98,13 +86,18 @@ Image.prototype.start = (cb) -> # Get image metadata ## Image.prototype.getMetadata = (src, cb) -> - metadata src, exif: @upload.opts.returnExif, cb + metadata src, exif: @upload.opts.returnExif, autoOrient: true, cb ## # Get image destination ## Image.prototype.getDest = (cb) -> prefix = @opts?.awsPath or @upload.opts.aws.path + + if @opts.path + return process.nextTick => + cb null, prefix + @opts.path + @upload._getDestPath prefix, cb ## @@ -123,10 +116,13 @@ Image.prototype.resizeVersions = (cb, results) -> ## Image.prototype.uploadVersions = (cb, results) -> if @upload.opts.original - results.versions.push - awsImageAcl: @upload.opts.original.awsImageAcl - original: true - path: @src + org = JSON.parse(JSON.stringify(@upload.opts.original)) + org.original = true + org.width = results.metadata.width + org.height = results.metadata.height + org.path = @src + + results.versions.push org map results.versions, @_upload.bind(@, results.dest), cb @@ -136,8 +132,8 @@ Image.prototype.uploadVersions = (cb, results) -> Image.prototype.removeVersions = (cb, results) -> each results.uploads, (image, callback) => if not @upload.opts.cleanup.original and image.original \ - or not @upload.opts.cleanup.versions and not image.original - return setTimeout callback, 0 + or not @upload.opts.cleanup.versions and not image.original + return setTimeout callback, 0 fs.unlink image.path, callback , (err) -> @@ -147,13 +143,20 @@ Image.prototype.removeVersions = (cb, results) -> # Upload image version to S3 ## Image.prototype._upload = (dest, version, cb) -> - format = extname version.path + version.awsImageAcl ?= @upload.opts.aws.acl + format = extname(version.path).substr(1).toLowerCase() options = - Key: dest + (version.suffix || '') + format - ACL: version.awsImageAcl or @upload.opts.aws.acl + Key: "#{dest}#{version.suffix or ''}.#{format}" + ACL: version.awsImageAcl Body: fs.createReadStream version.path - ContentType: "image/#{if format is '.jpg' then 'jpeg' else format.substr(1)}" + ContentType: "image/#{if format is 'jpg' then 'jpeg' else format}" + + if version.awsImageExpires + options.Expires = new Date(Date.now() + version.awsImageExpires) + + if version.awsImageMaxAge + options.CacheControl = "public, max-age=#{version.awsImageMaxAge}" @upload.s3.putObject options, (err, data) => return cb err if err diff --git a/test/suite.coffee b/test/suite.coffee index 9677544..20f2eff 100644 --- a/test/suite.coffee +++ b/test/suite.coffee @@ -15,31 +15,35 @@ beforeEach -> original: false original: awsImageAcl: 'private' + awsImageMaxAge: 31536000 versions: [{ maxHeight: 1040 maxWidth: 1040 + format: 'jpg' suffix: '-large' quality: 80 },{ - maxHeight: 780 maxWidth: 780 - aspect: '4:3' + aspect: '3:2!h' suffix: '-medium' },{ - maxHeight: 320 maxWidth: 320 - aspect: '4:3' + aspect: '16:9!h' suffix: '-small' },{ maxHeight: 100 - maxWidth: 100 aspect: '1:1' + format: 'png' suffix: '-thumb1' + awsImageExpires: 31536000 + cacheControl: 31536000 },{ maxHeight: 250 maxWidth: 250 aspect: '1:1' suffix: '-thumb2' + awsImageExpires: 31536000 + cacheControl: 31536000 }] # Mock S3 API calls @@ -75,7 +79,7 @@ describe 'Upload', -> assert upload.s3 instanceof require('aws-sdk').S3 assert.deepEqual upload.opts, aws: - acl: 'privat' + acl: 'private' httpOptions: timeout: 10000 maxRetries: 3 params: Bucket: 'myBucket' @@ -89,8 +93,8 @@ describe 'Upload', -> url: 'https://s3.amazonaws.com/myBucket/' it 'sets default url based on AWS region', -> - upload = new Upload 'myBucket', aws: region: 'my-region-1' - assert.equal upload.opts.url, 'https://s3-my-region-1.amazonaws.com/myBucket/' + upload = new Upload 'b', aws: region: 'my-region-1' + assert.equal upload.opts.url, 'https://s3-my-region-1.amazonaws.com/b/' it 'sets custom url', -> upload = new Upload 'myBucket', url: 'http://cdn.app.com/' @@ -119,21 +123,32 @@ describe 'Upload', -> describe '#_getRandomPath()', -> it 'returns a new random path', -> path = upload._getRandomPath() - assert(/^[A-Za-z0-9]{2}\/[A-Za-z0-9]{2}\/[A-Za-z0-9]{2}$/.test(path)) + assert(/^\w{2}(\/\w{2}){2}$/.test(path)) + + it 'returns custom random path', -> + upload = new Upload process.env.AWS_BUCKET_NAME, + randomPath: require('uuid').v1 + + path = upload._getRandomPath() + assert(/^\w+(-\w+){4}$/.test(path)) describe '#_getDestPath()', -> beforeEach -> upload._getRandomPath = -> return 'aa/bb/cc' it 'returns a random avaiable path', (done) -> - upload.s3.listObjects = (opts, cb) -> process.nextTick -> cb null, Contents: [] + upload.s3.listObjects = (opts, cb) -> + process.nextTick -> cb null, Contents: [] + upload._getDestPath 'some/prefix/', (err, path) -> assert.ifError err assert.equal path, 'some/prefix/aa/bb/cc' done() it 'returns error if no available path can be found', (done) -> - upload.s3.listObjects = (opts, cb) -> process.nextTick -> cb null, Contents: [opts.Prefix] + upload.s3.listObjects = (opts, cb) -> + process.nextTick -> cb null, Contents: [opts.Prefix] + upload._getDestPath 'some/prefix/', (err, path) -> assert err instanceof Error assert.equal err.message, 'Path some/prefix/aa/bb/cc not avaiable' @@ -239,6 +254,22 @@ describe 'Image', -> image._upload 'aa/bb/cc', version + it 'sets upload expire header for version', (done) -> + version = path: '/some/image.jpg', awsImageExpires: 1234 + image.upload.s3.putObject = (opts, cb) -> + assert opts.Expires - Date.now() <= 1234 + done() + + image._upload 'aa/bb/cc', version + + it 'sets upload cache-control header for version', (done) -> + version = path: '/some/image.jpg', awsImageMaxAge: 1234 + image.upload.s3.putObject = (opts, cb) -> + assert.equal opts.CacheControl, 'public, max-age=1234' + done() + + image._upload 'aa/bb/cc', version + it 'returns etag for uploaded version', (done) -> version = path: '/some/image.jpg' image._upload 'aa/bb/cc', version, (err, version) -> @@ -293,6 +324,21 @@ describe 'Image', -> assert.equal path, 'custom/path/aa/bb/cc' done() + it 'returns fixed upload path', (done) -> + image.opts.path = 'my/image' + image.getDest (err, path) -> + assert.ifError err + assert.equal path, 'images_test/my/image' + done() + + it 'returns fixed upload path with custom prefix', (done) -> + image.opts.awsPath = 'custom/path/' + image.opts.path = 'my/image' + image.getDest (err, path) -> + assert.ifError err + assert.equal path, 'custom/path/my/image' + done() + describe '#resizeVersions()', -> it 'resizes image versions', (done) -> image.getMetadata image.src, (err, metadata) -> @@ -330,25 +376,37 @@ describe 'Image', -> it 'uploads original image', (done) -> image._upload = (dest, version, cb) -> assert.deepEqual version, - awsImageAcl: 'private' + awsImageAcl: 'public' + awsImageExpires: 31536000 + awsImageMaxAge: 31536000 original: true + width: 111 + height: 222 path: image.src cb null, version - image.upload.opts.original = awsImageAcl: 'private' + image.upload.opts.original = + awsImageAcl: 'public' + awsImageExpires: 31536000 + awsImageMaxAge: 31536000 + image.uploadVersions (err, versions) -> assert.ifError err assert.deepEqual versions, [ - awsImageAcl: 'private' + awsImageAcl: 'public' + awsImageExpires: 31536000 + awsImageMaxAge: 31536000 original: true + width: 111 + height: 222 path: image.src ] done() - , versions: [], dest: '/foo/bar' + , versions: [], dest: '/foo/bar', metadata: width: 111, height: 222 describe '#removeVersions()', -> @@ -416,13 +474,41 @@ describe 'Integration Tests', -> assert.equal typeof image.etag, 'string' assert.equal typeof image.path, 'string' assert.equal typeof image.key, 'string' + /^images_test(\/[\w]{2}){3}/.test image.key + assert.equal typeof image.url, 'string' + + if image.original + assert.equal image.original, true + else + assert.equal typeof image.suffix, 'string' + assert.equal typeof image.height, 'number' + assert.equal typeof image.width, 'number' + + done() + + it 'uploads image to fixed path', (done) -> + @timeout 10000 + + file = __dirname + '/assets/portrait.jpg' + opts = path: 'path/to/image' + + upload.upload file, opts, (err, images, meta) -> + assert.ifError err + + for image in images + cleanup.push Key: image.key if image.key # clean up in AWS + + assert.equal typeof image.etag, 'string' + assert.equal typeof image.path, 'string' + assert.equal typeof image.key, 'string' + /^images_test\/path\/to\/image/.test image.key assert.equal typeof image.url, 'string' if image.original assert.equal image.original, true else assert.equal typeof image.suffix, 'string' - assert.equal typeof image.maxHeight, 'number' - assert.equal typeof image.maxWidth, 'number' + assert.equal typeof image.height, 'number' + assert.equal typeof image.width, 'number' done() diff --git a/wercker.yml b/wercker.yml index 8ee6e46..7d8262a 100644 --- a/wercker.yml +++ b/wercker.yml @@ -1,15 +1,25 @@ -box: wercker/nodejs +box: starefossen/iojs-imagemagick:1.0-6.8 + build: steps: - - npm-install - - npm-test - - script: name: echo nodejs information code: | echo "node version $(node -v) running" echo "npm version $(npm -v) running" + - script: + name: echo imagemagick information + code: echo "$(convert -version)" + + - npm-install + + - script: + name: jshint + code: npm run hint + + - npm-test + after-steps: - turistforeningen/slack-notifier: url: $SLACK_WEBHOOK_URL