Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add options: original_prefix, reproducible_prefix, and acl #27

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add options: original_prefix, reproducible_prefix, and acl #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dd1aeb4
Add original_prefix and reproducible_prefix option to store classes
hest Apr 28, 2015
cae32db
Add acl option to S3Store
hest Apr 28, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions sqlalchemy_imageattach/context.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,25 +195,31 @@ def __init__(self, get_current_object, repr_string=None):
self.repr_string = repr_string

def put_file(self, file, object_type, object_id, width, height,
mimetype, reproducible):
mimetype, reproducible=False):
self.get_current_object().put_file(
file, object_type, object_id, width, height,
mimetype, reproducible
mimetype, reproducible=reproducible
)

def delete_file(self, object_type, object_id, width, height, mimetype):
def delete_file(self, object_type, object_id, width, height,
mimetype, reproducible=False):
self.get_current_object().delete_file(
object_type, object_id, width, height, mimetype
object_type, object_id, width, height,
mimetype, reproducible=reproducible
)

def get_file(self, object_type, object_id, width, height, mimetype):
def get_file(self, object_type, object_id, width, height,
mimetype, reproducible=False):
return self.get_current_object().get_file(
object_type, object_id, width, height, mimetype
object_type, object_id, width, height,
mimetype, reproducible=reproducible
)

def get_url(self, object_type, object_id, width, height, mimetype):
def get_url(self, object_type, object_id, width, height,
mimetype, reproducible=False):
return self.get_current_object().get_url(
object_type, object_id, width, height, mimetype
object_type, object_id, width, height,
mimetype, reproducible=reproducible
)

def __eq__(self, other):
Expand Down
38 changes: 31 additions & 7 deletions sqlalchemy_imageattach/store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class Store(object):
"""

def put_file(self, file, object_type, object_id, width, height, mimetype,
reproducible):
reproducible=False):
"""Puts the ``file`` of the image.

:param file: the image file to put
Expand All @@ -48,6 +48,7 @@ def put_file(self, file, object_type, object_id, width, height, mimetype,
computing e.g. resized thumbnails.
``False`` if it cannot be reproduced
e.g. original images
default is ``False``
:type reproducible: :class:`bool`

.. note::
Expand All @@ -61,7 +62,8 @@ def put_file(self, file, object_type, object_id, width, height, mimetype,
"""
raise NotImplementedError('put_file() has to be implemented')

def delete_file(self, object_type, object_id, width, height, mimetype):
def delete_file(self, object_type, object_id, width, height, mimetype,
reproducible=False):
"""Deletes all reproducible files related to the image.
It doesn't raise any exception even if there's no such file.

Expand All @@ -77,11 +79,18 @@ def delete_file(self, object_type, object_id, width, height, mimetype):
:param mimetype: the mimetype of the image to delete
e.g. ``'image/jpeg'``
:type mimetype: :class:`basestring`
:param reproducible: ``True`` only if it's reproducible by
computing e.g. resized thumbnails.
``False`` if it cannot be reproduced
e.g. original images
default is ``False``
:type reproducible: :class:`bool`

"""
raise NotImplementedError('delete_file() has to be implemented')

def get_file(self, object_type, object_id, width, height, mimetype):
def get_file(self, object_type, object_id, width, height, mimetype,
reproducible=False):
"""Gets the file-like object of the given criteria.

:param object_type: the object type of the image to find
Expand All @@ -96,6 +105,12 @@ def get_file(self, object_type, object_id, width, height, mimetype):
:param mimetype: the mimetype of the image to find
e.g. ``'image/jpeg'``
:type mimetype: :class:`basestring`
:param reproducible: ``True`` only if it's reproducible by
computing e.g. resized thumbnails.
``False`` if it cannot be reproduced
e.g. original images
default is ``False``
:type reproducible: :class:`bool`
:returns: the file of the image
:rtype: file-like object, :class:`file`
:raises exceptions.IOError: when such file doesn't exist
Expand All @@ -111,7 +126,8 @@ def get_file(self, object_type, object_id, width, height, mimetype):
"""
raise NotImplementedError('get_file() has to be implemented')

def get_url(self, object_type, object_id, width, height, mimetype):
def get_url(self, object_type, object_id, width, height, mimetype,
reproducible=False):
"""Gets the file-like object of the given criteria.

:param object_type: the object type of the image to find
Expand All @@ -126,6 +142,12 @@ def get_url(self, object_type, object_id, width, height, mimetype):
:param mimetype: the mimetype of the image to find
e.g. ``'image/jpeg'``
:type mimetype: :class:`basestring`
:param reproducible: ``True`` only if it's reproducible by
computing e.g. resized thumbnails.
``False`` if it cannot be reproduced
e.g. original images
default is ``False``
:type reproducible: :class:`bool`
:returns: the url locating the image
:rtype: :class:`basestring`

Expand Down Expand Up @@ -162,7 +184,7 @@ def store(self, image, file):
'implements read() method, not ' + repr(file))
self.put_file(file, image.object_type, image.object_id,
image.width, image.height, image.mimetype,
not image.original)
reproducible=not image.original)

def delete(self, image):
"""Delete the file of the given ``image``.
Expand All @@ -176,7 +198,8 @@ def delete(self, image):
raise TypeError('image must be a sqlalchemy_imageattach.entity.'
'Image instance, not ' + repr(image))
self.delete_file(image.object_type, image.object_id,
image.width, image.height, image.mimetype)
image.width, image.height, image.mimetype,
reproducible=not image.original)

def open(self, image, use_seek=False):
"""Opens the file-like object of the given ``image``.
Expand Down Expand Up @@ -267,7 +290,8 @@ def locate(self, image):
raise TypeError('image must be a sqlalchemy_imageattach.entity.'
'Image instance, not ' + repr(image))
url = self.get_url(image.object_type, image.object_id,
image.width, image.height, image.mimetype)
image.width, image.height, image.mimetype,
reproducible=not image.original)
if '?' in url:
fmt = '{0}&_ts={1}'
else:
Expand Down
33 changes: 22 additions & 11 deletions sqlalchemy_imageattach/stores/fs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,19 +56,25 @@ class BaseFileSystemStore(Store):

"""

def __init__(self, path):
def __init__(self, path, original_prefix='', reproducible_prefix=''):
self.path = path

def get_path(self, object_type, object_id, width, height, mimetype):
if original_prefix.endswith('/'):
original_prefix = original_prefix.rstrip('/')
self.original_prefix = original_prefix
if reproducible_prefix.endswith('/'):
reproducible_prefix = reproducible_prefix.rstrip('/')
self.reproducible_prefix = reproducible_prefix

def get_path(self, object_type, object_id, width, height, mimetype, reproducible=False):
id_segment_a = str(object_id % 1000)
id_segment_b = str(object_id // 1000)
suffix = guess_extension(mimetype)
filename = '{0}.{1}x{2}{3}'.format(object_id, width, height, suffix)
return object_type, id_segment_a, id_segment_b, filename
prefix = self.reproducible_prefix if reproducible else self.original_prefix
return prefix, id_segment_a, id_segment_b, filename

def put_file(self, file, object_type, object_id, width, height, mimetype,
reproducible):
path = self.get_path(object_type, object_id, width, height, mimetype)
def put_file(self, file, object_type, object_id, width, height, mimetype, reproducible=False):
path = self.get_path(object_type, object_id, width, height, mimetype, reproducible=reproducible)
for i in range(len(path)):
d = os.path.join(self.path, *path[:i])
if not os.path.isdir(d):
Expand Down Expand Up @@ -104,8 +110,10 @@ class FileSystemStore(BaseFileSystemStore):

"""

def __init__(self, path, base_url):
super(FileSystemStore, self).__init__(path)
def __init__(self, path, base_url, original_prefix='', reproducible_prefix=''):
super(FileSystemStore, self).__init__(path,
original_prefix=original_prefix,
reproducible_prefix=reproducible_prefix)
if not base_url.endswith('/'):
base_url += '/'
self.base_url = base_url
Expand Down Expand Up @@ -174,10 +182,13 @@ class HttpExposedFileSystemStore(BaseFileSystemStore):

"""

def __init__(self, path, prefix='__images__', host_url_getter=None):
def __init__(self, path, prefix='__images__', host_url_getter=None,
original_prefix='', reproducible_prefix=''):
if not (callable(host_url_getter) or host_url_getter is None):
raise TypeError('host_url_getter must be callable')
super(HttpExposedFileSystemStore, self).__init__(path)
super(HttpExposedFileSystemStore, self).__init__(path,
original_prefix=original_prefix,
reproducible_prefix=reproducible_prefix)
if prefix.startswith('/'):
prefix = prefix[1:]
if prefix.endswith('/'):
Expand Down
53 changes: 40 additions & 13 deletions sqlalchemy_imageattach/stores/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import hashlib
import hmac
import logging
import os.path
try:
from urllib import request as urllib2
except ImportError:
Expand Down Expand Up @@ -137,9 +138,17 @@ class S3Store(Store):
:param max_age: the ``max-age`` seconds of :mailheader:`Cache-Control`.
default is :const:`DEFAULT_MAX_AGE`
:type max_age: :class:`numbers.Integral`
:param acl: the ``acl`` option of uploaded key. default is :const:`public-read`
:type acl: :class:`basestring`
:param prefix: the optional key prefix to logically separate stores
with the same bucket. not used by default
:type prefix: :class:`basestring`
:param original_prefix: the optional key prefix to logically separate stores
with the same bucket. not used by default
:type original_prefix: :class:`basestring`
:param reproducible_prefix: the optional key prefix to logically separate stores
with the same bucket. not used by default
:type reproducible_prefix: :class:`basestring`
:param public_base_url: an optional url base for public urls.
useful when used with cdn
:type public_base_url: :class:`basestring`
Expand All @@ -160,37 +169,56 @@ class S3Store(Store):
#: :mailheader:`Cache-Control`.
max_age = None

#: (:class:`basestring`) The ``acl`` option of uploaded key.
acl = None

#: (:class:`basestring`) The optional key prefix to logically separate
#: stores with the same bucket.
prefix = None

#: (:class:`basestring`) The optional key prefix to logically separate
#: stores with the same bucket.
original_prefix = None

#: (:class:`basestring`) The optional key prefix to logically separate
#: stores with the same bucket.
reproducible_prefix = None

#: (:class:`basestring`) The optional url base for public urls.
public_base_url = None

def __init__(self, bucket, access_key=None, secret_key=None,
max_age=DEFAULT_MAX_AGE, prefix='', public_base_url=None):
def __init__(self, bucket, access_key=None, secret_key=None, max_age=DEFAULT_MAX_AGE, acl='public-read',
prefix='', original_prefix='', reproducible_prefix='', public_base_url=None):
self.bucket = bucket
self.access_key = access_key
self.secret_key = secret_key
self.base_url = BASE_URL_FORMAT.format(bucket)
self.max_age = max_age
self.acl = acl
self.prefix = prefix.strip()
if self.prefix.endswith('/'):
self.prefix = self.prefix.rstrip('/')
self.original_prefix = original_prefix.strip()
if self.original_prefix.endswith('/'):
self.original_prefix = self.original_prefix.rstrip('/')
self.reproducible_prefix = reproducible_prefix.strip()
if self.reproducible_prefix.endswith('/'):
self.reproducible_prefix = self.reproducible_prefix.rstrip('/')
if public_base_url is None:
self.public_base_url = self.base_url
elif public_base_url.endswith('/'):
self.public_base_url = public_base_url.rstrip('/')
else:
self.public_base_url = public_base_url

def get_key(self, object_type, object_id, width, height, mimetype):
def get_key(self, object_type, object_id, width, height, mimetype, reproducible=False):
key = '{0}/{1}/{2}x{3}{4}'.format(
object_type, object_id, width, height,
guess_extension(mimetype)
)
if self.prefix:
return '{0}/{1}'.format(self.prefix, key)
prefix = os.path.join(self.prefix, self.reproducible_prefix if reproducible else self.original_prefix)
if prefix:
return '{0}/{1}'.format(prefix, key)
return key

def get_file(self, *args, **kwargs):
Expand All @@ -217,10 +245,10 @@ def make_request(self, url, *args, **kwargs):
secret_key=self.secret_key,
**kwargs)

def upload_file(self, url, data, content_type, rrs, acl='public-read'):
def upload_file(self, url, data, content_type, rrs):
headers = {
'Cache-Control': 'max-age=' + str(self.max_age),
'x-amz-acl': acl,
'x-amz-acl': self.acl,
'x-amz-storage-class': 'REDUCED_REDUNDANCY' if rrs else 'STANDARD'
}
request = self.make_request(
Expand All @@ -246,9 +274,8 @@ def upload_file(self, url, data, content_type, rrs, acl='public-read'):
else:
break

def put_file(self, file, object_type, object_id, width, height, mimetype,
reproducible):
url = self.get_s3_url(object_type, object_id, width, height, mimetype)
def put_file(self, file, object_type, object_id, width, height, mimetype, reproducible=False):
url = self.get_s3_url(object_type, object_id, width, height, mimetype, reproducible=reproducible)
self.upload_file(url, file.read(), mimetype, rrs=reproducible)

def delete_file(self, *args, **kwargs):
Expand Down Expand Up @@ -337,10 +364,10 @@ def get_url(self, *args, **kwargs):
def put_file(self, *args, **kwargs):
self.overriding.put_file(*args, **kwargs)

def delete_file(self, object_type, object_id, width, height, mimetype):
def delete_file(self, object_type, object_id, width, height, mimetype, reproducible=False):
args = object_type, object_id, width, height, mimetype
self.overriding.delete_file(*args)
url = self.overriding.get_s3_url(*args)
self.overriding.delete_file(*args, reproducible=reproducible)
url = self.overriding.get_s3_url(*args, reproducible=reproducible)
self.overriding.upload_file(
url,
data=b'',
Expand Down