From c2aa9732455a6b2e52ff3e8f55a7c166e20b0c83 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Thu, 4 Aug 2016 12:15:12 -0700 Subject: [PATCH] Remove contrib.multistore_file Remove dependent modules as well. Resolves #470. --- .../oauth2client.contrib.locked_file.rst | 7 - .../oauth2client.contrib.multistore_file.rst | 7 - docs/source/oauth2client.contrib.rst | 2 - oauth2client/contrib/_fcntl_opener.py | 81 --- oauth2client/contrib/_win32_opener.py | 106 ---- oauth2client/contrib/locked_file.py | 224 -------- oauth2client/contrib/multistore_file.py | 505 ------------------ tests/contrib/test_locked_file.py | 244 --------- tests/contrib/test_multistore_file.py | 381 ------------- 9 files changed, 1557 deletions(-) delete mode 100644 docs/source/oauth2client.contrib.locked_file.rst delete mode 100644 docs/source/oauth2client.contrib.multistore_file.rst delete mode 100644 oauth2client/contrib/_fcntl_opener.py delete mode 100644 oauth2client/contrib/_win32_opener.py delete mode 100644 oauth2client/contrib/locked_file.py delete mode 100644 oauth2client/contrib/multistore_file.py delete mode 100644 tests/contrib/test_locked_file.py delete mode 100644 tests/contrib/test_multistore_file.py diff --git a/docs/source/oauth2client.contrib.locked_file.rst b/docs/source/oauth2client.contrib.locked_file.rst deleted file mode 100644 index 1076e29f8..000000000 --- a/docs/source/oauth2client.contrib.locked_file.rst +++ /dev/null @@ -1,7 +0,0 @@ -oauth2client.contrib.locked_file module -======================================= - -.. automodule:: oauth2client.contrib.locked_file - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/oauth2client.contrib.multistore_file.rst b/docs/source/oauth2client.contrib.multistore_file.rst deleted file mode 100644 index 2787b107e..000000000 --- a/docs/source/oauth2client.contrib.multistore_file.rst +++ /dev/null @@ -1,7 +0,0 @@ -oauth2client.contrib.multistore_file module -=========================================== - -.. automodule:: oauth2client.contrib.multistore_file - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/oauth2client.contrib.rst b/docs/source/oauth2client.contrib.rst index 44be6f9df..98a3d1176 100644 --- a/docs/source/oauth2client.contrib.rst +++ b/docs/source/oauth2client.contrib.rst @@ -19,9 +19,7 @@ Submodules oauth2client.contrib.flask_util oauth2client.contrib.gce oauth2client.contrib.keyring_storage - oauth2client.contrib.locked_file oauth2client.contrib.multiprocess_file_storage - oauth2client.contrib.multistore_file oauth2client.contrib.sqlalchemy oauth2client.contrib.xsrfutil diff --git a/oauth2client/contrib/_fcntl_opener.py b/oauth2client/contrib/_fcntl_opener.py deleted file mode 100644 index c9777a9ba..000000000 --- a/oauth2client/contrib/_fcntl_opener.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import errno -import fcntl -import time - -from oauth2client import util -from oauth2client.contrib import locked_file - - -class _FcntlOpener(locked_file._Opener): - """Open, lock, and unlock a file using fcntl.lockf.""" - - def open_and_lock(self, timeout, delay): - """Open the file and lock it. - - Args: - timeout: float, How long to try to lock for. - delay: float, How long to wait between retries - - Raises: - AlreadyLockedException: if the lock is already acquired. - IOError: if the open fails. - IOError: if the file is a symbolic link. - """ - if self._locked: - raise locked_file.AlreadyLockedException( - 'File {0} is already locked'.format(self._filename)) - start_time = time.time() - - util.validate_file(self._filename) - try: - self._fh = open(self._filename, self._mode) - except IOError as e: - # If we can't access with _mode, try _fallback_mode and - # don't lock. - if e.errno in (errno.EPERM, errno.EACCES): - self._fh = open(self._filename, self._fallback_mode) - return - - # We opened in _mode, try to lock the file. - while True: - try: - fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX) - self._locked = True - return - except IOError as e: - # If not retrying, then just pass on the error. - if timeout == 0: - raise - if e.errno != errno.EACCES: - raise - # We could not acquire the lock. Try again. - if (time.time() - start_time) >= timeout: - locked_file.logger.warn('Could not lock %s in %s seconds', - self._filename, timeout) - if self._fh: - self._fh.close() - self._fh = open(self._filename, self._fallback_mode) - return - time.sleep(delay) - - def unlock_and_close(self): - """Close and unlock the file using the fcntl.lockf primitive.""" - if self._locked: - fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN) - self._locked = False - if self._fh: - self._fh.close() diff --git a/oauth2client/contrib/_win32_opener.py b/oauth2client/contrib/_win32_opener.py deleted file mode 100644 index 6fa019671..000000000 --- a/oauth2client/contrib/_win32_opener.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import errno -import time - -import pywintypes -import win32con -import win32file - -from oauth2client import util -from oauth2client.contrib import locked_file - - -class _Win32Opener(locked_file._Opener): - """Open, lock, and unlock a file using windows primitives.""" - - # Error #33: - # 'The process cannot access the file because another process' - FILE_IN_USE_ERROR = 33 - - # Error #158: - # 'The segment is already unlocked.' - FILE_ALREADY_UNLOCKED_ERROR = 158 - - def open_and_lock(self, timeout, delay): - """Open the file and lock it. - - Args: - timeout: float, How long to try to lock for. - delay: float, How long to wait between retries - - Raises: - AlreadyLockedException: if the lock is already acquired. - IOError: if the open fails. - IOError: if the file is a symbolic link. - """ - if self._locked: - raise locked_file.AlreadyLockedException( - 'File {0} is already locked'.format(self._filename)) - start_time = time.time() - - util.validate_file(self._filename) - try: - self._fh = open(self._filename, self._mode) - except IOError as e: - # If we can't access with _mode, try _fallback_mode - # and don't lock. - if e.errno == errno.EACCES: - self._fh = open(self._filename, self._fallback_mode) - return - - # We opened in _mode, try to lock the file. - while True: - try: - hfile = win32file._get_osfhandle(self._fh.fileno()) - win32file.LockFileEx( - hfile, - (win32con.LOCKFILE_FAIL_IMMEDIATELY | - win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000, - pywintypes.OVERLAPPED()) - self._locked = True - return - except pywintypes.error as e: - if timeout == 0: - raise - - # If the error is not that the file is already - # in use, raise. - if e[0] != _Win32Opener.FILE_IN_USE_ERROR: - raise - - # We could not acquire the lock. Try again. - if (time.time() - start_time) >= timeout: - locked_file.logger.warn('Could not lock %s in %s seconds', - self._filename, timeout) - if self._fh: - self._fh.close() - self._fh = open(self._filename, self._fallback_mode) - return - time.sleep(delay) - - def unlock_and_close(self): - """Close and unlock the file using the win32 primitive.""" - if self._locked: - try: - hfile = win32file._get_osfhandle(self._fh.fileno()) - win32file.UnlockFileEx(hfile, 0, -0x10000, - pywintypes.OVERLAPPED()) - except pywintypes.error as e: - if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR: - raise - self._locked = False - if self._fh: - self._fh.close() diff --git a/oauth2client/contrib/locked_file.py b/oauth2client/contrib/locked_file.py deleted file mode 100644 index 9c880d769..000000000 --- a/oauth2client/contrib/locked_file.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Locked file interface that should work on Unix and Windows pythons. - -This module first tries to use fcntl locking to ensure serialized access -to a file, then falls back on a lock file if that is unavialable. - -Usage:: - - f = LockedFile('filename', 'r+b', 'rb') - f.open_and_lock() - if f.is_locked(): - print('Acquired filename with r+b mode') - f.file_handle().write('locked data') - else: - print('Acquired filename with rb mode') - f.unlock_and_close() - -""" - -from __future__ import print_function - -import errno -import logging -import os -import time - -from oauth2client import util - - -__author__ = 'cache@google.com (David T McWherter)' - -logger = logging.getLogger(__name__) - - -class AlreadyLockedException(Exception): - """Trying to lock a file that has already been locked by the LockedFile.""" - pass - - -class _Opener(object): - """Base class for different locking primitives.""" - - def __init__(self, filename, mode, fallback_mode): - """Create an Opener. - - Args: - filename: string, The pathname of the file. - mode: string, The preferred mode to access the file with. - fallback_mode: string, The mode to use if locking fails. - """ - self._locked = False - self._filename = filename - self._mode = mode - self._fallback_mode = fallback_mode - self._fh = None - self._lock_fd = None - - def is_locked(self): - """Was the file locked.""" - return self._locked - - def file_handle(self): - """The file handle to the file. Valid only after opened.""" - return self._fh - - def filename(self): - """The filename that is being locked.""" - return self._filename - - def open_and_lock(self, timeout, delay): - """Open the file and lock it. - - Args: - timeout: float, How long to try to lock for. - delay: float, How long to wait between retries. - """ - pass - - def unlock_and_close(self): - """Unlock and close the file.""" - pass - - -class _PosixOpener(_Opener): - """Lock files using Posix advisory lock files.""" - - def open_and_lock(self, timeout, delay): - """Open the file and lock it. - - Tries to create a .lock file next to the file we're trying to open. - - Args: - timeout: float, How long to try to lock for. - delay: float, How long to wait between retries. - - Raises: - AlreadyLockedException: if the lock is already acquired. - IOError: if the open fails. - IOError: if the file is a symbolic link. - """ - if self._locked: - raise AlreadyLockedException( - 'File {0} is already locked'.format(self._filename)) - self._locked = False - - util.validate_file(self._filename) - try: - self._fh = open(self._filename, self._mode) - except IOError as e: - # If we can't access with _mode, try _fallback_mode and don't lock. - if e.errno == errno.EACCES: - self._fh = open(self._filename, self._fallback_mode) - return - - lock_filename = self._posix_lockfile(self._filename) - start_time = time.time() - while True: - try: - self._lock_fd = os.open(lock_filename, - os.O_CREAT | os.O_EXCL | os.O_RDWR) - self._locked = True - break - - except OSError as e: - if e.errno != errno.EEXIST: - raise - if (time.time() - start_time) >= timeout: - logger.warn('Could not acquire lock %s in %s seconds', - lock_filename, timeout) - # Close the file and open in fallback_mode. - if self._fh: - self._fh.close() - self._fh = open(self._filename, self._fallback_mode) - return - time.sleep(delay) - - def unlock_and_close(self): - """Unlock a file by removing the .lock file, and close the handle.""" - if self._locked: - lock_filename = self._posix_lockfile(self._filename) - os.close(self._lock_fd) - os.unlink(lock_filename) - self._locked = False - self._lock_fd = None - if self._fh: - self._fh.close() - - def _posix_lockfile(self, filename): - """The name of the lock file to use for posix locking.""" - return '{0}.lock'.format(filename) - - -class LockedFile(object): - """Represent a file that has exclusive access.""" - - @util.positional(4) - def __init__(self, filename, mode, fallback_mode, use_native_locking=True): - """Construct a LockedFile. - - Args: - filename: string, The path of the file to open. - mode: string, The mode to try to open the file with. - fallback_mode: string, The mode to use if locking fails. - use_native_locking: bool, Whether or not fcntl/win32 locking is - used. - """ - opener = None - if not opener and use_native_locking: - try: - from oauth2client.contrib._win32_opener import _Win32Opener - opener = _Win32Opener(filename, mode, fallback_mode) - except ImportError: - try: - from oauth2client.contrib._fcntl_opener import _FcntlOpener - opener = _FcntlOpener(filename, mode, fallback_mode) - except ImportError: - pass - - if not opener: - opener = _PosixOpener(filename, mode, fallback_mode) - - self._opener = opener - - def filename(self): - """Return the filename we were constructed with.""" - return self._opener._filename - - def file_handle(self): - """Return the file_handle to the opened file.""" - return self._opener.file_handle() - - def is_locked(self): - """Return whether we successfully locked the file.""" - return self._opener.is_locked() - - def open_and_lock(self, timeout=0, delay=0.05): - """Open the file, trying to lock it. - - Args: - timeout: float, The number of seconds to try to acquire the lock. - delay: float, The number of seconds to wait between retry attempts. - - Raises: - AlreadyLockedException: if the lock is already acquired. - IOError: if the open fails. - """ - self._opener.open_and_lock(timeout, delay) - - def unlock_and_close(self): - """Unlock and close a file.""" - self._opener.unlock_and_close() diff --git a/oauth2client/contrib/multistore_file.py b/oauth2client/contrib/multistore_file.py deleted file mode 100644 index 10f4cb407..000000000 --- a/oauth2client/contrib/multistore_file.py +++ /dev/null @@ -1,505 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Multi-credential file store with lock support. - -This module implements a JSON credential store where multiple -credentials can be stored in one file. That file supports locking -both in a single process and across processes. - -The credential themselves are keyed off of: - -* client_id -* user_agent -* scope - -The format of the stored data is like so:: - - { - 'file_version': 1, - 'data': [ - { - 'key': { - 'clientId': '', - 'userAgent': '', - 'scope': '' - }, - 'credential': { - # JSON serialized Credentials. - } - } - ] - } - -""" - -import errno -import json -import logging -import os -import threading - -from oauth2client import client -from oauth2client import util -from oauth2client.contrib import locked_file - -__author__ = 'jbeda@google.com (Joe Beda)' - -logger = logging.getLogger(__name__) - -logger.warning( - 'The oauth2client.contrib.multistore_file module has been deprecated and ' - 'will be removed in the next release of oauth2client. Please migrate to ' - 'multiprocess_file_storage.') - -# A dict from 'filename'->_MultiStore instances -_multistores = {} -_multistores_lock = threading.Lock() - - -class Error(Exception): - """Base error for this module.""" - - -class NewerCredentialStoreError(Error): - """The credential store is a newer version than supported.""" - - -def _dict_to_tuple_key(dictionary): - """Converts a dictionary to a tuple that can be used as an immutable key. - - The resulting key is always sorted so that logically equivalent - dictionaries always produce an identical tuple for a key. - - Args: - dictionary: the dictionary to use as the key. - - Returns: - A tuple representing the dictionary in it's naturally sorted ordering. - """ - return tuple(sorted(dictionary.items())) - - -@util.positional(4) -def get_credential_storage(filename, client_id, user_agent, scope, - warn_on_readonly=True): - """Get a Storage instance for a credential. - - Args: - filename: The JSON file storing a set of credentials - client_id: The client_id for the credential - user_agent: The user agent for the credential - scope: string or iterable of strings, Scope(s) being requested - warn_on_readonly: if True, log a warning if the store is readonly - - Returns: - An object derived from client.Storage for getting/setting the - credential. - """ - # Recreate the legacy key with these specific parameters - key = {'clientId': client_id, 'userAgent': user_agent, - 'scope': util.scopes_to_string(scope)} - return get_credential_storage_custom_key( - filename, key, warn_on_readonly=warn_on_readonly) - - -@util.positional(2) -def get_credential_storage_custom_string_key(filename, key_string, - warn_on_readonly=True): - """Get a Storage instance for a credential using a single string as a key. - - Allows you to provide a string as a custom key that will be used for - credential storage and retrieval. - - Args: - filename: The JSON file storing a set of credentials - key_string: A string to use as the key for storing this credential. - warn_on_readonly: if True, log a warning if the store is readonly - - Returns: - An object derived from client.Storage for getting/setting the - credential. - """ - # Create a key dictionary that can be used - key_dict = {'key': key_string} - return get_credential_storage_custom_key( - filename, key_dict, warn_on_readonly=warn_on_readonly) - - -@util.positional(2) -def get_credential_storage_custom_key(filename, key_dict, - warn_on_readonly=True): - """Get a Storage instance for a credential using a dictionary as a key. - - Allows you to provide a dictionary as a custom key that will be used for - credential storage and retrieval. - - Args: - filename: The JSON file storing a set of credentials - key_dict: A dictionary to use as the key for storing this credential. - There is no ordering of the keys in the dictionary. Logically - equivalent dictionaries will produce equivalent storage keys. - warn_on_readonly: if True, log a warning if the store is readonly - - Returns: - An object derived from client.Storage for getting/setting the - credential. - """ - multistore = _get_multistore(filename, warn_on_readonly=warn_on_readonly) - key = _dict_to_tuple_key(key_dict) - return multistore._get_storage(key) - - -@util.positional(1) -def get_all_credential_keys(filename, warn_on_readonly=True): - """Gets all the registered credential keys in the given Multistore. - - Args: - filename: The JSON file storing a set of credentials - warn_on_readonly: if True, log a warning if the store is readonly - - Returns: - A list of the credential keys present in the file. They are returned - as dictionaries that can be passed into - get_credential_storage_custom_key to get the actual credentials. - """ - multistore = _get_multistore(filename, warn_on_readonly=warn_on_readonly) - multistore._lock() - try: - return multistore._get_all_credential_keys() - finally: - multistore._unlock() - - -@util.positional(1) -def _get_multistore(filename, warn_on_readonly=True): - """A helper method to initialize the multistore with proper locking. - - Args: - filename: The JSON file storing a set of credentials - warn_on_readonly: if True, log a warning if the store is readonly - - Returns: - A multistore object - """ - filename = os.path.expanduser(filename) - _multistores_lock.acquire() - try: - multistore = _multistores.setdefault( - filename, _MultiStore(filename, warn_on_readonly=warn_on_readonly)) - finally: - _multistores_lock.release() - return multistore - - -class _MultiStore(object): - """A file backed store for multiple credentials.""" - - @util.positional(2) - def __init__(self, filename, warn_on_readonly=True): - """Initialize the class. - - This will create the file if necessary. - """ - self._file = locked_file.LockedFile(filename, 'r+', 'r') - self._thread_lock = threading.Lock() - self._read_only = False - self._warn_on_readonly = warn_on_readonly - - self._create_file_if_needed() - - # Cache of deserialized store. This is only valid after the - # _MultiStore is locked or _refresh_data_cache is called. This is - # of the form of: - # - # ((key, value), (key, value)...) -> OAuth2Credential - # - # If this is None, then the store hasn't been read yet. - self._data = None - - class _Storage(client.Storage): - """A Storage object that can read/write a single credential.""" - - def __init__(self, multistore, key): - self._multistore = multistore - self._key = key - - def acquire_lock(self): - """Acquires any lock necessary to access this Storage. - - This lock is not reentrant. - """ - self._multistore._lock() - - def release_lock(self): - """Release the Storage lock. - - Trying to release a lock that isn't held will result in a - RuntimeError. - """ - self._multistore._unlock() - - def locked_get(self): - """Retrieve credential. - - The Storage lock must be held when this is called. - - Returns: - oauth2client.client.Credentials - """ - credential = self._multistore._get_credential(self._key) - if credential: - credential.set_store(self) - return credential - - def locked_put(self, credentials): - """Write a credential. - - The Storage lock must be held when this is called. - - Args: - credentials: Credentials, the credentials to store. - """ - self._multistore._update_credential(self._key, credentials) - - def locked_delete(self): - """Delete a credential. - - The Storage lock must be held when this is called. - - Args: - credentials: Credentials, the credentials to store. - """ - self._multistore._delete_credential(self._key) - - def _create_file_if_needed(self): - """Create an empty file if necessary. - - This method will not initialize the file. Instead it implements a - simple version of "touch" to ensure the file has been created. - """ - if not os.path.exists(self._file.filename()): - old_umask = os.umask(0o177) - try: - open(self._file.filename(), 'a+b').close() - finally: - os.umask(old_umask) - - def _lock(self): - """Lock the entire multistore.""" - self._thread_lock.acquire() - try: - self._file.open_and_lock() - except (IOError, OSError) as e: - if e.errno == errno.ENOSYS: - logger.warn('File system does not support locking the ' - 'credentials file.') - elif e.errno == errno.ENOLCK: - logger.warn('File system is out of resources for writing the ' - 'credentials file (is your disk full?).') - elif e.errno == errno.EDEADLK: - logger.warn('Lock contention on multistore file, opening ' - 'in read-only mode.') - elif e.errno == errno.EACCES: - logger.warn('Cannot access credentials file.') - else: - raise - if not self._file.is_locked(): - self._read_only = True - if self._warn_on_readonly: - logger.warn('The credentials file (%s) is not writable. ' - 'Opening in read-only mode. Any refreshed ' - 'credentials will only be ' - 'valid for this run.', self._file.filename()) - - if os.path.getsize(self._file.filename()) == 0: - logger.debug('Initializing empty multistore file') - # The multistore is empty so write out an empty file. - self._data = {} - self._write() - elif not self._read_only or self._data is None: - # Only refresh the data if we are read/write or we haven't - # cached the data yet. If we are readonly, we assume is isn't - # changing out from under us and that we only have to read it - # once. This prevents us from whacking any new access keys that - # we have cached in memory but were unable to write out. - self._refresh_data_cache() - - def _unlock(self): - """Release the lock on the multistore.""" - self._file.unlock_and_close() - self._thread_lock.release() - - def _locked_json_read(self): - """Get the raw content of the multistore file. - - The multistore must be locked when this is called. - - Returns: - The contents of the multistore decoded as JSON. - """ - assert self._thread_lock.locked() - self._file.file_handle().seek(0) - return json.load(self._file.file_handle()) - - def _locked_json_write(self, data): - """Write a JSON serializable data structure to the multistore. - - The multistore must be locked when this is called. - - Args: - data: The data to be serialized and written. - """ - assert self._thread_lock.locked() - if self._read_only: - return - self._file.file_handle().seek(0) - json.dump(data, self._file.file_handle(), - sort_keys=True, indent=2, separators=(',', ': ')) - self._file.file_handle().truncate() - - def _refresh_data_cache(self): - """Refresh the contents of the multistore. - - The multistore must be locked when this is called. - - Raises: - NewerCredentialStoreError: Raised when a newer client has written - the store. - """ - self._data = {} - try: - raw_data = self._locked_json_read() - except Exception: - logger.warn('Credential data store could not be loaded. ' - 'Will ignore and overwrite.') - return - - version = 0 - try: - version = raw_data['file_version'] - except Exception: - logger.warn('Missing version for credential data store. It may be ' - 'corrupt or an old version. Overwriting.') - if version > 1: - raise NewerCredentialStoreError( - 'Credential file has file_version of {0}. ' - 'Only file_version of 1 is supported.'.format(version)) - - credentials = [] - try: - credentials = raw_data['data'] - except (TypeError, KeyError): - pass - - for cred_entry in credentials: - try: - key, credential = self._decode_credential_from_json(cred_entry) - self._data[key] = credential - except: - # If something goes wrong loading a credential, just ignore it - logger.info('Error decoding credential, skipping', - exc_info=True) - - def _decode_credential_from_json(self, cred_entry): - """Load a credential from our JSON serialization. - - Args: - cred_entry: A dict entry from the data member of our format - - Returns: - (key, cred) where the key is the key tuple and the cred is the - OAuth2Credential object. - """ - raw_key = cred_entry['key'] - key = _dict_to_tuple_key(raw_key) - credential = None - credential = client.Credentials.new_from_json( - json.dumps(cred_entry['credential'])) - return (key, credential) - - def _write(self): - """Write the cached data back out. - - The multistore must be locked. - """ - raw_data = {'file_version': 1} - raw_creds = [] - raw_data['data'] = raw_creds - for (cred_key, cred) in self._data.items(): - raw_key = dict(cred_key) - raw_cred = json.loads(cred.to_json()) - raw_creds.append({'key': raw_key, 'credential': raw_cred}) - self._locked_json_write(raw_data) - - def _get_all_credential_keys(self): - """Gets all the registered credential keys in the multistore. - - Returns: - A list of dictionaries corresponding to all the keys currently - registered - """ - return [dict(key) for key in self._data.keys()] - - def _get_credential(self, key): - """Get a credential from the multistore. - - The multistore must be locked. - - Args: - key: The key used to retrieve the credential - - Returns: - The credential specified or None if not present - """ - return self._data.get(key, None) - - def _update_credential(self, key, cred): - """Update a credential and write the multistore. - - This must be called when the multistore is locked. - - Args: - key: The key used to retrieve the credential - cred: The OAuth2Credential to update/set - """ - self._data[key] = cred - self._write() - - def _delete_credential(self, key): - """Delete a credential and write the multistore. - - This must be called when the multistore is locked. - - Args: - key: The key used to retrieve the credential - """ - try: - del self._data[key] - except KeyError: - pass - self._write() - - def _get_storage(self, key): - """Get a Storage object to get/set a credential. - - This Storage is a 'view' into the multistore. - - Args: - key: The key used to retrieve the credential - - Returns: - A Storage object that can be used to get/set this cred - """ - return self._Storage(self, key) diff --git a/tests/contrib/test_locked_file.py b/tests/contrib/test_locked_file.py deleted file mode 100644 index 384bef3f4..000000000 --- a/tests/contrib/test_locked_file.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2016 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import errno -import os -import sys -import tempfile - -import mock -import unittest2 - -from oauth2client.contrib import locked_file - - -class TestOpener(unittest2.TestCase): - def _make_one(self): - _filehandle, filename = tempfile.mkstemp() - os.close(_filehandle) - return locked_file._Opener(filename, 'r+', 'r'), filename - - def test_ctor(self): - instance, filename = self._make_one() - self.assertFalse(instance._locked) - self.assertEqual(instance._filename, filename) - self.assertEqual(instance._mode, 'r+') - self.assertEqual(instance._fallback_mode, 'r') - self.assertIsNone(instance._fh) - self.assertIsNone(instance._lock_fd) - - def test_is_locked(self): - instance, _ = self._make_one() - self.assertFalse(instance.is_locked()) - instance._locked = True - self.assertTrue(instance.is_locked()) - - def test_file_handle(self): - instance, _ = self._make_one() - self.assertIsNone(instance.file_handle()) - fh = mock.Mock() - instance._fh = fh - self.assertEqual(instance.file_handle(), fh) - - def test_filename(self): - instance, filename = self._make_one() - self.assertEqual(instance.filename(), filename) - - def test_open_and_lock(self): - instance, _ = self._make_one() - instance.open_and_lock(1, 1) - - def test_unlock_and_close(self): - instance, _ = self._make_one() - instance.unlock_and_close() - - -class TestPosixOpener(TestOpener): - def _make_one(self): - _filehandle, filename = tempfile.mkstemp() - os.close(_filehandle) - return locked_file._PosixOpener(filename, 'r+', 'r'), filename - - def test_relock_fail(self): - instance, _ = self._make_one() - instance.open_and_lock(1, 1) - - self.assertTrue(instance.is_locked()) - self.assertIsNotNone(instance.file_handle()) - with self.assertRaises(locked_file.AlreadyLockedException): - instance.open_and_lock(1, 1) - - @mock.patch('oauth2client.contrib.locked_file.open', create=True) - def test_lock_access_error_fallback_mode(self, mock_open): - # NOTE: This is a bad case. The behavior here should be that the - # error gets re-raised, but the module lets the if statement fall - # through. - instance, _ = self._make_one() - mock_open.side_effect = [IOError(errno.ENOENT, '')] - instance.open_and_lock(1, 1) - - self.assertIsNone(instance.file_handle()) - self.assertTrue(instance.is_locked()) - - @mock.patch('oauth2client.contrib.locked_file.open', create=True) - def test_lock_non_access_error(self, mock_open): - instance, _ = self._make_one() - fh_mock = mock.Mock() - mock_open.side_effect = [IOError(errno.EACCES, ''), fh_mock] - instance.open_and_lock(1, 1) - - self.assertEqual(instance.file_handle(), fh_mock) - self.assertFalse(instance.is_locked()) - - @mock.patch('oauth2client.contrib.locked_file.open', create=True) - def test_lock_unexpected_error(self, mock_open): - instance, _ = self._make_one() - - with mock.patch('os.open') as mock_os_open: - mock_os_open.side_effect = [OSError(errno.EPERM, '')] - with self.assertRaises(OSError): - instance.open_and_lock(1, 1) - - @mock.patch('oauth2client.contrib.locked_file.open', create=True) - @mock.patch('oauth2client.contrib.locked_file.logger') - @mock.patch('time.time') - def test_lock_timeout_error(self, mock_time, mock_logger, mock_open): - instance, _ = self._make_one() - # Make it seem like 10 seconds have passed between calls. - mock_time.side_effect = [0, 10] - - with mock.patch('os.open') as mock_os_open: - # Raising EEXIST should cause it to try to retry locking. - mock_os_open.side_effect = [OSError(errno.EEXIST, '')] - instance.open_and_lock(1, 1) - self.assertFalse(instance.is_locked()) - self.assertTrue(mock_logger.warn.called) - - @mock.patch('oauth2client.contrib.locked_file.open', create=True) - @mock.patch('oauth2client.contrib.locked_file.logger') - @mock.patch('time.time') - def test_lock_timeout_error_no_fh(self, mock_time, mock_logger, mock_open): - instance, _ = self._make_one() - # Make it seem like 10 seconds have passed between calls. - mock_time.side_effect = [0, 10] - # This will cause the retry loop to enter without a file handle. - fh_mock = mock.Mock() - mock_open.side_effect = [IOError(errno.ENOENT, ''), fh_mock] - - with mock.patch('os.open') as mock_os_open: - # Raising EEXIST should cause it to try to retry locking. - mock_os_open.side_effect = [OSError(errno.EEXIST, '')] - instance.open_and_lock(1, 1) - self.assertFalse(instance.is_locked()) - self.assertTrue(mock_logger.warn.called) - self.assertEqual(instance.file_handle(), fh_mock) - - @mock.patch('oauth2client.contrib.locked_file.open', create=True) - @mock.patch('time.time') - @mock.patch('time.sleep') - def test_lock_retry_success(self, mock_sleep, mock_time, mock_open): - instance, _ = self._make_one() - # Make it seem like 1 second has passed between calls. Extra values - # are needed by the logging module. - mock_time.side_effect = [0, 1] - - with mock.patch('os.open') as mock_os_open: - # Raising EEXIST should cause it to try to retry locking. - mock_os_open.side_effect = [ - OSError(errno.EEXIST, ''), mock.Mock()] - instance.open_and_lock(10, 1) - print(mock_os_open.call_args_list) - self.assertTrue(instance.is_locked()) - mock_sleep.assert_called_with(1) - - @mock.patch('oauth2client.contrib.locked_file.os') - def test_unlock(self, os_mock): - instance, _ = self._make_one() - instance._locked = True - lock_fd_mock = instance._lock_fd = mock.Mock() - instance._fh = mock.Mock() - - instance.unlock_and_close() - - self.assertFalse(instance.is_locked()) - os_mock.close.assert_called_once_with(lock_fd_mock) - self.assertTrue(os_mock.unlink.called) - self.assertTrue(instance._fh.close.called) - - -class TestLockedFile(unittest2.TestCase): - - @mock.patch('oauth2client.contrib.locked_file._PosixOpener') - def _make_one(self, opener_ctor_mock): - opener_mock = mock.Mock() - opener_ctor_mock.return_value = opener_mock - return locked_file.LockedFile( - 'a_file', 'r+', 'r', use_native_locking=False), opener_mock - - @mock.patch('oauth2client.contrib.locked_file._PosixOpener') - def test_ctor_minimal(self, opener_mock): - locked_file.LockedFile( - 'a_file', 'r+', 'r', use_native_locking=False) - opener_mock.assert_called_with('a_file', 'r+', 'r') - - @mock.patch.dict('sys.modules', { - 'oauth2client.contrib._win32_opener': mock.Mock()}) - def test_ctor_native_win32(self): - _win32_opener_mock = sys.modules['oauth2client.contrib._win32_opener'] - locked_file.LockedFile( - 'a_file', 'r+', 'r', use_native_locking=True) - _win32_opener_mock._Win32Opener.assert_called_with('a_file', 'r+', 'r') - - @mock.patch.dict('sys.modules', { - 'oauth2client.contrib._win32_opener': None, - 'oauth2client.contrib._fcntl_opener': mock.Mock()}) - def test_ctor_native_fcntl(self): - _fnctl_opener_mock = sys.modules['oauth2client.contrib._fcntl_opener'] - locked_file.LockedFile( - 'a_file', 'r+', 'r', use_native_locking=True) - _fnctl_opener_mock._FcntlOpener.assert_called_with('a_file', 'r+', 'r') - - @mock.patch('oauth2client.contrib.locked_file._PosixOpener') - @mock.patch.dict('sys.modules', { - 'oauth2client.contrib._win32_opener': None, - 'oauth2client.contrib._fcntl_opener': None}) - def test_ctor_native_posix_fallback(self, opener_mock): - locked_file.LockedFile( - 'a_file', 'r+', 'r', use_native_locking=True) - opener_mock.assert_called_with('a_file', 'r+', 'r') - - def test_filename(self): - instance, opener = self._make_one() - opener._filename = 'some file' - self.assertEqual(instance.filename(), 'some file') - - def test_file_handle(self): - instance, opener = self._make_one() - self.assertEqual(instance.file_handle(), opener.file_handle()) - self.assertTrue(opener.file_handle.called) - - def test_is_locked(self): - instance, opener = self._make_one() - self.assertEqual(instance.is_locked(), opener.is_locked()) - self.assertTrue(opener.is_locked.called) - - def test_open_and_lock(self): - instance, opener = self._make_one() - instance.open_and_lock() - opener.open_and_lock.assert_called_with(0, 0.05) - - def test_unlock_and_close(self): - instance, opener = self._make_one() - instance.unlock_and_close() - opener.unlock_and_close.assert_called_with() diff --git a/tests/contrib/test_multistore_file.py b/tests/contrib/test_multistore_file.py deleted file mode 100644 index ae0664d18..000000000 --- a/tests/contrib/test_multistore_file.py +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for oauth2client.multistore_file.""" - -import datetime -import errno -import os -import stat -import tempfile - -import mock -import unittest2 - -from oauth2client import client -from oauth2client import util -from oauth2client.contrib import multistore_file - -_filehandle, FILENAME = tempfile.mkstemp('oauth2client_test.data') -os.close(_filehandle) - - -class _MockLockedFile(object): - - def __init__(self, filename_str, error_class, error_code): - self.filename_str = filename_str - self.error_class = error_class - self.error_code = error_code - self.open_and_lock_called = False - - def open_and_lock(self): - self.open_and_lock_called = True - raise self.error_class(self.error_code, '') - - def is_locked(self): - return False - - def filename(self): - return self.filename_str - - -class Test__dict_to_tuple_key(unittest2.TestCase): - - def test_key_conversions(self): - key1, val1 = 'somekey', 'some value' - key2, val2 = 'another', 'something else' - key3, val3 = 'onemore', 'foo' - test_dict = { - key1: val1, - key2: val2, - key3: val3, - } - tuple_key = multistore_file._dict_to_tuple_key(test_dict) - - # the resulting key should be naturally sorted - expected_output = ( - (key2, val2), - (key3, val3), - (key1, val1), - ) - self.assertTupleEqual(expected_output, tuple_key) - # check we get the original dictionary back - self.assertDictEqual(test_dict, dict(tuple_key)) - - -class MultistoreFileTests(unittest2.TestCase): - - def tearDown(self): - try: - os.unlink(FILENAME) - except OSError: - pass - - def setUp(self): - try: - os.unlink(FILENAME) - except OSError: - pass - - def _create_test_credentials(self, client_id='some_client_id', - expiration=None): - access_token = 'foo' - client_secret = 'cOuDdkfjxxnv+' - refresh_token = '1/0/a.df219fjls0' - token_expiry = expiration or datetime.datetime.utcnow() - token_uri = 'https://www.google.com/accounts/o8/oauth2/token' - user_agent = 'refresh_checker/1.0' - - credentials = client.OAuth2Credentials( - access_token, client_id, client_secret, - refresh_token, token_expiry, token_uri, - user_agent) - return credentials - - def test_lock_file_raises_ioerror(self): - filehandle, filename = tempfile.mkstemp() - os.close(filehandle) - - try: - for error_code in (errno.EDEADLK, errno.ENOSYS, errno.ENOLCK, - errno.EACCES): - for error_class in (IOError, OSError): - multistore = multistore_file._MultiStore(filename) - multistore._file = _MockLockedFile( - filename, error_class, error_code) - # Should not raise though the underlying file class did. - multistore._lock() - self.assertTrue(multistore._file.open_and_lock_called) - finally: - os.unlink(filename) - - def test_lock_file_raise_unexpected_error(self): - filehandle, filename = tempfile.mkstemp() - os.close(filehandle) - - try: - multistore = multistore_file._MultiStore(filename) - multistore._file = _MockLockedFile(filename, IOError, errno.EBUSY) - with self.assertRaises(IOError): - multistore._lock() - self.assertTrue(multistore._file.open_and_lock_called) - finally: - os.unlink(filename) - - def test_read_only_file_fail_lock(self): - credentials = self._create_test_credentials() - - open(FILENAME, 'a+b').close() - os.chmod(FILENAME, 0o400) - - store = multistore_file.get_credential_storage( - FILENAME, - credentials.client_id, - credentials.user_agent, - ['some-scope', 'some-other-scope']) - - store.put(credentials) - if os.name == 'posix': # pragma: NO COVER - self.assertTrue(store._multistore._read_only) - os.chmod(FILENAME, 0o600) - - def test_read_only_file_fail_lock_no_warning(self): - open(FILENAME, 'a+b').close() - os.chmod(FILENAME, 0o400) - - multistore = multistore_file._MultiStore(FILENAME) - - with mock.patch.object(multistore_file.logger, 'warn') as mock_warn: - multistore._warn_on_readonly = False - multistore._lock() - self.assertFalse(mock_warn.called) - - def test_lock_skip_refresh(self): - with open(FILENAME, 'w') as f: - f.write('123') - os.chmod(FILENAME, 0o400) - - multistore = multistore_file._MultiStore(FILENAME) - - refresh_patch = mock.patch.object( - multistore, '_refresh_data_cache') - - with refresh_patch as refresh_mock: - multistore._data = {} - multistore._lock() - self.assertFalse(refresh_mock.called) - - @unittest2.skipIf(not hasattr(os, 'symlink'), 'No symlink available') - def test_multistore_no_symbolic_link_files(self): - SYMFILENAME = FILENAME + 'sym' - os.symlink(FILENAME, SYMFILENAME) - store = multistore_file.get_credential_storage( - SYMFILENAME, - 'some_client_id', - 'user-agent/1.0', - ['some-scope', 'some-other-scope']) - try: - with self.assertRaises(IOError): - store.get() - finally: - os.unlink(SYMFILENAME) - - def test_multistore_non_existent_file(self): - store = multistore_file.get_credential_storage( - FILENAME, - 'some_client_id', - 'user-agent/1.0', - ['some-scope', 'some-other-scope']) - - credentials = store.get() - self.assertIsNone(credentials) - - def test_multistore_file(self): - credentials = self._create_test_credentials() - - store = multistore_file.get_credential_storage( - FILENAME, - credentials.client_id, - credentials.user_agent, - ['some-scope', 'some-other-scope']) - - # Save credentials - store.put(credentials) - credentials = store.get() - - self.assertIsNotNone(credentials) - self.assertEquals('foo', credentials.access_token) - - # Delete credentials - store.delete() - credentials = store.get() - - self.assertIsNone(credentials) - - if os.name == 'posix': # pragma: NO COVER - self.assertEquals( - 0o600, stat.S_IMODE(os.stat(FILENAME).st_mode)) - - def test_multistore_file_custom_key(self): - credentials = self._create_test_credentials() - - custom_key = {'myapp': 'testing', 'clientid': 'some client'} - store = multistore_file.get_credential_storage_custom_key( - FILENAME, custom_key) - - store.put(credentials) - stored_credentials = store.get() - - self.assertIsNotNone(stored_credentials) - self.assertEqual(credentials.access_token, - stored_credentials.access_token) - - store.delete() - stored_credentials = store.get() - - self.assertIsNone(stored_credentials) - - def test_multistore_file_custom_string_key(self): - credentials = self._create_test_credentials() - - # store with string key - store = multistore_file.get_credential_storage_custom_string_key( - FILENAME, 'mykey') - - store.put(credentials) - stored_credentials = store.get() - - self.assertIsNotNone(stored_credentials) - self.assertEqual(credentials.access_token, - stored_credentials.access_token) - - # try retrieving with a dictionary - multistore_file.get_credential_storage_custom_string_key( - FILENAME, {'key': 'mykey'}) - stored_credentials = store.get() - self.assertIsNotNone(stored_credentials) - self.assertEqual(credentials.access_token, - stored_credentials.access_token) - - store.delete() - stored_credentials = store.get() - - self.assertIsNone(stored_credentials) - - def test_multistore_file_backwards_compatibility(self): - credentials = self._create_test_credentials() - scopes = ['scope1', 'scope2'] - - # store the credentials using the legacy key method - store = multistore_file.get_credential_storage( - FILENAME, 'client_id', 'user_agent', scopes) - store.put(credentials) - - # retrieve the credentials using a custom key that matches the - # legacy key - key = {'clientId': 'client_id', 'userAgent': 'user_agent', - 'scope': util.scopes_to_string(scopes)} - store = multistore_file.get_credential_storage_custom_key( - FILENAME, key) - stored_credentials = store.get() - - self.assertEqual(credentials.access_token, - stored_credentials.access_token) - - def test_multistore_file_get_all_keys(self): - # start with no keys - keys = multistore_file.get_all_credential_keys(FILENAME) - self.assertEquals([], keys) - - # store credentials - credentials = self._create_test_credentials(client_id='client1') - custom_key = {'myapp': 'testing', 'clientid': 'client1'} - store1 = multistore_file.get_credential_storage_custom_key( - FILENAME, custom_key) - store1.put(credentials) - - keys = multistore_file.get_all_credential_keys(FILENAME) - self.assertEquals([custom_key], keys) - - # store more credentials - credentials = self._create_test_credentials(client_id='client2') - string_key = 'string_key' - store2 = multistore_file.get_credential_storage_custom_string_key( - FILENAME, string_key) - store2.put(credentials) - - keys = multistore_file.get_all_credential_keys(FILENAME) - self.assertEquals(2, len(keys)) - self.assertTrue(custom_key in keys) - self.assertTrue({'key': string_key} in keys) - - # back to no keys - store1.delete() - store2.delete() - keys = multistore_file.get_all_credential_keys(FILENAME) - self.assertEquals([], keys) - - def _refresh_data_cache_helper(self): - multistore = multistore_file._MultiStore(FILENAME) - json_patch = mock.patch.object(multistore, '_locked_json_read') - - return multistore, json_patch - - def test__refresh_data_cache_bad_json(self): - multistore, json_patch = self._refresh_data_cache_helper() - - with json_patch as json_mock: - json_mock.side_effect = ValueError('') - multistore._refresh_data_cache() - self.assertTrue(json_mock.called) - self.assertEqual(multistore._data, {}) - - def test__refresh_data_cache_bad_version(self): - multistore, json_patch = self._refresh_data_cache_helper() - - with json_patch as json_mock: - json_mock.return_value = {} - multistore._refresh_data_cache() - self.assertTrue(json_mock.called) - self.assertEqual(multistore._data, {}) - - def test__refresh_data_cache_newer_version(self): - multistore, json_patch = self._refresh_data_cache_helper() - - with json_patch as json_mock: - json_mock.return_value = {'file_version': 5} - with self.assertRaises(multistore_file.NewerCredentialStoreError): - multistore._refresh_data_cache() - self.assertTrue(json_mock.called) - - def test__refresh_data_cache_bad_credentials(self): - multistore, json_patch = self._refresh_data_cache_helper() - - with json_patch as json_mock: - json_mock.return_value = { - 'file_version': 1, - 'data': [ - {'lol': 'this is a bad credential object.'} - ]} - multistore._refresh_data_cache() - self.assertTrue(json_mock.called) - self.assertEqual(multistore._data, {}) - - def test__delete_credential_nonexistent(self): - multistore = multistore_file._MultiStore(FILENAME) - - with mock.patch.object(multistore, '_write') as write_mock: - multistore._data = {} - multistore._delete_credential('nonexistent_key') - self.assertTrue(write_mock.called)