No longer allow ipaddress objects in pure Python implementation

.travis-yapf.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+diff=$(yapf -rd maxminddb tests)
+
+if [[ $? != 0 ]]; then
+    echo "yapf failed to run."
+    echo "$diff"
+    exit $?
+elif [[ $diff ]]; then
+    echo "$diff"
+    exit 1
+else
+    exit 0
+fi

.travis.yml

@@ -17,13 +17,15 @@ before_install:
   - sudo make install
   - sudo ldconfig
   - cd ..
-  - pip install pylint coveralls
+  - pip install coveralls
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pip install pylint yapf; fi
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi
 
 script:
   - if [[ $TRAVIS_PYTHON_VERSION != 'pypy' ]]; then export MM_FORCE_EXT_TESTS=1; fi
   - CFLAGS="-Werror -Wall -Wextra" python setup.py test
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pylint --rcfile .pylintrc maxminddb/*.py; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then pylint --rcfile .pylintrc maxminddb/*.py; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then ./.travis-yapf.sh; fi
 
 after_success:
   - coveralls

HISTORY.rst

@@ -3,6 +3,20 @@
 History
 -------
 
+1.4.0
+++++++++++++++++++
+
+* IMPORTANT: Previously, the pure Python reader would allow
+  `ipaddress.IPv4Address` and `ipaddress.IPv6Address` objects when calling
+  `.get()`. This would fail with the C extension. The fact that these objects
+  worked at all was an implementation details and has varied with different
+  releases. This release makes the pure Python implementation consistent
+  with the extension. A `TypeError` will now be thrown if you attempt to
+  use these types with either the pure Python implementation or the
+  extension. The IP address passed to `.get()` should be a string type.
+* Fix issue where incorrect size was used when unpacking some types with the
+  pure Python reader. Reported by Lee Symes. GitHub #30.
+
 1.3.0 (2017-03-13)
 ++++++++++++++++++
 

docs/conf.py

@@ -18,7 +18,6 @@
 sys.path.insert(0, os.path.abspath('..'))
 import maxminddb
 
-
 __version__ = maxminddb.__version__
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -33,8 +32,10 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
-              'sphinx.ext.intersphinx', 'sphinx.ext.coverage']
+extensions = [
+    'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
+    'sphinx.ext.coverage'
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -95,7 +96,6 @@
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []
 
-
 # -- Options for HTML output ---------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -175,7 +175,6 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'maxminddbdoc'
 
-
 # -- Options for LaTeX output --------------------------------------------
 
 latex_elements = {
@@ -192,8 +191,8 @@
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ('index', 'maxminddb.tex', 'maxminddb Documentation',
-     'Gregory Oschwald', 'manual'),
+    ('index', 'maxminddb.tex', 'maxminddb Documentation', 'Gregory Oschwald',
+     'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -216,29 +215,24 @@
 # If false, no module index is generated.
 #latex_domain_indices = True
 
-
 # -- Options for manual page output --------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'maxminddb', 'maxminddb Documentation',
-     ['Gregory Oschwald'], 1)
-]
+man_pages = [('index', 'maxminddb', 'maxminddb Documentation',
+              ['Gregory Oschwald'], 1)]
 
 # If true, show URL addresses after external links.
 #man_show_urls = False
 
-
 # -- Options for Texinfo output ------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    ('index', 'maxminddb', 'maxminddb Documentation',
-     'Gregory Oschwald', 'maxminddb', 'MaxMind DB Reader',
-     'Miscellaneous'),
+    ('index', 'maxminddb', 'maxminddb Documentation', 'Gregory Oschwald',
+     'maxminddb', 'MaxMind DB Reader', 'Miscellaneous'),
 ]
 
 # Documents to append as an appendix to all manuals.
@@ -250,6 +244,5 @@
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
 
-
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'http://docs.python.org/': None}

examples/benchmark.py

@@ -11,12 +11,11 @@
 import timeit
 
 parser = argparse.ArgumentParser(description='Benchmark maxminddb.')
-parser.add_argument('--count', default=250000, type=int,
-                    help='number of lookups')
-parser.add_argument('--mode', default=0, type=int,
-                    help='reader mode to use')
-parser.add_argument('--file', default='GeoIP2-City.mmdb',
-                    help='path to mmdb file')
+parser.add_argument(
+    '--count', default=250000, type=int, help='number of lookups')
+parser.add_argument('--mode', default=0, type=int, help='reader mode to use')
+parser.add_argument(
+    '--file', default='GeoIP2-City.mmdb', help='path to mmdb file')
 
 args = parser.parse_args()
 
@@ -28,8 +27,9 @@ def lookup_ip_address():
     record = reader.get(str(ip))
 
 
-elapsed = timeit.timeit('lookup_ip_address()',
-                        setup='from __main__ import lookup_ip_address',
-                        number=args.count)
+elapsed = timeit.timeit(
+    'lookup_ip_address()',
+    setup='from __main__ import lookup_ip_address',
+    number=args.count)
 
 print(args.count / elapsed, 'lookups per second')

maxminddb/__init__.py

@@ -27,11 +27,13 @@ def open_database(database, mode=MODE_AUTO):
             * MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that
                           order. Default mode.
     """
-    has_extension = maxminddb.extension and hasattr(maxminddb.extension, 'Reader')
+    has_extension = maxminddb.extension and hasattr(maxminddb.extension,
+                                                    'Reader')
     if (mode == MODE_AUTO and has_extension) or mode == MODE_MMAP_EXT:
         if not has_extension:
             raise ValueError(
-                "MODE_MMAP_EXT requires the maxminddb.extension module to be available")
+                "MODE_MMAP_EXT requires the maxminddb.extension module to be available"
+            )
         return maxminddb.extension.Reader(database)
     elif mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY):
         return maxminddb.reader.Reader(database, mode)

maxminddb/compat.py

@@ -21,6 +21,10 @@ def int_from_bytes(b):
         return 0
 
     byte_from_int = chr
+
+    string_type = basestring
+
+    string_type_name = 'string'
 else:
 
     def compat_ip_address(address):
@@ -33,3 +37,7 @@ def compat_ip_address(address):
     int_from_bytes = lambda x: int.from_bytes(x, 'big')
 
     byte_from_int = lambda x: bytes([x])
+
+    string_type = str
+
+    string_type_name = string_type.__name__

maxminddb/decoder.py

@@ -50,7 +50,7 @@ def _decode_packed_type(type_code, type_size, pad=False):
         def unpack_type(self, size, offset):
             if not pad:
                 self._verify_size(size, type_size)
-            new_offset = offset + type_size
+            new_offset = offset + size
             packed_bytes = self._buffer[offset:new_offset]
             if pad:
                 packed_bytes = packed_bytes.rjust(type_size, b'\x00')
@@ -105,8 +105,7 @@ def _decode_utf8_string(self, size, offset):
         5: _decode_uint,  # uint16
         6: _decode_uint,  # uint32
         7: _decode_map,
-        8: _decode_packed_type(
-            b'!i', 4, pad=True),  # int32
+        8: _decode_packed_type(b'!i', 4, pad=True),  # int32
         9: _decode_uint,  # uint64
         10: _decode_uint,  # uint128
         11: _decode_array,

maxminddb/reader.py

@@ -15,7 +15,8 @@
 
 import struct
 
-from maxminddb.compat import byte_from_int, compat_ip_address
+from maxminddb.compat import (byte_from_int, compat_ip_address, string_type,
+                              string_type_name)
 from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY
 from maxminddb.decoder import Decoder
 from maxminddb.errors import InvalidDatabaseError
@@ -93,6 +94,9 @@ def get(self, ip_address):
         Arguments:
         ip_address -- an IP address in the standard string notation
         """
+        if not isinstance(ip_address, string_type):
+            raise TypeError('argument 1 must be %s, not %s' %
+                            (string_type_name, type(ip_address).__name__))
 
         address = compat_ip_address(ip_address)
 
@@ -155,14 +159,13 @@ def _read_node(self, node_number, index):
             else:
                 middle = (0xF0 & middle) >> 4
             offset = base_offset + index * 4
-            node_bytes = byte_from_int(middle) + self._buffer[offset:offset +
-                                                              3]
+            node_bytes = byte_from_int(middle) + self._buffer[offset:offset + 3]
         elif record_size == 32:
             offset = base_offset + index * 4
             node_bytes = self._buffer[offset:offset + 4]
         else:
-            raise InvalidDatabaseError('Unknown record size: {0}'.format(
-                record_size))
+            raise InvalidDatabaseError(
+                'Unknown record size: {0}'.format(record_size))
         return struct.unpack(b'!I', node_bytes)[0]
 
     def _resolve_data_pointer(self, pointer):

setup.py

@@ -16,8 +16,8 @@
 JYTHON = sys.platform.startswith('java')
 requirements = []
 
-if sys.version_info[0] == 2 or (sys.version_info[0] == 3 and
-                                sys.version_info[1] < 3):
+if sys.version_info[0] == 2 or (sys.version_info[0] == 3
+                                and sys.version_info[1] < 3):
     requirements.append('ipaddress')
 
 compile_args = ['-Wall', '-Wextra']
@@ -30,7 +30,8 @@
         'maxminddb.extension',
         libraries=['maxminddb'],
         sources=['maxminddb/extension/maxminddb.c'],
-        extra_compile_args=compile_args, )
+        extra_compile_args=compile_args,
+    )
 ]
 
 # Cargo cult code for installing extension with pure Python fallback.
@@ -95,8 +96,8 @@ def status_msgs(*msgs):
 def find_packages(location):
     packages = []
     for pkg in ['maxminddb']:
-        for _dir, subdirectories, files in (
-                os.walk(os.path.join(location, pkg))):
+        for _dir, subdirectories, files in (os.walk(
+                os.path.join(location, pkg))):
             if '__init__.py' in files:
                 tokens = _dir.split(os.sep)[len(location.split(os.sep)):]
                 packages.append(".".join(tokens))
@@ -108,7 +109,8 @@ def run_setup(with_cext):
     if with_cext:
         if Feature:
             kwargs['features'] = {
-                'extension': Feature(
+                'extension':
+                Feature(
                     "optional C implementation",
                     standard=True,
                     ext_modules=ext_module)

tests/decoder_test.py

@@ -91,15 +91,15 @@ def test_map(self):
             },
             (b'\xe1\x44\x6e\x61\x6d\x65\xe2\x42\x65\x6e'
              b'\x43\x46\x6f\x6f\x42\x7a\x68\x43\xe4\xba\xba'): {
-                 'name': {
-                     'en': 'Foo',
-                     'zh': '人'
-                 }
-             },
+                'name': {
+                    'en': 'Foo',
+                    'zh': '人'
+                }
+            },
             (b'\xe1\x49\x6c\x61\x6e\x67\x75\x61\x67\x65\x73'
              b'\x02\x04\x42\x65\x6e\x42\x7a\x68'): {
-                 'languages': ['en', 'zh']
-             },
+                'languages': ['en', 'zh']
+            },
         }
         self.validate_type_decoding('maps', maps)
 
@@ -120,37 +120,46 @@ def test_pointer(self):
         self.validate_type_decoding('pointers', pointers)
 
     strings = {
-        b"\x40": '',
-        b"\x41\x31": '1',
-        b"\x43\xE4\xBA\xBA": '人',
+        b"\x40":
+        '',
+        b"\x41\x31":
+        '1',
+        b"\x43\xE4\xBA\xBA":
+        '人',
         (b"\x5b\x31\x32\x33\x34"
          b"\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35"
          b"\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37"):
         '123456789012345678901234567',
         (b"\x5c\x31\x32\x33\x34"
          b"\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35"
          b"\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36"
-         b"\x37\x38"): '1234567890123456789012345678',
+         b"\x37\x38"):
+        '1234567890123456789012345678',
         (b"\x5d\x00\x31\x32\x33"
          b"\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34"
          b"\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35"
-         b"\x36\x37\x38\x39"): '12345678901234567890123456789',
+         b"\x36\x37\x38\x39"):
+        '12345678901234567890123456789',
         (b"\x5d\x01\x31\x32\x33"
          b"\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34"
          b"\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35"
-         b"\x36\x37\x38\x39\x30"): '123456789012345678901234567890',
-        b'\x5e\x00\xd7' + 500 * b'\x78': 'x' * 500,
-        b'\x5e\x06\xb3' + 2000 * b'\x78': 'x' * 2000,
-        b'\x5f\x00\x10\x53' + 70000 * b'\x78': 'x' * 70000,
+         b"\x36\x37\x38\x39\x30"):
+        '123456789012345678901234567890',
+        b'\x5e\x00\xd7' + 500 * b'\x78':
+        'x' * 500,
+        b'\x5e\x06\xb3' + 2000 * b'\x78':
+        'x' * 2000,
+        b'\x5f\x00\x10\x53' + 70000 * b'\x78':
+        'x' * 70000,
     }
 
     def test_string(self):
         self.validate_type_decoding('string', self.strings)
 
     def test_byte(self):
         # Python 2.6 doesn't support dictionary comprehension
-        b = dict((byte_from_int(0xc0 ^ int_from_byte(k[0])) + k[1:],
-                  v.encode('utf-8')) for k, v in self.strings.items())
+        b = dict((byte_from_int(0xc0 ^ int_from_byte(k[0])) + k[1:], v.encode(
+            'utf-8')) for k, v in self.strings.items())
         self.validate_type_decoding('byte', b)
 
     def test_uint16(self):
@@ -207,7 +216,8 @@ def check_decoding(self, type, input, expected, name=None):
         decoder = Decoder(db, pointer_test=True)
         (
             actual,
-            _, ) = decoder.decode(0)
+            _,
+        ) = decoder.decode(0)
 
         if type in ('float', 'double'):
             self.assertAlmostEqual(expected, actual, places=3, msg=type)