Skip to content

Commit

Permalink
Update add_ons_to_gss command to support Python 2.
Browse files Browse the repository at this point in the history
The command would error when running with Python 2.7, failing to
recognise the Welsh characters `ô` and `â` with "UnicodeEncodeError:
'ascii' codec can't encode character u'\xf4' in position 16: ordinal not
in range(128)". This is due to `csv.reader` changing between Python 2
and 3 - in 2 it expected a file-like object on which next() would return
raw bytes, and in 3 one which would return unicode strings. We opt to
explicitly check which version of Python is running and run the old code
for < Python 3.

We also stop assigning `name` for the 'code-change.csv' as it is not
used anywhere else in the script and the encoding probably won't be
handled correctly with Python 2. We stop assigning `type` and `name` for
'missing-codes.csv' for the same reasons.
  • Loading branch information
davidbasalla authored and dracos committed Sep 30, 2016
1 parent f92e797 commit d659d40
Showing 1 changed file with 17 additions and 16 deletions.
33 changes: 17 additions & 16 deletions mapit_gb/management/commands/mapit_UK_add_ons_to_gss.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,23 @@
import codecs
import csv
import os.path
import sys
from django.core.management.base import BaseCommand
from mapit.models import Area, CodeType
from psycopg2 import IntegrityError

python_version = sys.version_info[0]


def open_csv(filename):
if python_version < 3:
o = open(filename)
else:
o = codecs.open(filename, 'r', 'latin-1')
mapping = csv.reader(o)
next(mapping)
return mapping


def process(new_code, old_code):
try:
Expand All @@ -30,22 +43,10 @@ class Command(BaseCommand):
help = 'Inserts the old ONS codes into mapit'

def handle(self, **options):
code_changes = codecs.open(os.path.join(
os.path.dirname(__file__),
'../../data/BL-2010-10-code-change.csv'
), 'r', 'latin-1')
mapping = csv.reader(code_changes)
next(mapping)
for row in mapping:
new_code, name, old_code = row[0], row[1], row[3]
for row in open_csv(os.path.join(os.path.dirname(__file__), '../../data/BL-2010-10-code-change.csv')):
new_code, old_code = row[0], row[3]
process(new_code, old_code)

missing_codes = codecs.open(os.path.join(
os.path.dirname(__file__),
'../../data/BL-2010-10-missing-codes.csv'
), 'r', 'latin-1')
mapping = csv.reader(missing_codes)
next(mapping)
for row in mapping:
type, new_code, old_code, name = row
for row in open_csv(os.path.join(os.path.dirname(__file__), '../../data/BL-2010-10-missing-codes.csv')):
new_code, old_code = row[1], row[2]
process(new_code, old_code)

0 comments on commit d659d40

Please sign in to comment.