Upgrade to python 3 (mitmedialab#281)

.python-version

@@ -1 +1 @@
-gobo
+gobo-3.7.3

.travis.yml

@@ -1,7 +1,7 @@
 language:
   - python
 python:
-  - "2.7"
+  - "3.7"
 before_install:
   - nvm install 10.13.0
 stages:

README.md

@@ -15,7 +15,7 @@ Gobo is a [Flask](http://flask.pocoo.org)-based server side, which uses [React](
 
 ### Backend
 
-Gobo uses Python 2.7.x.
+Gobo uses Python 3.7.x.
 
 Create `config.py` in `server/config/` using the provided template to hold the right api keys and database url.
 
@@ -29,7 +29,7 @@ brew install pyenv
 
 Then install the versions of Python we need:
 ```
-pyenv install 2.7.10
+pyenv install 3.7.3
 ```
 
 #### PyEnv-VirtualEnv
@@ -49,7 +49,7 @@ And then create a virtualenv for this project.  The name is important, because t
 refers to it so it loads automatically when you enter the directory (if `eval "$(pyenv virtualenv-init -)"` 
 is in your `.profile`):
 ```
-pyenv virtualenv 2.7.10 gobo
+pyenv virtualenv 3.7.3 gobo-3.7.3
 ```
 
 #### Requirements and Database
@@ -97,14 +97,13 @@ Run the Flask server locally:
 $ ./run.sh
 ```
 
-In order to fetch posts from FB and Twitter you need to run the redis-server and celery worker locally.  Open 2 new shell terminals, and activate the virtualenv. Then run:
+In order to fetch posts from Facebook, Twitter, and Mastodon you need to run the redis-server and celery worker locally.  Open 2 new shell terminals. Then run:
 ```shell
 $ redis-server
 ```
 
 And in the other one:
 ```shell
-$ source venv/bin/activate
 $ celery -A server.scripts.tasks worker
 ```
 

requirements/common.txt

@@ -1,11 +1,11 @@
 alembic==0.9.2
 alembic-autogenerate-enums==0.0.2
-amqp==2.2.1
+amqp==2.4.2
 bcrypt==3.1.3
 beautifulsoup4==4.6.0
-billiard==3.5.0.3
+billiard==3.6.0.0
 blinker==1.4
-celery==4.1.0
+celery==4.3.0
 certifi==2017.4.17
 cffi==1.10.0
 chardet==3.0.4
@@ -21,13 +21,13 @@ Flask-Mail==0.9.1
 Flask-Migrate==2.0.4
 Flask-SQLAlchemy==2.2
 google-api-python-client==1.6.2
-greenlet==0.4.12
+greenlet==0.4.15
 gunicorn==19.7.1
 httplib2==0.10.3
 idna==2.5
 itsdangerous==0.24
 Jinja2==2.10.1
-kombu==4.1.0
+kombu==4.5.0
 Mako==1.0.6
 MarkupSafe==1.0
 Mastodon.py==1.3.1
@@ -43,7 +43,7 @@ python-dotenv==0.10.0
 python-editor==1.0.3
 pytz==2017.2
 raven==6.2.1
-redis==2.10.5
+redis==3.2.1
 requests==2.21.0
 requests-oauthlib==0.8.0
 rsa==3.4.2
@@ -53,5 +53,5 @@ twython==3.5.0
 Unidecode==0.4.21
 uritemplate==3.0.0
 urllib3==1.24.2
-vine==1.1.4
+vine==1.3.0
 Werkzeug==0.14.1

requirements/local.txt

@@ -1,2 +1,2 @@
 -r common.txt
-pylint==1.9.3
+pylint==2.3.1

runtime.txt

@@ -1 +1 @@
-python-2.7.15
+python-3.7.3

server/commands/commands.py

@@ -43,7 +43,7 @@ def create_keyword_rule(creator_id, creator_display_name, title, description, ex
     rule = KeywordRule(creator_id, creator_display_name, title, description, shareable, source, link, split_terms)
     db.session.add(rule)
     db.session.commit()
-    print "Successfully added rule ID: {}".format(rule.id)
+    print("Successfully added rule ID: {}".format(rule.id))
     db.session.close()
 
 
@@ -68,7 +68,7 @@ def create_additive_rule(creator_id, creator_display_name, title, description, l
                         split_terms)
     db.session.add(rule)
     db.session.commit()
-    print "Successfully added rule ID: {}".format(rule.id)
+    print("Successfully added rule ID: {}".format(rule.id))
     db.session.close()
 
 
@@ -78,7 +78,7 @@ def create_additive_rule(creator_id, creator_display_name, title, description, l
 def delete_rule(rule_id):
     db.session.delete(Rule.query.filter_by(id=rule_id).first())
     db.session.commit()
-    print "Deleted rule ID: {}".format(rule_id)
+    print("Deleted rule ID: {}".format(rule_id))
     db.session.close()
 
 
@@ -98,7 +98,7 @@ def share_rule_to_user(user_id, rule_id, enabled):
         db.session.add(setting)
         action = "added"
     db.session.commit()
-    print "Successfully {} setting ID: {}".format(action, setting.id)
+    print("Successfully {} setting ID: {}".format(action, setting.id))
     db.session.close()
 
 
@@ -116,7 +116,7 @@ def share_rule_all_users(rule_id, enabled):
             setting = UserRule(user.id, rule_id, enabled)
             db.session.add(setting)
     db.session.commit()
-    print "Successfully added settings"
+    print("Successfully added settings")
     db.session.close()
 
 
@@ -132,15 +132,15 @@ def share_rule_all_users(rule_id, enabled):
 def add_additive_rule_link(rule_id, level, source, link, name, display_uri):
     """Add link for an additive rule"""
     if source not in ['twitter', 'facebook']:
-        print "Only Facebook and Twitter links are supported currently"
+        print("Only Facebook and Twitter links are supported currently")
         return
 
     rule = AdditiveRule.query.filter_by(id=rule_id).first()
     if rule:
         rule_link = AdditiveRuleLink(rule_id, source, link, level, name, display_uri)
         db.session.add(rule_link)
         db.session.commit()
-        print "Successfully added link"
+        print("Successfully added link")
         db.session.close()
     else:
-        print "No rule ID '{}' found".format(rule_id)
+        print("No rule ID '{}' found".format(rule_id))

server/config/configTemplate.py

@@ -1,4 +1,4 @@
-class BaseConfig(object):
+class BaseConfig:
     # Controls lots of internal things.  Set to True locally; False on production.
     DEBUG = True
 

server/enums.py

@@ -10,6 +10,6 @@ class GenderEnum(enum.Enum):
     def fromString(cls, string):
         if string.lower() == "female":
             return cls.female
-        elif string.lower() == "male":
+        if string.lower() == "male":
             return cls.male
         return cls.unknown

server/factory.py

@@ -1,11 +1,12 @@
+from __future__ import absolute_import
 import os
 from importlib import import_module
 
 from celery import Celery
 from flask import Flask
 from raven.contrib.flask import Sentry
 
-from commands.commands import add_additive_rule_link, create_additive_rule, create_db, create_keyword_rule, \
+from .commands.commands import add_additive_rule_link, create_additive_rule, create_db, create_keyword_rule, \
     delete_rule, drop_db, share_rule_to_user, share_rule_all_users
 
 from .core import db, bcrypt, login_manager, mail, migrate
@@ -63,7 +64,7 @@ def create_celery_app(app=None):
     celery.conf.update({
         'worker_max_tasks_per_child': 10,
         'task_soft_time_limit': thirty_minutes,
-        'worker_max_memory_per_child': 400000  # 400MB
+        'worker_max_memory_per_child': 400000,  # 400MB
     })
 
     TaskBase = celery.Task

server/models.py

@@ -1,5 +1,6 @@
 # pylint: disable=too-many-instance-attributes,no-self-use,too-many-arguments
 
+from builtins import str
 import datetime
 
 from sqlalchemy import event
@@ -102,7 +103,8 @@ def serialize(self):
             d['mastodon_name'] = self.mastodon_auth.username
             d['mastodon_domain'] = self.mastodon_auth.app.domain
         d['avatar'] = self.twitter_data['profile_image_url_https'] if self.twitter_data else self.facebook_picture_url
-        d['hide_tracking'] = True if self.hide_tracking else False
+
+        d['hide_tracking'] = False if self.hide_tracking is None else self.hide_tracking
         return d
 
     def update_last_login(self):

server/scripts/analyze_modules.py

@@ -91,7 +91,7 @@ def analyze_virality(post_id):
     comments = post.get_comments_count()
     shares = post.get_shares_count()
     total_reaction = likes+shares+comments
-    post.virality_count = max(post.virality_count, total_reaction)
+    post.virality_count = max(post.virality_count if post.has_virality() else 0, total_reaction)
     db.session.commit()
 
 

server/scripts/delete_old_posts.py

@@ -1,6 +1,6 @@
 import os
 import logging
-import urlparse
+from urllib.parse import urlparse
 
 import psycopg2
 
@@ -77,7 +77,7 @@ def run_and_return_result(sql):
     conn.commit()
 except Exception as e:
     logger.exception(e)
-    print "There was an error executing posts clean up"
+    print("There was an error executing posts clean up")
 
 cur.close()
 conn.close()

server/scripts/gender_classifier/NameClassifier_light.py

@@ -4,8 +4,8 @@
 import os
 import unicodedata
 
-from genderComputer.genderComputer import GenderComputer
-from utils import *
+from .genderComputer.genderComputer import GenderComputer
+from .utils import *
 
 
 class NameClassifier:

server/scripts/gender_classifier/genderComputer/dictUtils.py

@@ -57,7 +57,7 @@ class MyDict:
     def __init__(self, path=None, encod=None):
         self.data = {}
         if path is not None:
-            fdict = open(path, "r")
+            fdict = open(path, "rb")
             if encod is not None:
                 dict1 = pickle.load(fdict, encoding=encod)
             else:
@@ -123,7 +123,7 @@ def saveAsCSV(self, path):
             f.write('%s\n' % rs)
 
         f.close()
-        print "Wrote dictionary to %s" % path
+        print("Wrote dictionary to %s" % path)
 
     def __getitem__(self, key):
         if key in self.data:

server/scripts/gender_classifier/genderComputer/genderComputer.py

@@ -21,11 +21,11 @@
 import re
 import csv
 from unidecode import unidecode
-from unicodeMagic import UnicodeReader
-from dictUtils import MyDict
-from nameUtils import only_greek_chars, only_cyrillic_chars
-from nameUtils import leet2eng, inverseNameParts, extractFirstName
-from filters import normaliseCountryName
+from .unicodeMagic import UnicodeReader
+from .dictUtils import MyDict
+from .nameUtils import only_greek_chars, only_cyrillic_chars
+from .nameUtils import leet2eng, inverseNameParts, extractFirstName
+from .filters import normaliseCountryName
 
 
 def simplifiedGender(gender):
@@ -69,20 +69,20 @@ def loadGenderList(gender, country, dataPath, hasHeader, script):
             except:
                 '''If second column does not exist, default to count=1'''
                 count = 1
-                if names.has_key(name):
+                if name in names:
                     '''If here then I've seen this name before, modulo case.
                     Only count once (there is no frequency information anyway)'''
                     count = 0
-            if names.has_key(name):
+            if name in names:
                 names[name] += count
             else:
                 names[name] = count
         fd.close()
 
         '''Add versions without diacritics'''
-        for name in names.keys():
+        for name in list(names):
             dname = unidecode(name)
-            if not names.has_key(dname):
+            if not dname in names:
                 names[dname] = names[name]
 
         return names
@@ -96,7 +96,7 @@ def loadGenderList(gender, country, dataPath, hasHeader, script):
 
 
 def getScriptMap(dataPath):
-    fd = open(os.path.join(dataPath, 'script_map.csv'), 'rb')
+    fd = open(os.path.join(dataPath, 'script_map.csv'), 'rt')
     reader = csv.reader(fd)
 
     scripts_dict = dict()
@@ -111,7 +111,7 @@ def getScriptMap(dataPath):
 
         colnum = len(row)
 
-        for i in xrange(2, colnum):
+        for i in range(2, colnum):
             scripts_dict[row[0]].append(row[i].strip())
     fd.close()
     return (scripts_dict)
@@ -349,7 +349,7 @@ def __init__(self, nameListsPath):
         # for country in self.countryStats.keys():
         # 	self.countryStats[country] = self.countryStats[country] / total
         #
-        print 'Finished initialization'
+        print('Finished initialization')
 
     '''Look <firstName> (and potentially its diminutives) up for <country>.
 	Decide gender based on frequency.'''
@@ -647,7 +647,7 @@ def resolveFirstNameOverall(self, firstName, withDiminutives, script):
                 break
 
         # If all countries have twitter user stats, then use stats of twitter users, else use stats of internet users
-        for cnt, gnd in countriesGenders.iteritems():
+        for cnt, gnd in countriesGenders.items():
             userNum = self.countryStats[cnt][1] if hasStat else self.countryStats[cnt][0]
             #
             # try:
@@ -860,12 +860,11 @@ def resolveGenderByCountry(self, name, country, script='Latin'):
     dataPath = os.path.abspath(".")
     gc = GenderComputer(os.path.join(dataPath, 'nameLists'))
 
-    print 'Test suite 1'
+    print('Test suite 1')
     for (name, country) in testSuite1:
-        print [unidecode(name), country], gc.render(name, country, None, "Latin")
-        print [unidecode(name), country], gc.resolveGender(name, "Latin")
+        print([unidecode(name), country], gc.render(name, country, None, "Latin"))
+        print([unidecode(name), country], gc.resolveGender(name, "Latin"))
 
-    print
-    print 'Test suite 2'
+    print('Test suite 2')
     for (name, country) in testSuite2:
         print [unidecode(name), country], gc.resolveGender(name, country, None, "Latin")