Merge pull request #1320 from gittip/remove-orm

port back to postgres.py

.gitignore

@@ -15,3 +15,6 @@ gittip.css
 .vagrant
 node_modules/
 .DS_Store
+docs/_build
+docs/gittip
+docs/gittip.rst

README.md

@@ -278,27 +278,22 @@ Now, you need to setup the database.
 Local Database Setup
 --------------------
 
-For advanced development and testing databse changes, you need to configure
-authentication and set up a gittip database.
-
-You need [Postgres](http://www.postgresql.org/download/). We're working
-on [porting](https://github.com/gittip/www.gittip.com/issues?milestone=28&state=open)
-Gittip from raw SQL to a declarative ORM with SQLAlchemy. After that we may be
-able to remove the hard dependency on Postgres so you can use SQLite in
-development, but for now you need Postgres.
-
-The best version of Postgres to use is 9.2, because that's what is being
-run in production at Heroku. Version 9.1 is the second-best, because Gittip
-uses the [hstore](http://www.postgresql.org/docs/9.2/static/hstore.html)
-extension for unstructured data, and that isn't bundled with earlier
-versions than 9.1. If you're on a Mac, maybe try out Heroku's
+For advanced development and testing database changes, you need a local
+installation of [Postgres](http://www.postgresql.org/download/). The best
+version of Postgres to use is 9.1.9, because that's what we're using in
+production at Heroku. Gittip uses the
+[hstore](http://www.postgresql.org/docs/9.1/static/hstore.html) extension for
+unstructured data, and that isn't bundled with earlier versions than 9.1. If
+you're on a Mac, maybe try out Heroku's
 [Postgres.app](http://www.postgresql.org/download/). If installing using a
 package manager, you may need several packages. On Ubuntu and Debian, the
-required packages are: `postgresql` (base), `libpq5-dev`/`libpq-dev`, (includes headers needed
-to build the `psycopg2` Python library), `postgresql-contrib` (includes
-hstore), `python-dev` (includes Python header files for `psycopg2`).
+required packages are: `postgresql` (base), `libpq5-dev`/`libpq-dev`, (includes
+headers needed to build the `psycopg2` Python library), `postgresql-contrib`
+(includes hstore), `python-dev` (includes Python header files for `psycopg2`).
+
+If you are receiving issues from `psycopg2`, please [ensure their its are
+met](http://initd.org/psycopg/docs/faq.html#problems-compiling-and-deploying-psycopg2).
 
-If you are receiving issues from `psycopg2`, please [ensure their dependencies are met](http://initd.org/psycopg/docs/faq.html#problems-compiling-and-deploying-psycopg2).
 
 ### Authentication
 
@@ -341,13 +336,15 @@ we've run against the production database. You should never change
 commands that have already been run. New DDL will be (manually) run against the
 production database as part of deployment.
 
+
 ### Example data
 
 The gittip database created in the last step is empty. To populate it with
 some fake data, so that more of the site is functional, run this command:
 
     $ make data
 
+
 ### Notes for Mac OS X users
 
 If when running the tests you see errors of the form:

configure-aspen.py

@@ -4,12 +4,9 @@
 
 import gittip
 import gittip.wireup
-import gittip.authentication
-import gittip.orm
-import gittip.csrf
-import gittip.cache_static
-import gittip.models.participant
-from aspen import log_dammit
+import gittip.security.authentication
+import gittip.security.csrf
+import gittip.utils.cache_static
 
 
 version_file = os.path.join(website.www_root, 'version.txt')
@@ -21,7 +18,7 @@
 
 
 gittip.wireup.canonical()
-gittip.wireup.db()
+website.db = gittip.wireup.db()
 gittip.wireup.billing()
 gittip.wireup.username_restrictions(website)
 gittip.wireup.sentry(website)
@@ -43,16 +40,19 @@ def up_minthreads(website):
 
 website.hooks.inbound_early += [ gittip.canonize
                                , gittip.configure_payments
-                               , gittip.authentication.inbound
-                               , gittip.csrf.inbound
+                               , gittip.security.authentication.inbound
+                               , gittip.security.csrf.inbound
                                 ]
 
-website.hooks.inbound_core += [gittip.cache_static.inbound]
+website.hooks.inbound_late += [ gittip.utils.cache_static.inbound
+                              #, gittip.security.authentication.check_role
+                              #, participant.typecast
+                              #, community.typecast
+                               ]
 
-website.hooks.outbound += [ gittip.authentication.outbound
-                          , gittip.csrf.outbound
-                          , gittip.orm.rollback
-                          , gittip.cache_static.outbound
+website.hooks.outbound += [ gittip.security.authentication.outbound
+                          , gittip.security.csrf.outbound
+                          , gittip.utils.cache_static.outbound
                            ]
 
 
@@ -97,83 +97,9 @@ def add_stuff(request):
 
 UPDATE_HOMEPAGE_EVERY = int(os.environ['UPDATE_HOMEPAGE_EVERY'])
 def update_homepage_queries():
+    from gittip import utils
     while 1:
-        with gittip.db.get_transaction() as txn:
-            log_dammit("updating homepage queries")
-            start = time.time()
-            txn.execute("""
-
-            DROP TABLE IF EXISTS _homepage_new_participants;
-            CREATE TABLE _homepage_new_participants AS
-                  SELECT username, claimed_time FROM (
-                      SELECT DISTINCT ON (p.username)
-                             p.username
-                           , claimed_time
-                        FROM participants p
-                        JOIN elsewhere e
-                          ON p.username = participant
-                       WHERE claimed_time IS NOT null
-                         AND is_suspicious IS NOT true
-                         ) AS foo
-                ORDER BY claimed_time DESC;
-
-            DROP TABLE IF EXISTS _homepage_top_givers;
-            CREATE TABLE _homepage_top_givers AS
-                SELECT tipper AS username, anonymous, sum(amount) AS amount
-                  FROM (    SELECT DISTINCT ON (tipper, tippee)
-                                   amount
-                                 , tipper
-                              FROM tips
-                              JOIN participants p ON p.username = tipper
-                              JOIN participants p2 ON p2.username = tippee
-                              JOIN elsewhere ON elsewhere.participant = tippee
-                             WHERE p.last_bill_result = ''
-                               AND p.is_suspicious IS NOT true
-                               AND p2.claimed_time IS NOT NULL
-                               AND elsewhere.is_locked = false
-                          ORDER BY tipper, tippee, mtime DESC
-                          ) AS foo
-                  JOIN participants p ON p.username = tipper
-                 WHERE is_suspicious IS NOT true
-              GROUP BY tipper, anonymous
-              ORDER BY amount DESC;
-
-            DROP TABLE IF EXISTS _homepage_top_receivers;
-            CREATE TABLE _homepage_top_receivers AS
-                SELECT tippee AS username, claimed_time, sum(amount) AS amount
-                  FROM (    SELECT DISTINCT ON (tipper, tippee)
-                                   amount
-                                 , tippee
-                              FROM tips
-                              JOIN participants p ON p.username = tipper
-                              JOIN elsewhere ON elsewhere.participant = tippee
-                             WHERE last_bill_result = ''
-                               AND elsewhere.is_locked = false
-                               AND is_suspicious IS NOT true
-                               AND claimed_time IS NOT null
-                          ORDER BY tipper, tippee, mtime DESC
-                          ) AS foo
-                  JOIN participants p ON p.username = tippee
-                 WHERE is_suspicious IS NOT true
-              GROUP BY tippee, claimed_time
-              ORDER BY amount DESC;
-
-            DROP TABLE IF EXISTS homepage_new_participants;
-            ALTER TABLE _homepage_new_participants
-              RENAME TO homepage_new_participants;
-
-            DROP TABLE IF EXISTS homepage_top_givers;
-            ALTER TABLE _homepage_top_givers
-              RENAME TO homepage_top_givers;
-
-            DROP TABLE IF EXISTS homepage_top_receivers;
-            ALTER TABLE _homepage_top_receivers
-              RENAME TO homepage_top_receivers;
-
-            """)
-            end = time.time()
-            elapsed = end - start
-            log_dammit("updated homepage queries in %.2f seconds" % elapsed)
+        utils.update_homepage_queries_once(website.db)
         time.sleep(UPDATE_HOMEPAGE_EVERY)
 
 homepage_updater = threading.Thread(target=update_homepage_queries)

default_local.env

@@ -1,3 +1,4 @@
+PYTHONDONTWRITEBYTECODE=true
 CANONICAL_HOST=
 CANONICAL_SCHEME=http
 MIN_THREADS=10

default_tests.env

@@ -1,3 +1,4 @@
+PYTHONDONTWRITEBYTECODE=true
 CANONICAL_HOST=
 CANONICAL_SCHEME=http
 MIN_THREADS=10

docs/Makefile

@@ -0,0 +1,49 @@
+# Makefile for Sphinx documentation
+# Trimmed up from the 
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(SPHINXOPTS) .
+
+.PHONY: help clean html linkcheck doctest
+
+default: html
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	rm -rf gittip* $(BUILDDIR)/*
+
+rst:
+	AUTOLIB_LIBRARY_ROOT=../gittip ./autolib.py
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

docs/autolib.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""Generate *.rst files to mirror *.py files in a Python library.
+
+This script is conceptually similar to the sphinx-apidoc script bundled with
+Sphinx:
+
+    http://sphinx-doc.org/man/sphinx-apidoc.html
+
+We produce different *.rst output, however.
+
+"""
+from __future__ import print_function, unicode_literals
+import os
+
+
+w = lambda f, s, *a, **kw: print(s.format(*a, **kw), file=f)
+
+
+def rst_for_module(toc_path):
+    """Given a toc_path, write rst and return a file object.
+    """
+
+    f = open(toc_path + '.rst', 'w+')
+
+    heading = ":mod:`{}`".format(os.path.basename(toc_path))
+    dotted = toc_path.replace('/', '.')
+
+    w(f, heading)
+    w(f, "=" * len(heading))
+    w(f, ".. automodule:: {}", dotted)
+
+    return f
+
+
+def rst_for_package(root, dirs, files):
+    """Given ../mylib/path/to/package and lists of dir/file names, write rst.
+    """
+
+    doc_path = root[3:]
+    if not os.path.isdir(doc_path):
+        os.mkdir(doc_path)
+
+
+    # Start a rst doc for this package.
+    # =================================
+
+    f = rst_for_module(doc_path)
+
+
+    # Add a table of contents.
+    # ========================
+
+    w(f, ".. toctree::")
+
+    def toc(doc_path, name):
+        parent = os.path.dirname(doc_path)
+        toc_path = os.path.join(doc_path[len(parent):].lstrip('/'), name)
+        if toc_path.endswith('.py'):
+            toc_path = toc_path[:-len('.py')]
+        w(f, "    {}", toc_path)
+        return os.path.join(parent, toc_path)
+
+    for name in sorted(dirs + files):
+        if name in dirs:
+            toc(doc_path, name)
+        else:
+            if not name.endswith('.py'): continue
+            if name == '__init__.py': continue
+
+            toc_path = toc(doc_path, name)
+
+
+            # Write a rst file for each module.
+            # =================================
+
+            rst_for_module(toc_path)
+
+
+def main():
+    library_root = os.environ['AUTOLIB_LIBRARY_ROOT']
+    for root, dirs, files in os.walk(library_root):
+        rst_for_package(root, dirs, files)
+
+
+if __name__ == '__main__':
+    main()