Problem importing celery to enable tasks

.github/workflows/docs.yml

@@ -17,4 +17,4 @@ jobs:
       - name: Build docs
         run: |
           cd docs
-          sphinx-build -b html -n -d _build/doctrees   . _build/html
+          sphinx-build -b html -n -d _build/doctrees . _build/html

.github/workflows/lint.yml

@@ -0,0 +1,38 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  flake8:
+    name: flake8
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Install flake8
+        run: pip install --upgrade flake8
+      - name: Run flake8
+        uses: liskin/gh-problem-matcher-wrap@v2
+        with:
+          linters: flake8
+          run: flake8
+
+  isort:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - run: python -m pip install isort
+      - name: isort
+        uses: liskin/gh-problem-matcher-wrap@v2
+        with:
+          linters: isort
+          run: isort --check --diff explorer

.github/workflows/test.yml

@@ -8,11 +8,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ 3.6, 3.7, 3.8]
+        python-version: [ 3.8]
         requirements-file: [
-            django-2.2.txt,
-            django-3.0.txt,
-            django-3.1.txt,
             django-3.2.txt
         ]
         os: [

.gitignore

@@ -14,6 +14,7 @@
 tmp
 venv/
 .venv/
+.tox/
 
 # Sphinx documentation
 docs/_build/

.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+ci:
+    autofix_commit_msg: |
+        ci: auto fixes from pre-commit hooks
+
+        for more information, see https://pre-commit.ci
+    autofix_prs: false
+    autoupdate_commit_msg: 'ci: pre-commit autoupdate'
+    autoupdate_schedule: monthly
+
+repos:
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.37.3
+    hooks:
+      - id: pyupgrade
+        args: ["--py38-plus"]
+
+  - repo: https://github.com/adamchainz/django-upgrade
+    rev: '1.7.0'
+    hooks:
+      - id: django-upgrade
+        args: [--target-version, "3.2"]
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.2
+    hooks:
+      - id: flake8
+
+  - repo: https://github.com/asottile/yesqa
+    rev: v1.3.0
+    hooks:
+      - id: yesqa
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-merge-conflict
+      - id: mixed-line-ending
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort

docs/conf.py

@@ -16,6 +16,7 @@
 
 import explorer
 
+
 sys.path.insert(0, os.path.abspath('../'))
 sys.path.insert(0, os.path.abspath('.'))
 

docs/features.rst

@@ -45,7 +45,7 @@ Snapshots
        'schedule': crontab(hour=1, minute=0)
    }
 
-- Requires celery, obviously. Also uses djcelery and tinys3. All
+- Requires celery, obviously. Also uses boto3. All
   of these deps are optional and can be installed with
   ``pip install -r optional-requirements.txt``
 - The checkbox for opting a query into a snapshot is ALL THE WAY

docs/settings.rst

@@ -228,6 +228,17 @@ S3 Bucket for snapshot upload
    EXPLORER_S3_BUCKET = None
 
 
+S3 link expiration
+******************
+
+S3 link expiration time. Defaults to 3600 seconds (1hr) if not specified.
+Links are generated as presigned urls for security
+
+.. code-block:: python
+
+   EXPLORER_S3_S3_LINK_EXPIRATION = 3600
+
+
 From email
 **********
 

explorer/__init__.py

@@ -22,5 +22,3 @@ def get_version(short=False):
 
 
 __version__ = get_version()
-
-default_app_config = 'explorer.apps.ExplorerAppConfig'

explorer/actions.py

@@ -1,8 +1,8 @@
+import tempfile
 from collections import defaultdict
 from datetime import date
-import tempfile
-from zipfile import ZipFile
 from wsgiref.util import FileWrapper
+from zipfile import ZipFile
 
 from django.http import HttpResponse
 

explorer/app_settings.py

@@ -1,6 +1,8 @@
-from django.conf import settings
 from pydoc import locate
 
+from django.conf import settings
+
+
 # The 'correct' configuration for Explorer looks like:
 
 # EXPLORER_CONNECTIONS = {
@@ -124,6 +126,7 @@
 S3_ACCESS_KEY = getattr(settings, "EXPLORER_S3_ACCESS_KEY", None)
 S3_SECRET_KEY = getattr(settings, "EXPLORER_S3_SECRET_KEY", None)
 S3_BUCKET = getattr(settings, "EXPLORER_S3_BUCKET", None)
+S3_LINK_EXPIRATION: int = getattr(settings, "EXPLORER_S3_S3_LINK_EXPIRATION", 3600)
 FROM_EMAIL = getattr(
     settings, 'EXPLORER_FROM_EMAIL', '[email protected]'
 )

explorer/charts.py

@@ -1,12 +1,13 @@
-from typing import Optional, Iterable
 from io import BytesIO
+from typing import Iterable, Optional
 
 from django.core.exceptions import ImproperlyConfigured
 
+
 try:
     import matplotlib.pyplot as plt
-    from matplotlib.figure import Figure
     import seaborn as sns
+    from matplotlib.figure import Figure
 except ImportError:
     from . import app_settings
 

explorer/exporters.py

@@ -1,10 +1,9 @@
 import codecs
 import csv
 import json
-import string
 import uuid
 from datetime import datetime
-from io import StringIO, BytesIO
+from io import BytesIO, StringIO
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.utils.module_loading import import_string

explorer/forms.py

@@ -1,12 +1,8 @@
-from django.forms import (
-    BooleanField, CharField, ModelForm, ValidationError
-)
+from django.forms import BooleanField, CharField, ModelForm, ValidationError
 from django.forms.widgets import CheckboxInput, Select
 
-from explorer.app_settings import (
-    EXPLORER_DEFAULT_CONNECTION, EXPLORER_CONNECTIONS
-)
-from explorer.models import Query, MSG_FAILED_BLACKLIST
+from explorer.app_settings import EXPLORER_CONNECTIONS, EXPLORER_DEFAULT_CONNECTION
+from explorer.models import MSG_FAILED_BLACKLIST, Query
 
 
 class SqlField(CharField):

explorer/migrations/0001_initial.py

@@ -1,6 +1,6 @@
-from django.db import models, migrations
 import django.db.models.deletion
 from django.conf import settings
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

explorer/migrations/0002_auto_20150501_1515.py

@@ -1,4 +1,4 @@
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

explorer/migrations/0003_query_snapshot.py

@@ -1,4 +1,4 @@
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

explorer/migrations/0006_query_connection.py

@@ -1,4 +1,4 @@
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

explorer/migrations/0007_querylog_connection.py

@@ -1,4 +1,4 @@
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

explorer/models.py

@@ -2,21 +2,17 @@
 from time import time
 
 from django.conf import settings
-from django.db import models, DatabaseError, transaction
+from django.db import DatabaseError, models, transaction
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
 from explorer import app_settings
 from explorer.utils import (
-    passes_blacklist,
-    swap_params,
-    extract_params,
-    shared_dict_update,
-    get_s3_bucket,
-    get_params_for_url,
-    get_valid_connection
+    extract_params, get_params_for_url, get_s3_bucket, get_valid_connection, passes_blacklist, s3_url,
+    shared_dict_update, swap_params,
 )
 
+
 MSG_FAILED_BLACKLIST = "Query failed the SQL blacklist: %s"
 
 
@@ -137,13 +133,13 @@ def shared(self):
     def snapshots(self):
         if app_settings.ENABLE_TASKS:
             b = get_s3_bucket()
-            keys = b.list(prefix=f'query-{self.id}/snap-')
-            keys_s = sorted(keys, key=lambda k: k.last_modified)
+            objects = b.objects.filter(Prefix=f'query-{self.id}/snap-')
+            objects_s = sorted(objects, key=lambda k: k.last_modified)
             return [
                 SnapShot(
-                    k.generate_url(expires_in=0, query_auth=False),
-                    k.last_modified
-                ) for k in keys_s
+                    s3_url(b, o.key),
+                    o.last_modified
+                ) for o in objects_s
             ]
 
 

explorer/permissions.py

@@ -6,7 +6,7 @@ def view_permission(request, **kwargs):
     return app_settings.EXPLORER_PERMISSION_VIEW(request)\
         or user_can_see_query(request, **kwargs)\
         or (app_settings.EXPLORER_TOKEN_AUTH_ENABLED()
-            and (request.META.get('HTTP_X_API_TOKEN') ==
+            and (request.headers.get('X-Api-Token') ==
                  app_settings.EXPLORER_TOKEN
                  or request.GET.get('token') == app_settings.EXPLORER_TOKEN))
 

explorer/schema.py

@@ -1,12 +1,8 @@
 from django.core.cache import cache
 
 from explorer.app_settings import (
-    EXPLORER_SCHEMA_INCLUDE_TABLE_PREFIXES,
-    EXPLORER_SCHEMA_EXCLUDE_TABLE_PREFIXES,
-    EXPLORER_SCHEMA_INCLUDE_VIEWS,
-    ENABLE_TASKS,
-    EXPLORER_ASYNC_SCHEMA,
-    EXPLORER_CONNECTIONS
+    ENABLE_TASKS, EXPLORER_ASYNC_SCHEMA, EXPLORER_CONNECTIONS, EXPLORER_SCHEMA_EXCLUDE_TABLE_PREFIXES,
+    EXPLORER_SCHEMA_INCLUDE_TABLE_PREFIXES, EXPLORER_SCHEMA_INCLUDE_VIEWS,
 )
 from explorer.tasks import build_schema_cache_async
 from explorer.utils import get_valid_connection