Capture referrals

Implement Referral, Referrals and /api/analytics/referrals. Report referrals with the client and
display them for staff.

Along the way implement Endpoint.get_arg(), add Expect compilation and refactor Location.parse() and
Resource.parse(), make ExpectFunc obj non-generic, add formatDate() transform, move date format
constants to transforms and align .micro-lgroup style with .micro-entity-list.

Breaking changes:

- Move SHORT_DATE_FORMAT and SHORT_DATE_TIME_FORMAT to micro.bind.transforms
- Make ExpectFunc obj non-generic

Close #88.

boilerplate/client/package.json

@@ -5,7 +5,7 @@
         "clean": "rm -rf node_modules"
     },
     "dependencies": {
-        "@noyainrain/micro": "^0.44"
+        "@noyainrain/micro": "^0.45"
     },
     "devDependencies": {
         "eslint": "~5.15",

boilerplate/requirements.txt

@@ -1 +1 @@
-noyainrain.micro ~= 0.44.0
+noyainrain.micro ~= 0.45.0

client/bind.js

@@ -435,6 +435,19 @@ micro.bind.transforms = {
         return micro.bind.transforms.format(ctx, n === 1 ? singular : plural, ...args);
     },
 
+    /**
+     * Return a string representation of the given :class:`Date` *date*.
+     *
+     * Alternatively, *date* may be a string parsable by :class:`Date`. *format* is equivalent to
+     * the *options* argument of :meth:`Date.toLocaleString`.
+     */
+    formatDate(ctx, date, format) {
+        if (typeof date === "string") {
+            date = new Date(date);
+        }
+        return date.toLocaleString("en", format);
+    },
+
     /**
      * Project *arr* into the DOM.
      *

client/components/analytics.html

@@ -12,6 +12,36 @@ <h1>User statistics</h1>
             labels="Users, Actual users, Monthly active users"
         ></micro-chart>
     </section>
+
+    <section>
+        <h1>Referrals</h1>
+        <div class="micro-lgroup">
+            <ul class="micro-timeline" data-content="list referrals.items 'referral'">
+                <template>
+                    <li>
+                        <time
+                            data-date-time="referral.time"
+                            data-content="formatDate referral.time SHORT_DATE_TIME_FORMAT"
+                        ></time>
+                        <a data-href="referral.url" data-content="referral.url"></a>
+                    </li>
+                </template>
+            </ul>
+            <ul data-content="switch referralsComplete false">
+                <template>
+                    <li>
+                        <button
+                            is="micro-button" class="micro-analytics-more-referrals link"
+                            data-run="bind fetchCollection referrals 10"
+                            data-shortcut="new Shortcut 'M'"
+                        >
+                            <i class="fa fa-ellipsis-v"></i> More
+                        </button>
+                    </li>
+                </template>
+            </ul>
+        </div>
+    </section>
 </template>
 <style>
     .micro-analytics-user-statistics {

client/components/analytics.js

@@ -42,7 +42,22 @@ micro.components.analytics.AnalyticsPage = class extends micro.Page {
         this.appendChild(
             document.importNode(ui.querySelector("#micro-analytics-page-template").content, true)
         );
+        this._data = new micro.bind.Watchable({
+            referrals: new micro.Collection("/api/analytics/referrals"),
+            referralsComplete: false
+        });
+        micro.bind.bind(this.children, this._data);
+
         this.contentNode = this.querySelector(".micro-analytics-content");
+        this._data.referrals.events.addEventListener("fetch", () => {
+            this._data.referralsComplete = this._data.referrals.complete;
+        });
+    }
+
+    attachedCallback() {
+        super.attachedCallback();
+        const button = this.querySelector(".micro-analytics-more-referrals");
+        this.ready.when(button.trigger().catch(micro.util.catch));
     }
 };
 document.registerElement("micro-analytics-page", micro.components.analytics.AnalyticsPage);
@@ -168,11 +183,12 @@ micro.components.analytics.Chart = class extends HTMLElement {
             });
 
             if (datasets[0].data.length >= 2) {
-                const from = datasets[0].data[0].t.toLocaleDateString(
-                    "en", micro.SHORT_DATE_FORMAT
+                const from = micro.bind.transforms.formatDate(
+                    null, datasets[0].data[0].t, micro.bind.transforms.SHORT_DATE_FORMAT
                 );
-                const to = datasets[0].data[datasets[0].data.length - 1].t.toLocaleDateString(
-                    "en", micro.SHORT_DATE_FORMAT
+                const to = micro.bind.transforms.formatDate(
+                    null, datasets[0].data[datasets[0].data.length - 1].t,
+                    micro.bind.transforms.SHORT_DATE_FORMAT
                 );
                 const summary = datasets.map(
                     dataset => `${dataset.label}: ${dataset.data[0].y} - ${dataset.data[dataset.data.length - 1].y}`

client/index.js

@@ -23,15 +23,6 @@
 micro.util.watchErrors();
 
 micro.LIST_LIMIT = 100;
-micro.SHORT_DATE_FORMAT = {
-    year: "numeric",
-    month: "short",
-    day: "numeric"
-};
-micro.SHORT_DATE_TIME_FORMAT = Object.assign({
-    hour: "2-digit",
-    minute: "2-digit"
-}, micro.SHORT_DATE_FORMAT);
 
 /**
  * Find the first ancestor of *elem* that satisfies *predicate*.
@@ -265,6 +256,14 @@ micro.UI = class extends HTMLBodyElement {
                         this.dispatchEvent(new CustomEvent("user-edit", {detail: {user}}));
                         const settings = await ui.call("GET", "/api/settings");
                         this.dispatchEvent(new CustomEvent("settings-edit", {detail: {settings}}));
+                        if (
+                            document.referrer &&
+                            new URL(document.referrer).origin !== location.origin
+                        ) {
+                            await ui.call(
+                                "POST", "/api/analytics/referrals", {url: document.referrer}
+                            );
+                        }
                     } catch (e) {
                         if (!(e instanceof micro.NetworkError)) {
                             throw e;
@@ -1972,8 +1971,9 @@ micro.ActivityPage = class extends micro.Page {
             let li = document.createElement("li");
             let time = document.createElement("time");
             time.dateTime = event.time;
-            time.textContent =
-                new Date(event.time).toLocaleString("en", micro.SHORT_DATE_TIME_FORMAT);
+            time.textContent = micro.bind.transforms.formatDate(
+                null, event.time, micro.bind.transforms.SHORT_DATE_TIME_FORMAT
+            );
             li.appendChild(time);
             li.appendChild(ui.renderEvent[event.type](event));
             ul.appendChild(li);
@@ -1984,6 +1984,20 @@ micro.ActivityPage = class extends micro.Page {
 };
 
 Object.assign(micro.bind.transforms, {
+    SHORT_DATE_FORMAT: {
+        year: "numeric",
+        month: "short",
+        day: "numeric"
+    },
+
+    SHORT_DATE_TIME_FORMAT: {
+        year: "numeric",
+        month: "short",
+        day: "numeric",
+        hour: "2-digit",
+        minute: "2-digit"
+    },
+
     /**
      * Render *markup text* into a :class:`DocumentFragment`.
      *

client/micro.css

@@ -168,13 +168,13 @@ strong {
 }
 
 .micro-lgroup {
-    list-style: none;
+    margin: var(--micro-size-rm) 0;
 }
 
 .micro-lgroup > ul,
 .micro-lgroup > ol {
-    padding: 0;
     margin: 0;
+    list-style: none;
 }
 
 .micro-icon-title {

client/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@noyainrain/micro",
-    "version": "0.44.0",
+    "version": "0.45.0",
     "description": "Toolkit for social micro web apps.",
     "repository": "noyainrain/micro",
     "license": "LGPL-3.0",

client/templates.html

@@ -487,15 +487,15 @@ <h1>Edit site settings</h1>
 <template class="micro-activity-page-template">
     <h1>Site activity</h1>
     <p><small><i class='fa fa-eye'></i> Only visible to staff members</small></p>
-    <ul>
-        <li class="micro-lgroup">
-            <ul class="micro-timeline"></ul>
-        </li>
-        <li class="micro-activity-show-more">
-            <button is="micro-button" class="link">
-                <i class="fa fa-ellipsis-v"></i> Show more
-            </button>
-        </li>
+    <div class="micro-lgroup">
+        <ul class="micro-timeline"></ul>
+        <ul>
+            <li class="micro-activity-show-more">
+                <button is="micro-button" class="link">
+                    <i class="fa fa-ellipsis-v"></i> More
+                </button>
+            </li>
+        </ul>
     </ul>
 </template>
 

micro/analytics.py

@@ -24,11 +24,15 @@
 from datetime import datetime, timedelta
 import typing
 from typing import Callable, Dict, Iterator, List, Mapping, Optional, cast
+from urllib.parse import urlsplit
 
-from .util import expect_type, parse_isotime
+from . import error
+from .jsonredis import RedisSortedSet
+from .micro import Collection, Object, User
+from .util import expect_type, parse_isotime, randstr
 
 if typing.TYPE_CHECKING:
-    from micro import Application, User
+    from .micro import Application
 
 StatisticFunc = Callable[[datetime], float]
 
@@ -43,6 +47,10 @@ class Analytics:
 
        Statistics over time by topic.
 
+    .. attribute:: referrals
+
+       Referrals from other sites.
+
     .. attribute:: app
 
        Context application.
@@ -57,6 +65,7 @@ def __init__(self, *, definitions: Mapping[str, StatisticFunc] = {},
             **definitions
         } # type: Dict[str, StatisticFunc]
         self.statistics = {topic: Statistic(topic, app=app) for topic in self.definitions}
+        self.referrals = Referrals(app=app)
         self.app = app
 
     def collect_statistics(self, *, _t: datetime = None) -> None:
@@ -88,7 +97,7 @@ def _count_users_active(self, t: datetime) -> float:
         return sum(1 for user in self._actual_users()
                    if t - user.authenticate_time <= timedelta(days=30)) # type: ignore
 
-    def _actual_users(self) -> Iterator['User']:
+    def _actual_users(self) -> Iterator[User]:
         return (user for user in self.app.users[:] # type: ignore
                 if user.authenticate_time - user.create_time >= timedelta(days=1)) # type: ignore
 
@@ -109,14 +118,14 @@ def __init__(self, topic: str, *, app: 'Application') -> None:
         self.app = app
         self._key = 'analytics.statistics.{}'.format(self.topic)
 
-    def get(self, *, user: Optional['User']) -> List['Point']:
+    def get(self, *, user: Optional[User]) -> List['Point']:
         """See :http:get:`/api/analytics/statistics/(topic)`."""
         if not user in self.app.settings.staff: # type: ignore
             raise PermissionError()
         return [Point.parse(json.loads(p.decode())) for p # type: ignore
                 in self.app.r.r.zrange(self._key, 0, -1)] # type: ignore
 
-    def json(self, *, user: 'User' = None) -> Dict[str, object]:
+    def json(self, *, user: User = None) -> Dict[str, object]:
         """See :meth:`micro.JSONifiable.json`."""
         return {'items': [p.json() for p in self.get(user=user)]}
 
@@ -144,3 +153,35 @@ def json(self) -> Dict[str, object]:
 
     def __eq__(self, other: object) -> bool:
         return isinstance(other, Point) and self.t == other.t and self.v == other.v
+
+class Referral(Object):
+    """See :ref:`Referral`."""
+
+    def __init__(self, *, id: str, app: 'Application', url: str, time: str) -> None:
+        super().__init__(id=id, app=app)
+        self.url = url
+        self.time = parse_isotime(time, aware=True)
+
+    def json(self, restricted: bool = False, include: bool = False) -> Dict[str, object]:
+        return {
+            **super().json(restricted=restricted, include=include),
+            'url': self.url,
+            'time': self.time.isoformat()
+        }
+
+class Referrals(Collection[Referral]):
+    """See :ref:`Referrals`."""
+
+    def __init__(self, *, app: 'Application') -> None:
+        super().__init__(RedisSortedSet('analytics.referrals', app.r.r),
+                         check=lambda key: self.app.check_user_is_staff(), app=app) # type: ignore
+
+    def add(self, url: str, *, user: Optional[User]) -> Referral:
+        """See :http:post:`/api/analytics/referrals`."""
+        # pylint: disable=unused-argument; part of API
+        if urlsplit(url).scheme not in {'http', 'https'}:
+            raise error.ValueError('Bad url scheme {}'.format(url))
+        referral = Referral(id=randstr(), app=self.app, url=url, time=self.app.now().isoformat())
+        self.app.r.oset(referral.id, referral)
+        self.app.r.r.zadd(self.ids.key, {referral.id.encode(): -referral.time.timestamp()})
+        return referral

micro/doc/general.inc

@@ -259,6 +259,40 @@ Statistic data point.
 
    Statistic value.
 
+.. _Referral:
+
+Referral
+^^^^^^^^
+
+Referral from another site.
+
+.. include:: micro/object-attributes.inc
+
+.. describe:: url
+
+   HTTP(S) referrer URL.
+
+.. describe:: time
+
+   Date and time of the referral.
+
+.. _Referrals:
+
+Referrals
+^^^^^^^^^
+
+:ref:`Referral` :ref:`Collection`, latest first.
+
+Only staff members can query the referrals.
+
+.. include:: micro/collection-endpoints.inc
+
+.. http:post:: /api/analytics/referrals
+
+   ``{url}``
+
+   Record a :ref:`Referral` from *url* and return it.
+
 Errors
 ------