System Status page and API (#1812).

cypress/integration/system_status.js

@@ -0,0 +1,9 @@
+describe('System Status', () => {
+    it('System Status loaded', () => {
+        cy.get('[data-attr=menu-item-settings]').click()
+        cy.get('[data-attr=menu-item-system-status]').click()
+        cy.get('h1').should('contain', 'System Status')
+        cy.get('table').should('contain', 'Postgres Event table')
+        cy.get('table').should('contain', 'Redis current queue depth')
+    })
+})

frontend/src/layout/Sidebar.js

@@ -20,6 +20,7 @@ import {
     TeamOutlined,
     LockOutlined,
     WalletOutlined,
+    DatabaseOutlined,
 } from '@ant-design/icons'
 import { useActions, useValues } from 'kea'
 import { Link } from 'lib/components/Link'
@@ -45,7 +46,7 @@ function Logo() {
     )
 }
 
-// to show the right page in the sidebar
+// to show the right page i n the sidebar
 const sceneOverride = {
     action: 'actions',
     person: 'people',
@@ -63,6 +64,7 @@ const submenuOverride = {
     annotations: 'settings',
     billing: 'settings',
     licenses: 'settings',
+    systemStatus: 'settings',
 }
 
 export function Sidebar({ user, sidebarCollapsed, setSidebarCollapsed }) {
@@ -259,6 +261,14 @@ export function Sidebar({ user, sidebarCollapsed, setSidebarCollapsed }) {
                             </Menu.Item>
                         )}
 
+                        {(!user.is_multi_tenancy || (user.is_multi_tenancy && user.is_staff)) && (
+                            <Menu.Item key="systemStatus" style={itemStyle} data-attr="menu-item-system-status">
+                                <DatabaseOutlined />
+                                <span className="sidebar-label">System Status</span>
+                                <Link to={'/system_status'} onClick={collapseSidebar} />
+                            </Menu.Item>
+                        )}
+
                         {!user.is_multi_tenancy && user.ee_available && (
                             <Menu.Item key="licenses" style={itemStyle} data-attr="menu-item-licenses">
                                 <LockOutlined />

frontend/src/scenes/sceneLogic.js

@@ -23,6 +23,7 @@ export const scenes = {
     annotations: () => import(/* webpackChunkName: 'annotations' */ './annotations/AnnotationsScene'),
     team: () => import(/* webpackChunkName: 'team' */ './team/Team'),
     licenses: () => import(/* webpackChunkName: 'setup' */ './setup/Licenses'),
+    systemStatus: () => import(/* webpackChunkName: 'setup' */ './system_status/SystemStatus'),
     preflight: () => import(/* webpackChunkName: 'preflightCheck' */ './setup/PreflightCheck'),
     signup: () => import(/* webpackChunkName: 'signup' */ './team/Signup'),
     ingestion: () => import(/* webpackChunkName: 'ingestion' */ './ingestion/IngestionWizard'),
@@ -56,6 +57,7 @@ export const routes = {
     '/annotations': 'annotations',
     '/team': 'team',
     '/setup/licenses': 'licenses',
+    '/system_status': 'systemStatus',
     '/preflight': 'preflight',
     '/signup': 'signup',
     '/ingestion': 'ingestion',

frontend/src/scenes/system_status/SystemStatus.tsx

@@ -0,0 +1,47 @@
+import React from 'react'
+import { hot } from 'react-hot-loader/root'
+import { Alert, Table } from 'antd'
+import { systemStatusLogic } from './systemStatusLogic'
+import { useValues } from 'kea'
+
+const columns = [
+    {
+        title: 'Metric',
+        dataIndex: 'metric',
+    },
+    {
+        title: 'Value',
+        dataIndex: 'value',
+    },
+]
+
+export const SystemStatus = hot(_Status)
+function _Status(): JSX.Element {
+    const { systemStatus, systemStatusLoading, error } = useValues(systemStatusLogic)
+    return (
+        <div>
+            <h1 className="page-header">System Status</h1>
+            <p style={{ maxWidth: 600 }}>
+                <i>Here you can find all the critical runtime details about your PostHog installation.</i>
+            </p>
+            <br />
+            {error && (
+                <Alert
+                    message={error.detail || <span>Something went wrong. Please try again or contact us.</span>}
+                    type="error"
+                />
+            )}
+            <br />
+            <Table
+                data-attr="system-status-table"
+                size="small"
+                rowKey={(item): string => item.metric}
+                pagination={{ pageSize: 99999, hideOnSinglePage: true }}
+                rowClassName="cursor-pointer"
+                dataSource={systemStatus}
+                columns={columns}
+                loading={systemStatusLoading}
+            />
+        </div>
+    )
+}

frontend/src/scenes/system_status/systemStatusLogic.ts

@@ -0,0 +1,46 @@
+import api from 'lib/api'
+import { kea } from 'kea'
+
+interface Error {
+    detail: string
+    code: string
+}
+
+interface SystemStatus {
+    metric: string
+    value: string
+}
+
+export const systemStatusLogic = kea({
+    actions: {
+        setError: (error: Error) => ({ error }),
+        addSystemStatus: (systemStatus: SystemStatus) => ({ systemStatus }),
+    },
+    loaders: {
+        systemStatus: [
+            [],
+            {
+                loadSystemStatus: async () => {
+                    return (await api.get('_system_status')).results
+                },
+            },
+        ],
+    },
+    reducers: {
+        systemStatus: {
+            addSystemStatus: (state: Array<SystemStatus>, { systemStatus }) => [systemStatus, ...state],
+        },
+        error: [
+            false,
+            {
+                setError: (_, { error }) => error,
+            },
+        ],
+    },
+
+    events: ({ actions }) => ({
+        afterMount: () => {
+            actions.loadSystemStatus()
+        },
+    }),
+})

posthog/urls.py

@@ -23,7 +23,7 @@
 from .api import api_not_found, capture, dashboard, decide, router, team, user
 from .models import Event, Team, User
 from .utils import render_template
-from .views import health, preflight_check, stats
+from .views import health, preflight_check, stats, system_status
 
 
 def home(request, **kwargs):
@@ -207,6 +207,7 @@ def opt_slash_path(route: str, view: Callable, name: Optional[str] = None) -> st
     opt_slash_path("_health", health),
     opt_slash_path("_stats", stats),
     opt_slash_path("_preflight", preflight_check),
+    opt_slash_path("_system_status", system_status),
     # admin
     path("admin/", admin.site.urls),
     path("admin/", include("loginas.urls")),

posthog/utils.py

@@ -18,6 +18,7 @@
 from dateutil import parser
 from dateutil.relativedelta import relativedelta
 from django.conf import settings
+from django.db.utils import DatabaseError
 from django.http import HttpRequest, HttpResponse
 from django.template.loader import get_template
 from django.utils import timezone
@@ -365,3 +366,52 @@ def get_machine_id() -> str:
     # MAC addresses are 6 bits long, so overflow shouldn't happen
     # hashing here as we don't care about the actual address, just it being rather consistent
     return hashlib.md5(uuid.getnode().to_bytes(6, "little")).hexdigest()
+
+
+def get_table_size(table_name):
+    from django.db import connection
+
+    query = (
+        f'SELECT pg_size_pretty(pg_total_relation_size(relid)) AS "size" '
+        f"FROM pg_catalog.pg_statio_user_tables "
+        f"WHERE relname = '{table_name}'"
+    )
+    cursor = connection.cursor()
+    cursor.execute(query)
+    return dict_from_cursor_fetchall(cursor)
+
+
+def get_table_approx_count(table_name):
+    from django.db import connection
+
+    query = f"SELECT reltuples::BIGINT as \"approx_count\" FROM pg_class WHERE relname = '{table_name}'"
+    cursor = connection.cursor()
+    cursor.execute(query)
+    return dict_from_cursor_fetchall(cursor)
+
+
+def is_postgres_alive() -> bool:
+    from posthog.models import User
+
+    try:
+        User.objects.count()
+        return True
+    except DatabaseError:
+        return False
+
+
+def is_redis_alive() -> bool:
+    try:
+        return get_redis_heartbeat() != "offline"
+    except BaseException:
+        return False
+
+
+def get_redis_info() -> dict:
+    redis_instance = redis.from_url(settings.REDIS_URL, db=0)
+    return redis_instance.info()
+
+
+def get_redis_queue_depth() -> int:
+    redis_instance = redis.from_url(settings.REDIS_URL, db=0)
+    return redis_instance.llen("celery")

posthog/views.py

@@ -1,10 +1,20 @@
 from typing import Dict, Union
 
-from django.db import DatabaseError
+from django.conf import settings
+from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse, JsonResponse
 from django.views.decorators.cache import never_cache
+from rest_framework.exceptions import AuthenticationFailed
+
+from posthog.utils import (
+    get_redis_info,
+    get_redis_queue_depth,
+    get_table_approx_count,
+    get_table_size,
+    is_postgres_alive,
+    is_redis_alive,
+)
 
-from .models.user import User
 from .utils import get_redis_heartbeat
 
 
@@ -19,19 +29,55 @@ def stats(request):
 
 
 @never_cache
-def preflight_check(request):
-    redis: bool = False
-    db: bool = False
+@login_required
+def system_status(request):
+    is_multitenancy: bool = getattr(settings, "MULTI_TENANCY", False)
+
+    if is_multitenancy and not request.user.is_staff:
+        raise AuthenticationFailed(detail="You're not authorized.")
+
+    from .models import Element, Event
+
+    redis_alive = is_redis_alive()
+    postgres_alive = is_postgres_alive()
+
+    metrics = list()
+
+    metrics.append({"metric": "Redis alive", "value": str(redis_alive)})
+    metrics.append({"metric": "Postgres DB alive", "value": str(postgres_alive)})
 
-    try:
-        redis = get_redis_heartbeat() != "offline"
-    except BaseException:
-        pass
+    if postgres_alive:
+        event_table_count = get_table_approx_count(Event._meta.db_table)[0]["approx_count"]
+        event_table_size = get_table_size(Event._meta.db_table)[0]["size"]
 
-    try:
-        User.objects.count()
-        db = True
-    except DatabaseError:
-        pass
+        element_table_count = get_table_approx_count(Element._meta.db_table)[0]["approx_count"]
+        element_table_size = get_table_size(Element._meta.db_table)[0]["size"]
 
-    return JsonResponse({"django": True, "redis": redis, "db": db})
+        metrics.append(
+            {"metric": "Postgres Element table", "value": f"ca {element_table_count} rows ({element_table_size})"}
+        )
+        metrics.append({"metric": "Postgres Event table", "value": f"ca {event_table_count} rows ({event_table_size})"})
+
+    if redis_alive:
+        import redis
+
+        try:
+            redis_info = get_redis_info()
+            redis_queue_depth = get_redis_queue_depth()
+            metrics.append({"metric": "Redis current queue depth", "value": f"{redis_queue_depth}"})
+            metrics.append({"metric": "Redis memory used", "value": f"{redis_info['used_memory_human']}"})
+            metrics.append({"metric": "Redis memory peak", "value": f"{redis_info['used_memory_peak_human']}"})
+            metrics.append(
+                {"metric": "Redis total memory available", "value": f"{redis_info['total_system_memory_human']}"}
+            )
+        except redis.exceptions.ConnectionError as e:
+            metrics.append(
+                {"metric": "Redis metrics", "value": f"Redis connected but then failed to return metrics: {e}"}
+            )
+
+    return JsonResponse({"results": metrics})
+
+
+@never_cache
+def preflight_check(request):
+    return JsonResponse({"django": True, "redis": is_redis_alive(), "db": is_postgres_alive()})