Add support for killstreaks

Add support for parsing killstreaks from logs. These can be useful for
finding clips for frag movies. I didn't add them to the totals page
since as an extrema-based statistic, the only thing we can really show
is max/count. I think those are better shown on the logs page.

Closes: #75
Signed-off-by: Sean Anderson <[email protected]>

cluster.sql

@@ -7,6 +7,7 @@
 
 CLUSTER VERBOSE player_stats_backing USING player_stats_pkey;
 CLUSTER VERBOSE player_stats_extra USING player_stats_extra_pkey;
+CLUSTER VERBOSE killstreak USING killstreak_pkey;
 CLUSTER VERBOSE medic_stats USING medic_stats_pkey;
 CLUSTER VERBOSE class_stats USING class_stats_pkey;
 CLUSTER VERBOSE weapon_stats USING weapon_stats_pkey;

trends/importer/logs.py

@@ -357,6 +357,23 @@ def import_log(c, logid, log):
                                  %(dmg)s, %(avg_dmg)s, %(shots)s, %(hits)s
                              );""", weapon)
 
+    for killstreak in log.get('killstreaks', ()):
+        try:
+            steamid = SteamID(killstreak['steamid'])
+        except ValueError:
+            continue
+
+        try:
+            c.execute("SAVEPOINT before_killstreak;")
+            c.execute("""INSERT INTO killstreak (
+                             logid, playerid, time, kills
+                         ) VALUES (
+                            %s, (SELECT playerid FROM player WHERE steamid64 = %s), %s, %s
+                         );""", (logid, steamid, killstreak['time'], killstreak['streak']))
+        except (psycopg2.errors.ForeignKeyViolation, psycopg2.errors.NotNullViolation):
+            logging.warning("%s is only present in killstreak for log %s", steamid, logid)
+            c.execute("ROLLBACK TO SAVEPOINT before_killstreak;")
+
     for (seq, msg) in enumerate(log['chat']):
         try:
             steamid = SteamID(msg['steamid']) if msg['steamid'] != 'Console' else None
@@ -784,6 +801,9 @@ def import_logs(c, fetcher, update_only):
                            PRIMARY KEY ({})
                        );""".format(table[0], table[0], table[1]))
     # This doesn't include foreign keys, so include some which we want to handle in import_log
+    cur.execute("""ALTER TABLE killstreak
+                   ADD FOREIGN KEY (logid, playerid)
+                       REFERENCES player_stats_backing (logid, playerid);""")
     cur.execute("""ALTER TABLE heal_stats
                    ADD FOREIGN KEY (logid, healer)
                        REFERENCES player_stats_backing (logid, playerid),

trends/migrations/killstreak.py

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright (C) 2024 Sean Anderson <[email protected]>
+
+import logging
+import sys
+
+import psycopg2.extras
+
+from ..importer.cli import init_logging
+from ..sql import db_connect
+from ..steamid import SteamID
+from ..util import chunk
+
+def migrate(database):
+    init_logging(logging.DEBUG)
+    with db_connect(database) as c:
+        with c.cursor() as cur:
+            logging.info("BEGIN")
+            cur.execute("BEGIN");
+            logging.info("CREATE TABLE killstreak")
+            cur.execute("""
+                CREATE TABLE killstreak (
+                    logid INT NOT NULL,
+                    playerid INT NOT NULL,
+                    time INT NOT NULL,
+                    kills INT NOT NULL CHECK (kills > 0),
+                    PRIMARY KEY (playerid, logid, time)
+                );""")
+
+            with c.cursor(name='streaks') as streaks:
+                streaks.execute("""
+                    SELECT
+                       logid,
+                       streak -> 'steamid' AS steamid,
+                       streak -> 'time' AS time,
+                       streak -> 'streak' AS kills
+                    FROM (SELECT
+                           logid,
+                           json_array_elements(data -> 'killstreaks') AS streak
+                        FROM log_json
+                    ) AS killstreak;""")
+                for streaks in chunk(streaks, streaks.itersize):
+                    values = []
+                    for streak in streaks:
+                        try:
+                            values.append((streak[0], SteamID(streak[1]), streak[2], streak[3]))
+                        except ValueError:
+                            continue
+                    logging.info("INSERT killstreak")
+                    psycopg2.extras.execute_values(cur, """
+                        INSERT INTO killstreak (
+                            logid,
+                            playerid,
+                            time,
+                            kills
+                        ) SELECT
+                            logid,
+                            playerid,
+                            time,
+                            killstreak.kills
+                        FROM (VALUES %s) AS killstreak (logid, steamid64, time, kills)
+                        JOIN player USING (steamid64)
+                        JOIN player_stats USING (logid, playerid)
+                        ON CONFLICT DO NOTHING;""",
+                        values, "(%s, %s, %s, %s)")
+
+            logging.info("ALTER TABLE")
+            cur.execute("""
+                ALTER TABLE killstreak
+                ADD FOREIGN KEY (logid, playerid)
+                REFERENCES player_stats_backing (logid, playerid);""")
+            logging.info("COMMIT")
+            cur.execute("COMMIT;")
+
+if __name__ == "__main__":
+    migrate(sys.argv[1])

trends/schema.sql

@@ -608,6 +608,15 @@ CREATE TABLE IF NOT EXISTS player_stats_extra (
 	CHECK ((dmg_real ISNULL) = (dt_real ISNULL))
 );
 
+CREATE TABLE IF NOT EXISTS killstreak (
+	logid INT NOT NULL,
+	playerid INT NOT NULL,
+	time INT NOT NULL,
+	kills INT NOT NULL CHECK (kills > 0),
+	PRIMARY KEY (playerid, logid, time),
+	FOREIGN KEY (logid, playerid) REFERENCES player_stats_backing (logid, playerid)
+);
+
 CREATE TABLE IF NOT EXISTS medic_stats (
 	logid INT NOT NULL,
 	playerid INT NOT NULL,

trends/site/player.py

@@ -85,6 +85,8 @@ def get_overview():
     'headshots': "headshots",
     'headshots_hit': "headshots_hit",
     'sentries': "sentries",
+    # Not in extra but close enough...
+    'mks': "mks",
 }
 
 # Columns not in player_stats
@@ -175,6 +177,13 @@ def get_logs(c, playerid, filters, extra=False, order_clause="logid DESC", limit
            LEFT JOIN heal_stats_given AS hsg USING (logid, playerid)
            LEFT JOIN heal_stats_received AS hsr USING (logid, playerid)
            {"LEFT JOIN player_stats_extra AS pse USING (logid, playerid)" if extra else ""}
+           {'''LEFT JOIN (SELECT
+                       logid,
+                       playerid,
+                       max(kills) AS mks
+                   FROM killstreak
+                   GROUP BY logid, playerid
+               ) AS ks USING (logid, playerid)''' if extra else ""}
            WHERE ps.playerid = %(playerid)s
            ORDER BY {order_clause} NULLS LAST, logid DESC
            LIMIT %(limit)s OFFSET %(offset)s;""",

trends/site/root.py

@@ -481,6 +481,35 @@ def player_key(player):
                       GROUP BY event;""", params)
     events = { event_stats['event']: event_stats['events'] for event_stats in events.fetchall() }
 
+    killstreaks = db.cursor()
+    killstreaks.execute("""SELECT
+                               logid,
+                               title,
+                               array_agg(json_build_object(
+                                   'team', team,
+                                   'steamid64', steamid64,
+                                   'name', name,
+                                   'time', killstreak.time,
+                                   'kills', kills
+                               ) ORDER BY killstreak.time) AS killstreaks
+                           FROM (SELECT
+                                   logid,
+                                   playerid,
+                                   team,
+                                   name,
+                                   time,
+                                   killstreak.kills
+                               FROM killstreak
+                               JOIN player_stats USING (logid, playerid)
+                               JOIN name USING (nameid)
+                               WHERE logid IN %(logids)s
+                           ) AS killstreak
+                           JOIN log USING (logid)
+                           JOIN player USING (playerid)
+                           GROUP BY logid, title
+                           ORDER BY array_position(%(llogids)s, logid);""", params)
+    killstreaks = killstreaks.fetchall()
+
     chats = db.cursor()
     chats.execute("""SELECT
                         logid,
@@ -510,7 +539,8 @@ def player_key(player):
 
     return flask.render_template("log.html", logids=logids, logs=logs, matches=matches,
                                  rounds=rounds.fetchall(), players=players, totals=totals,
-                                 medics=medics, events=events, chats=chats)
+                                 medics=medics, events=events, killstreaks=killstreaks,
+                                 chats=chats)
 
 metrics_extension = PrometheusMetrics.for_app_factory(group_by='endpoint', path=None)
 

trends/site/templates/log.html

@@ -627,6 +627,33 @@ <h3>Deaths</h3>
 	{{ event(events['death']) }}
 	<h3>Assists</h3>
 	{{ event(events['assist']) }}
+	{% if killstreaks | count %}
+	<h2>Killstreaks</h2>
+	{% for killstreak in killstreaks %}
+	<h3>{{ killstreak['title'] }}</h3>
+	<table>
+		<thead class="sortable"><tr>
+			<th>Time</th>
+			<th>Team</th>
+			<th>Player</th>
+			<th>Kills</th>
+		</tr></thead>
+		<tbody>
+		{% for streak in killstreak['killstreaks'] %}
+		<tr>
+			{{ duration_col(streak.time) }}
+			<td class="left {{ streak.team | lower }}">{{ streak.team }}</td>
+			<td class="left">
+				{{ playerlink(streak.steamid64, streak.name,
+					      players[streak.steamid64]) }}
+			</td>
+			<td>{{ streak.kills }}</td>
+		</tr>
+		{% endfor %}
+		</tbody>
+	</table>
+	{% endfor %}
+	{% endif %}
 	<h2>Chat</h2>
 	{% for chat in chats %}
 	<h3>{{ chat['title'] }}</h3>

trends/site/templates/macros/logs.html

@@ -28,6 +28,7 @@
 	<th><abbr title="Healing Recieved per Minute ">HR/M</abbr></th>
 	<th><abbr title="Accuracy">Acc</abbr></th>
 	{% if extra %}
+	<th><abbr title="Longest killstreak">LKS</abbr></th>
 	<th><abbr title="Most kills in one life">K/1</abbr></th>
 	<th><abbr title="Airshots">AS</abbr></th>
 	<th><abbr title="Medkit score: small = 1, medium = 2, large = 3">MS</abbr></th>
@@ -86,6 +87,7 @@
 			<td>{{ optint(log['hpm_recieved']) }}</td>
 			<td>{{ optformat("{:.0%}", log['acc']) }}</td>
 			{% if extra %}
+			<td>{{ optint(log.mks) }}</td>
 			<td>{{ optint(log.lks) }}</td>
 			<td>{{ optint(log.airshots) }}</td>
 			<td>{{ optint(log.medkits) }}</td>

trends/site/templates/player/logs.html

@@ -30,6 +30,7 @@ <h2>Logs</h2>
 		'hrm': "Heals received per minute",
 		'acc': "Accuracy",
 		'date': "Date",
+		'mks': "Longest killstreak",
 		'lks': "Most kills in one life",
 		'airshots': "Airshots",
 		'medkits': "Medkit score",

trends/sql.py

@@ -110,6 +110,7 @@ def db_init(c):
 log_tables = (('log', 'logid'), ('log_json', 'logid'), ('round', 'logid, seq'),
              ('player_stats_backing', 'playerid, logid'),
              ('player_stats_extra', 'playerid, logid'),
+             ('killstreak', 'playerid, logid, time'),
              ('medic_stats', 'playerid, logid'),
              ('heal_stats', 'logid, healer, healee'),
              ('class_stats', 'playerid, logid, classid'),