From 985148ee7e019c2c80bae4754a12b6a06f009a13 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Fri, 8 Jul 2022 10:48:25 +0000 Subject: [PATCH 001/213] feat: buffer 2.0 (#10653) * feat: buffer 2.0 proposal * add tests * prevent infinite retrying * perf * updates * tweaks * Update latest_migrations.manifest * Update plugin-server/src/main/ingestion-queues/buffer.ts * update * updates * fix migrations issue * reliability uopdates * fix tests * test fix * e2e test * test * test * ?? * cleanup --- latest_migrations.manifest | 2 +- .../batch-processing/each-batch-buffer.ts | 69 ----------------- .../src/main/ingestion-queues/buffer.ts | 75 +++++++++++++++++++ .../src/main/ingestion-queues/kafka-queue.ts | 31 +------- .../src/main/ingestion-queues/queue.ts | 5 -- plugin-server/src/main/pluginsServer.ts | 13 ++++ plugin-server/src/types.ts | 1 - plugin-server/src/utils/db/db.ts | 9 +++ .../event-pipeline/1-emitToBufferStep.ts | 4 +- .../src/worker/ingestion/process-event.ts | 10 --- plugin-server/tests/e2e.buffer.test.ts | 72 +----------------- plugin-server/tests/helpers/kafka.ts | 35 +-------- plugin-server/tests/helpers/sql.ts | 3 +- plugin-server/tests/main/db.test.ts | 15 ++++ .../main/ingestion-queues/buffer.test.ts | 50 +++++++++++++ .../main/ingestion-queues/each-batch.test.ts | 34 --------- .../main/ingestion-queues/kafka-queue.test.ts | 3 - .../event-pipeline/emitToBufferStep.test.ts | 14 ++-- posthog/migrations/0251_event_buffer.py | 22 ++++++ posthog/models/__init__.py | 2 + posthog/models/event_buffer.py | 7 ++ 21 files changed, 211 insertions(+), 265 deletions(-) delete mode 100644 plugin-server/src/main/ingestion-queues/batch-processing/each-batch-buffer.ts create mode 100644 plugin-server/src/main/ingestion-queues/buffer.ts create mode 100644 plugin-server/tests/main/ingestion-queues/buffer.test.ts create mode 100644 posthog/migrations/0251_event_buffer.py create mode 100644 posthog/models/event_buffer.py diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 81191ee4283729..0d53977d79fc8e 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -3,7 +3,7 @@ auth: 0012_alter_user_first_name_max_length axes: 0006_remove_accesslog_trusted contenttypes: 0002_remove_content_type_name ee: 0013_silence_deprecated_tags_warnings -posthog: 0250_exportedasset_created_by +posthog: 0251_event_buffer rest_hooks: 0002_swappable_hook_model sessions: 0001_initial social_django: 0010_uid_db_index diff --git a/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-buffer.ts b/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-buffer.ts deleted file mode 100644 index 83ffe8e8a503bf..00000000000000 --- a/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-buffer.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { EachBatchPayload, KafkaMessage } from 'kafkajs' - -import { status } from '../../../utils/status' -import { runInstrumentedFunction } from '../../utils' -import { KafkaQueue } from '../kafka-queue' - -export async function eachMessageBuffer( - message: KafkaMessage, - resolveOffset: EachBatchPayload['resolveOffset'], - heartbeat: EachBatchPayload['heartbeat'], - queue: KafkaQueue -): Promise { - const bufferEvent = JSON.parse(message.value!.toString()) - await runInstrumentedFunction({ - server: queue.pluginsServer, - event: bufferEvent, - func: () => queue.workerMethods.runBufferEventPipeline(bufferEvent), - statsKey: `kafka_queue.ingest_buffer_event`, - timeoutMessage: 'After 30 seconds still running runBufferEventPipeline', - }) - resolveOffset(message.offset) - await heartbeat() -} - -export async function eachBatchBuffer( - { batch, resolveOffset, heartbeat, commitOffsetsIfNecessary, isStale, isRunning }: EachBatchPayload, - queue: KafkaQueue -): Promise { - if (batch.messages.length === 0 || !isRunning() || isStale()) { - status.info('🚪', `Bailing out of a batch of ${batch.messages.length} buffer events`, { - isRunning: isRunning(), - isStale: isStale(), - }) - await heartbeat() - return - } - - const batchStartTimer = new Date() - - /** First message to be processed post-sleep. Undefined means there's no sleep needed. */ - let cutoffMessage: KafkaMessage | undefined - /** How long should we sleep for until we have a desired delay in processing. */ - let consumerSleepMs = 0 - for (const message of batch.messages) { - // Kafka timestamps are Unix timestamps in string format - const processAt = Number(message.timestamp) + queue.pluginsServer.BUFFER_CONVERSION_SECONDS * 1000 - const delayUntilTimeToProcess = processAt - Date.now() - - if (delayUntilTimeToProcess <= 0 && !cutoffMessage) { - await eachMessageBuffer(message, resolveOffset, heartbeat, queue) - } else { - if (!cutoffMessage) { - cutoffMessage = message - } - if (delayUntilTimeToProcess > consumerSleepMs) { - consumerSleepMs = delayUntilTimeToProcess - } - } - } - await commitOffsetsIfNecessary() - if (cutoffMessage) { - // Pause the consumer for this partition until we can process all unprocessed messages from this batch - // This will also seek within the partition to the offset of the cutoff message - queue.pluginsServer.statsd?.gauge('buffer_sleep', consumerSleepMs, { partition: String(batch.partition) }) - await queue.bufferSleep(consumerSleepMs, batch.partition, cutoffMessage.offset, heartbeat) - } - - queue.pluginsServer.statsd?.timing('kafka_queue.each_batch_buffer', batchStartTimer) -} diff --git a/plugin-server/src/main/ingestion-queues/buffer.ts b/plugin-server/src/main/ingestion-queues/buffer.ts new file mode 100644 index 00000000000000..3f0e76f3a26b5b --- /dev/null +++ b/plugin-server/src/main/ingestion-queues/buffer.ts @@ -0,0 +1,75 @@ +import Piscina from '@posthog/piscina' +import { PluginEvent } from '@posthog/plugin-scaffold' + +import { Hub } from '../../types' +import { runInstrumentedFunction } from '../utils' + +export function runBufferEventPipeline(hub: Hub, piscina: Piscina, event: PluginEvent) { + hub.lastActivity = new Date().valueOf() + hub.lastActivityType = 'runBufferEventPipeline' + return piscina.run({ task: 'runBufferEventPipeline', args: { event } }) +} + +export async function runBuffer(hub: Hub, piscina: Piscina): Promise { + let eventRows: { id: number; event: PluginEvent }[] = [] + await hub.db.postgresTransaction(async (client) => { + const eventsResult = await client.query(` + UPDATE posthog_eventbuffer SET locked=true WHERE id IN ( + SELECT id FROM posthog_eventbuffer + WHERE process_at <= now() AND process_at > (now() - INTERVAL '30 minute') AND locked=false + ORDER BY id + LIMIT 10 + FOR UPDATE SKIP LOCKED + ) + RETURNING id, event + `) + eventRows = eventsResult.rows + }) + + const idsToDelete: number[] = [] + + // We don't indiscriminately delete all IDs to prevent the case when we don't trigger `runInstrumentedFunction` + // Once that runs, events will either go to the events table or the dead letter queue + const processBufferEvent = async (event: PluginEvent, id: number) => { + await runInstrumentedFunction({ + server: hub, + event: event, + func: () => runBufferEventPipeline(hub, piscina, event), + statsKey: `kafka_queue.ingest_buffer_event`, + timeoutMessage: 'After 30 seconds still running runBufferEventPipeline', + }) + idsToDelete.push(id) + } + + await Promise.all(eventRows.map((eventRow) => processBufferEvent(eventRow.event, eventRow.id))) + + if (idsToDelete.length > 0) { + await hub.db.postgresQuery( + `DELETE FROM posthog_eventbuffer WHERE id IN (${idsToDelete.join(',')})`, + [], + 'completeBufferEvent' + ) + hub.statsd?.increment('events_deleted_from_buffer', idsToDelete.length) + } +} + +export async function clearBufferLocks(hub: Hub): Promise { + /* + * If we crash during runBuffer we may end up with 2 scenarios: + * 1. "locked" rows with events that were never processed (crashed after fetching and before running the pipeline) + * 2. "locked" rows with events that were processed (crashed after the pipeline and before deletion) + * This clears any old locks such that the events are processed again. If there are any duplicates ClickHouse should collapse them. + */ + const recordsUpdated = await hub.db.postgresQuery( + `UPDATE posthog_eventbuffer + SET locked=false, process_at=now() + WHERE locked=true AND process_at < (now() - INTERVAL '30 minute') + RETURNING 1`, + [], + 'clearBufferLocks' + ) + + if (recordsUpdated.rowCount > 0 && hub.statsd) { + hub.statsd.increment('buffer_locks_cleared', recordsUpdated.rowCount) + } +} diff --git a/plugin-server/src/main/ingestion-queues/kafka-queue.ts b/plugin-server/src/main/ingestion-queues/kafka-queue.ts index 0424e66b728128..63c0d165a705b8 100644 --- a/plugin-server/src/main/ingestion-queues/kafka-queue.ts +++ b/plugin-server/src/main/ingestion-queues/kafka-queue.ts @@ -5,9 +5,8 @@ import { Hub, WorkerMethods } from '../../types' import { timeoutGuard } from '../../utils/db/utils' import { status } from '../../utils/status' import { killGracefully } from '../../utils/utils' -import { KAFKA_BUFFER, KAFKA_EVENTS_JSON, prefix as KAFKA_PREFIX } from './../../config/kafka-topics' +import { KAFKA_EVENTS_JSON, prefix as KAFKA_PREFIX } from './../../config/kafka-topics' import { eachBatchAsyncHandlers } from './batch-processing/each-batch-async-handlers' -import { eachBatchBuffer } from './batch-processing/each-batch-buffer' import { eachBatchIngestion } from './batch-processing/each-batch-ingestion' import { addMetricsEventListeners, emitConsumerGroupMetrics } from './kafka-metrics' @@ -25,9 +24,7 @@ export class KafkaQueue { private consumer: Consumer private consumerGroupMemberId: string | null private wasConsumerRan: boolean - private sleepTimeout: NodeJS.Timeout | null private ingestionTopic: string - private bufferTopic: string private eventsTopic: string private eachBatch: Record @@ -37,16 +34,13 @@ export class KafkaQueue { this.consumer = KafkaQueue.buildConsumer(this.kafka, this.consumerGroupId()) this.wasConsumerRan = false this.workerMethods = workerMethods - this.sleepTimeout = null this.consumerGroupMemberId = null this.consumerReady = false this.ingestionTopic = this.pluginsServer.KAFKA_CONSUMPTION_TOPIC! - this.bufferTopic = KAFKA_BUFFER this.eventsTopic = KAFKA_EVENTS_JSON this.eachBatch = { [this.ingestionTopic]: eachBatchIngestion, - [this.bufferTopic]: eachBatchBuffer, [this.eventsTopic]: eachBatchAsyncHandlers, } } @@ -56,7 +50,6 @@ export class KafkaQueue { if (this.pluginsServer.capabilities.ingestion) { topics.push(this.ingestionTopic) - topics.push(this.bufferTopic) } else if (this.pluginsServer.capabilities.processAsyncHandlers) { topics.push(this.eventsTopic) } else { @@ -140,28 +133,6 @@ export class KafkaQueue { return await startPromise } - async bufferSleep( - sleepMs: number, - partition: number, - offset: string, - heartbeat: () => Promise - ): Promise { - await this.pause(this.bufferTopic, partition) - const sleepHeartbeatInterval = setInterval(async () => { - await heartbeat() // Send a heartbeat once a second so that the broker knows we're still alive - }, 1000) - this.sleepTimeout = setTimeout(() => { - if (this.sleepTimeout) { - clearTimeout(this.sleepTimeout) - } - clearInterval(sleepHeartbeatInterval) - // Seek so that after resuming we start exactly where we left off (with the cutoff message) - // instead of skipping over messages left for later in the last batch - this.consumer.seek({ topic: this.bufferTopic, partition, offset }) - this.resume(this.bufferTopic, partition) - }, sleepMs) - } - async pause(targetTopic: string, partition?: number): Promise { if (this.wasConsumerRan && !this.isPaused(targetTopic, partition)) { const pausePayload: ConsumerManagementPayload = { topic: targetTopic } diff --git a/plugin-server/src/main/ingestion-queues/queue.ts b/plugin-server/src/main/ingestion-queues/queue.ts index 85d2edeca542d0..8dd748b7393c6b 100644 --- a/plugin-server/src/main/ingestion-queues/queue.ts +++ b/plugin-server/src/main/ingestion-queues/queue.ts @@ -25,11 +25,6 @@ export async function startQueues( workerMethods: Partial = {} ): Promise { const mergedWorkerMethods = { - runBufferEventPipeline: (event: PluginEvent) => { - server.lastActivity = new Date().valueOf() - server.lastActivityType = 'runBufferEventPipeline' - return piscina.run({ task: 'runBufferEventPipeline', args: { event } }) - }, runAsyncHandlersEventPipeline: (event: IngestionEvent) => { server.lastActivity = new Date().valueOf() server.lastActivityType = 'runAsyncHandlersEventPipeline' diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index 642e4b35c51f16..7b21b271e7cfe3 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -23,6 +23,7 @@ import { cancelAllScheduledJobs } from '../utils/node-schedule' import { PubSub } from '../utils/pubsub' import { status } from '../utils/status' import { delay, getPiscinaStats, stalenessCheck } from '../utils/utils' +import { clearBufferLocks, runBuffer } from './ingestion-queues/buffer' import { KafkaQueue } from './ingestion-queues/kafka-queue' import { startQueues } from './ingestion-queues/queue' import { startJobQueueConsumer } from './job-queues/job-queue-consumer' @@ -234,6 +235,18 @@ export async function startPluginsServer( } }) + // every 5 seconds process buffer events + schedule.scheduleJob('*/5 * * * * *', async () => { + if (piscina) { + await runBuffer(hub!, piscina) + } + }) + + // every 30 minutes clear any locks that may have lingered on the buffer table + schedule.scheduleJob('*/30 * * * *', async () => { + await clearBufferLocks(hub!) + }) + // every minute log information on kafka consumer if (queue) { schedule.scheduleJob('0 * * * * *', async () => { diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index dbc646456998be..58fcfe54e31aab 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -389,7 +389,6 @@ export interface PluginTask { } export type WorkerMethods = { - runBufferEventPipeline: (event: PluginEvent) => Promise runAsyncHandlersEventPipeline: (event: IngestionEvent) => Promise runEventPipeline: (event: PluginEvent) => Promise } diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index 300ce2a5d9c816..ae8b57ec1458ac 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -2041,4 +2041,13 @@ export class DB { ) return response.rowCount > 0 } + + public async addEventToBuffer(event: Record, processAt: DateTime): Promise { + await this.postgresQuery( + `INSERT INTO posthog_eventbuffer (event, process_at, locked) VALUES ($1, $2, $3)`, + [event, processAt.toISO(), false], + 'addEventToBuffer' + ) + this.statsd?.increment('events_sent_to_buffer') + } } diff --git a/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts index 88a9fcd2022126..94eb11c075dfe3 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts @@ -1,4 +1,5 @@ import { PluginEvent } from '@posthog/plugin-scaffold' +import { DateTime } from 'luxon' import { Hub, IngestionPersonData, TeamId } from '../../../types' import { EventPipelineRunner, StepResult } from './runner' @@ -20,7 +21,8 @@ export async function emitToBufferStep( const person = await runner.hub.db.fetchPerson(event.team_id, event.distinct_id) if (shouldBuffer(runner.hub, event, person, event.team_id)) { - await runner.hub.eventsProcessor.produceEventToBuffer(event) + const processEventAt = DateTime.now().plus({ seconds: runner.hub.BUFFER_CONVERSION_SECONDS }) + await runner.hub.db.addEventToBuffer(event, processEventAt) return null } else { return runner.nextStep('pluginsProcessEventStep', event, person) diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 48c06fae9c5765..1d1dffb76684cc 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -1,6 +1,5 @@ import ClickHouse from '@posthog/clickhouse' import { PluginEvent, Properties } from '@posthog/plugin-scaffold' -import crypto from 'crypto' import { DateTime } from 'luxon' import { Event as EventProto, IEvent } from '../../config/idl/protos' @@ -21,7 +20,6 @@ import { elementsToString, extractElements } from '../../utils/db/elements-chain import { KafkaProducerWrapper } from '../../utils/db/kafka-producer-wrapper' import { safeClickhouseString, sanitizeEventName, timeoutGuard } from '../../utils/db/utils' import { castTimestampOrNow, UUID } from '../../utils/utils' -import { KAFKA_BUFFER } from './../../config/kafka-topics' import { GroupTypeManager } from './group-type-manager' import { addGroupProperties } from './groups' import { upsertGroup } from './properties-updater' @@ -271,14 +269,6 @@ export class EventsProcessor { return { ...preIngestionEvent, person: personInfo } } - async produceEventToBuffer(bufferEvent: PluginEvent): Promise { - const partitionKeyHash = crypto.createHash('sha256') - partitionKeyHash.update(`${bufferEvent.team_id}:${bufferEvent.distinct_id}`) - const partitionKey = partitionKeyHash.digest('hex') - - await this.kafkaProducer.queueSingleJsonMessage(KAFKA_BUFFER, partitionKey, bufferEvent) - } - private async createSessionRecordingEvent( uuid: string, team_id: number, diff --git a/plugin-server/tests/e2e.buffer.test.ts b/plugin-server/tests/e2e.buffer.test.ts index 9eb9724b8c558a..95c9d400d0cc3d 100644 --- a/plugin-server/tests/e2e.buffer.test.ts +++ b/plugin-server/tests/e2e.buffer.test.ts @@ -1,17 +1,15 @@ import IORedis from 'ioredis' -import { DateTime } from 'luxon' import { ONE_HOUR } from '../src/config/constants' -import { KAFKA_BUFFER } from '../src/config/kafka-topics' import { startPluginsServer } from '../src/main/pluginsServer' import { LogLevel, PluginsServerConfig } from '../src/types' import { Hub } from '../src/types' -import { delay, UUIDT } from '../src/utils/utils' +import { UUIDT } from '../src/utils/utils' import { makePiscina } from '../src/worker/piscina' import { createPosthog, DummyPostHog } from '../src/worker/vm/extensions/posthog' import { writeToFile } from '../src/worker/vm/extensions/test-utils' import { delayUntilEventIngested, resetTestDatabaseClickhouse } from './helpers/clickhouse' -import { spyOnKafka } from './helpers/kafka' +import { resetKafka } from './helpers/kafka' import { pluginConfig39 } from './helpers/plugins' import { resetTestDatabase } from './helpers/sql' @@ -52,8 +50,6 @@ export function onEvent (event, { global }) { }` describe('E2E with buffer enabled', () => { - const delayUntilBufferMessageProduced = spyOnKafka(KAFKA_BUFFER, extraServerConfig) - let hub: Hub let stopServer: () => Promise let posthog: DummyPostHog @@ -63,6 +59,7 @@ describe('E2E with buffer enabled', () => { testConsole.reset() await resetTestDatabase(indexJs) await resetTestDatabaseClickhouse(extraServerConfig) + await resetKafka(extraServerConfig) const startResponse = await startPluginsServer(extraServerConfig, makePiscina) hub = startResponse.hub stopServer = startResponse.stop @@ -84,11 +81,9 @@ describe('E2E with buffer enabled', () => { await posthog.capture('custom event via buffer', { name: 'hehe', uuid }) await hub.kafkaProducer.flush() - const bufferTopicMessages = await delayUntilBufferMessageProduced() - await delayUntilEventIngested(() => hub.db.fetchEvents(), undefined, undefined, 200) + await delayUntilEventIngested(() => hub.db.fetchEvents(), undefined, undefined, 500) const events = await hub.db.fetchEvents() - expect(bufferTopicMessages.filter((message) => message.properties.uuid === uuid).length).toBe(1) expect(events.length).toBe(1) // processEvent ran and modified @@ -98,64 +93,5 @@ describe('E2E with buffer enabled', () => { // onEvent ran expect(testConsole.read()).toEqual([['processEvent'], ['onEvent', 'custom event via buffer']]) }) - - test('three events captured, processed, ingested', async () => { - expect((await hub.db.fetchEvents()).length).toBe(0) - - const uuid1 = new UUIDT().toString() - const uuid2 = new UUIDT().toString() - const uuid3 = new UUIDT().toString() - - // Batch 1 - await posthog.capture('custom event via buffer', { name: 'hehe', uuid: uuid1 }) - await posthog.capture('custom event via buffer', { name: 'hoho', uuid: uuid2 }) - await hub.kafkaProducer.flush() - // Batch 2 - waiting for a few seconds so that the event lands into a separate consumer batch - await delay(5000) - await posthog.capture('custom event via buffer', { name: 'hihi', uuid: uuid3 }) - await hub.kafkaProducer.flush() - - const bufferTopicMessages = await delayUntilBufferMessageProduced(3) - const events = await delayUntilEventIngested(() => hub.db.fetchEvents(), 3, undefined, 200) - - expect(bufferTopicMessages.filter((message) => message.properties.uuid === uuid1).length).toBe(1) - expect(bufferTopicMessages.filter((message) => message.properties.uuid === uuid2).length).toBe(1) - expect(bufferTopicMessages.filter((message) => message.properties.uuid === uuid3).length).toBe(1) - expect(events.length).toBe(3) - - // At least BUFFER_CONVERSION_SECONDS must have elapsed for each event between queuing and saving - expect( - DateTime.fromSQL(events[0].created_at, { zone: 'utc' }) - .diff(DateTime.fromISO(events[0].timestamp)) - .toMillis() - ).toBeGreaterThan(hub.BUFFER_CONVERSION_SECONDS * 1000) - expect( - DateTime.fromSQL(events[1].created_at, { zone: 'utc' }) - .diff(DateTime.fromISO(events[1].timestamp)) - .toMillis() - ).toBeGreaterThan(hub.BUFFER_CONVERSION_SECONDS * 1000) - expect( - DateTime.fromSQL(events[2].created_at, { zone: 'utc' }) - .diff(DateTime.fromISO(events[2].timestamp)) - .toMillis() - ).toBeGreaterThan(hub.BUFFER_CONVERSION_SECONDS * 1000) - - // processEvent ran and modified - expect(events[0].properties.processed).toEqual('hell yes') - expect(events[1].properties.processed).toEqual('hell yes') - expect(events[2].properties.processed).toEqual('hell yes') - const eventPluginUuids = events.map((event) => event.properties.upperUuid).sort() - expect(eventPluginUuids).toStrictEqual([uuid1.toUpperCase(), uuid2.toUpperCase(), uuid3.toUpperCase()]) - - // onEvent ran - expect(testConsole.read()).toEqual([ - ['processEvent'], - ['onEvent', 'custom event via buffer'], - ['processEvent'], - ['onEvent', 'custom event via buffer'], - ['processEvent'], - ['onEvent', 'custom event via buffer'], - ]) - }) }) }) diff --git a/plugin-server/tests/helpers/kafka.ts b/plugin-server/tests/helpers/kafka.ts index f28f66c0b3ca5a..c4159bc3f9270c 100644 --- a/plugin-server/tests/helpers/kafka.ts +++ b/plugin-server/tests/helpers/kafka.ts @@ -1,4 +1,4 @@ -import { Consumer, Kafka, logLevel } from 'kafkajs' +import { Kafka, logLevel } from 'kafkajs' import { defaultConfig, overrideWithEnv } from '../../src/config/config' import { @@ -15,7 +15,6 @@ import { import { PluginsServerConfig } from '../../src/types' import { UUIDT } from '../../src/utils/utils' import { KAFKA_EVENTS_DEAD_LETTER_QUEUE } from './../../src/config/kafka-topics' -import { delayUntilEventIngested } from './clickhouse' /** Clear the Kafka queue and return Kafka object */ export async function resetKafka(extraServerConfig?: Partial): Promise { @@ -57,35 +56,3 @@ async function createTopics(kafka: Kafka, topics: string[]) { } await admin.disconnect() } - -export function spyOnKafka( - topic: string, - serverConfig?: Partial -): (minLength?: number) => Promise { - let bufferTopicMessages: any[] - let bufferConsumer: Consumer - - beforeAll(async () => { - const kafka = await resetKafka(serverConfig) - bufferConsumer = kafka.consumer({ groupId: 'e2e-buffer-test' }) - await bufferConsumer.subscribe({ topic }) - await bufferConsumer.run({ - eachMessage: ({ message }) => { - const messageValueParsed = JSON.parse(message.value!.toString()) - bufferTopicMessages.push(messageValueParsed) - return Promise.resolve() // Not really needed but KafkaJS's typing accepts promises only - }, - }) - }) - - beforeEach(() => { - bufferTopicMessages = [] - }) - - afterAll(async () => { - await bufferConsumer.stop() - await bufferConsumer.disconnect() - }) - - return async (minLength) => await delayUntilEventIngested(() => bufferTopicMessages, minLength) -} diff --git a/plugin-server/tests/helpers/sql.ts b/plugin-server/tests/helpers/sql.ts index 0a9798e34a598d..e616e93fdf7591 100644 --- a/plugin-server/tests/helpers/sql.ts +++ b/plugin-server/tests/helpers/sql.ts @@ -56,7 +56,8 @@ TRUNCATE TABLE posthog_team, posthog_organizationmembership, posthog_organization, - posthog_user + posthog_user, + posthog_eventbuffer CASCADE ` diff --git a/plugin-server/tests/main/db.test.ts b/plugin-server/tests/main/db.test.ts index 6093aedf3c0428..a6a7e65523ce5c 100644 --- a/plugin-server/tests/main/db.test.ts +++ b/plugin-server/tests/main/db.test.ts @@ -1026,4 +1026,19 @@ describe('DB', () => { ) }) }) + + describe('addEventToBuffer', () => { + test('inserts event correctly', async () => { + const processAt = DateTime.now() + await db.addEventToBuffer({ foo: 'bar' }, processAt) + + const bufferResult = await db.postgresQuery( + 'SELECT event, process_at FROM posthog_eventbuffer', + [], + 'addEventToBufferTest' + ) + expect(bufferResult.rows[0].event).toEqual({ foo: 'bar' }) + expect(processAt).toEqual(DateTime.fromISO(bufferResult.rows[0].process_at)) + }) + }) }) diff --git a/plugin-server/tests/main/ingestion-queues/buffer.test.ts b/plugin-server/tests/main/ingestion-queues/buffer.test.ts new file mode 100644 index 00000000000000..9751bbc9a0318f --- /dev/null +++ b/plugin-server/tests/main/ingestion-queues/buffer.test.ts @@ -0,0 +1,50 @@ +import Piscina from '@posthog/piscina' +import { DateTime } from 'luxon' + +import { runBuffer } from '../../../src/main/ingestion-queues/buffer' +import { runInstrumentedFunction } from '../../../src/main/utils' +import { Hub } from '../../../src/types' +import { DB } from '../../../src/utils/db/db' +import { createHub } from '../../../src/utils/db/hub' +import { resetTestDatabase } from '../../helpers/sql' + +// jest.mock('../../../src/utils') +jest.mock('../../../src/main/utils') + +describe('Event buffer', () => { + let hub: Hub + let closeServer: () => Promise + let db: DB + + beforeEach(async () => { + ;[hub, closeServer] = await createHub() + await resetTestDatabase(undefined, {}, {}, { withExtendedTestData: false }) + db = hub.db + }) + + afterEach(async () => { + await closeServer() + jest.clearAllMocks() + }) + + describe('runBuffer', () => { + test('processes events from buffer and deletes them', async () => { + const processAt = DateTime.now() + await db.addEventToBuffer({ foo: 'bar' }, processAt) + await db.addEventToBuffer({ foo: 'bar' }, processAt) + + await runBuffer(hub, {} as Piscina) + + expect(runInstrumentedFunction).toHaveBeenCalledTimes(2) + expect(runInstrumentedFunction).toHaveBeenLastCalledWith(expect.objectContaining({ event: { foo: 'bar' } })) + + const countResult = await db.postgresQuery( + 'SELECT count(*) FROM posthog_eventbuffer', + [], + 'eventBufferCountTest' + ) + + expect(Number(countResult.rows[0].count)).toEqual(0) + }) + }) +}) diff --git a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts index 674cd34dcca0e9..547df77e399314 100644 --- a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts +++ b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts @@ -1,6 +1,5 @@ import { eachBatch } from '../../../src/main/ingestion-queues/batch-processing/each-batch' import { eachBatchAsyncHandlers } from '../../../src/main/ingestion-queues/batch-processing/each-batch-async-handlers' -import { eachBatchBuffer } from '../../../src/main/ingestion-queues/batch-processing/each-batch-buffer' import { eachBatchIngestion } from '../../../src/main/ingestion-queues/batch-processing/each-batch-ingestion' import { ClickhouseEventKafka } from '../../../src/types' import { groupIntoBatches } from '../../../src/utils/utils' @@ -191,37 +190,4 @@ describe('eachBatchX', () => { }) }) }) - - describe('eachBatchBuffer', () => { - it('eachBatchBuffer triggers sleep for partition if a messages is newer than BUFFER_CONVERSION_SECONDS', async () => { - const systemDate = new Date(2022, 1, 1, 1, 0, 0, 0) - jest.useFakeTimers('modern') - jest.setSystemTime(systemDate) - - // the message is sent at the same time as the system, meaning we sleep for BUFFER_CONVERSION_SECONDS * 1000 - const batch = createBatch({ ...event, offset: '294' }, systemDate) - - await eachBatchBuffer(batch, queue) - - expect(queue.bufferSleep).toHaveBeenCalledWith(60000, 0, '294', expect.any(Function)) - - jest.clearAllTimers() - }) - - it('eachBatchBuffer processes a batch if the messages are older than BUFFER_CONVERSION_SECONDS', async () => { - const systemDate = new Date(2022, 1, 1, 1, 0, 0, 0) - jest.useFakeTimers('modern') - jest.setSystemTime(systemDate) - - // the message is sent at the same time as the system, meaning we sleep for BUFFER_CONVERSION_SECONDS * 1000 - const batch = createBatch(event, new Date(2020, 1, 1, 1, 0, 0, 0)) - - await eachBatchBuffer(batch, queue) - - expect(queue.workerMethods.runBufferEventPipeline).toHaveBeenCalledWith(event) - expect(queue.bufferSleep).not.toHaveBeenCalled() - - jest.clearAllTimers() - }) - }) }) diff --git a/plugin-server/tests/main/ingestion-queues/kafka-queue.test.ts b/plugin-server/tests/main/ingestion-queues/kafka-queue.test.ts index 3ba76a43af7c49..b97762ada1b8c6 100644 --- a/plugin-server/tests/main/ingestion-queues/kafka-queue.test.ts +++ b/plugin-server/tests/main/ingestion-queues/kafka-queue.test.ts @@ -41,7 +41,6 @@ describe.skip('KafkaQueue', () => { await resetTestDatabaseClickhouse(extraServerConfig) pluginServer = await startPluginsServer(extraServerConfig, makePiscina) hub = pluginServer.hub - piscina = pluginServer.piscina stopServer = pluginServer.stop posthog = createPosthog(hub, pluginConfig39) }) @@ -58,8 +57,6 @@ describe.skip('KafkaQueue', () => { gauge: jest.fn(), } as any - jest.spyOn(hub.eventsProcessor, 'produceEventToBuffer') - const uuid = new UUIDT().toString() await posthog.capture('custom event', { name: 'haha', uuid, distinct_id: 'some_id' }) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts index d6aea425a88a25..a12164943aca4e 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts @@ -43,19 +43,17 @@ beforeEach(() => { hub: { CONVERSION_BUFFER_ENABLED: true, BUFFER_CONVERSION_SECONDS: 60, - db: { fetchPerson: jest.fn().mockResolvedValue(existingPerson) }, - eventsProcessor: { - produceEventToBuffer: jest.fn(), - }, + db: { fetchPerson: jest.fn().mockResolvedValue(existingPerson), addEventToBuffer: jest.fn() }, + eventsProcessor: {}, }, } }) describe('emitToBufferStep()', () => { - it('calls `produceEventToBuffer` if event should be buffered, stops processing', async () => { + it('calls `addEventToBuffer` if event should be buffered, stops processing', async () => { const response = await emitToBufferStep(runner, pluginEvent, () => true) - expect(runner.hub.eventsProcessor.produceEventToBuffer).toHaveBeenCalledWith(pluginEvent) + expect(runner.hub.db.addEventToBuffer).toHaveBeenCalledWith(pluginEvent, expect.any(DateTime)) expect(runner.hub.db.fetchPerson).toHaveBeenCalledWith(2, 'my_id') expect(response).toEqual(null) }) @@ -65,7 +63,7 @@ describe('emitToBufferStep()', () => { expect(response).toEqual(['pluginsProcessEventStep', pluginEvent, existingPerson]) expect(runner.hub.db.fetchPerson).toHaveBeenCalledWith(2, 'my_id') - expect(runner.hub.eventsProcessor.produceEventToBuffer).not.toHaveBeenCalled() + expect(runner.hub.db.addEventToBuffer).not.toHaveBeenCalled() }) it('calls `processPersonsStep` for $snapshot events', async () => { @@ -75,7 +73,7 @@ describe('emitToBufferStep()', () => { expect(response).toEqual(['processPersonsStep', event, undefined]) expect(runner.hub.db.fetchPerson).not.toHaveBeenCalled() - expect(runner.hub.eventsProcessor.produceEventToBuffer).not.toHaveBeenCalled() + expect(runner.hub.db.addEventToBuffer).not.toHaveBeenCalled() }) }) diff --git a/posthog/migrations/0251_event_buffer.py b/posthog/migrations/0251_event_buffer.py new file mode 100644 index 00000000000000..af6b8f3b105b65 --- /dev/null +++ b/posthog/migrations/0251_event_buffer.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.11 on 2021-01-05 14:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("posthog", "0250_exportedasset_created_by"), + ] + + operations = [ + migrations.CreateModel( + name="EventBuffer", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("event", models.JSONField(null=True, blank=True)), + ("process_at", models.DateTimeField()), + ("locked", models.BooleanField()), + ], + ), + ] diff --git a/posthog/models/__init__.py b/posthog/models/__init__.py index 5c02ef09bca25c..67fcd5a720541f 100644 --- a/posthog/models/__init__.py +++ b/posthog/models/__init__.py @@ -9,6 +9,7 @@ from .element_group import ElementGroup from .entity import Entity from .event.event import Event +from .event_buffer import EventBuffer from .event_definition import EventDefinition from .event_property import EventProperty from .experiment import Experiment @@ -49,6 +50,7 @@ "ElementGroup", "Entity", "Event", + "EventBuffer", "EventDefinition", "EventProperty", "Experiment", diff --git a/posthog/models/event_buffer.py b/posthog/models/event_buffer.py new file mode 100644 index 00000000000000..f8cfed80775cdb --- /dev/null +++ b/posthog/models/event_buffer.py @@ -0,0 +1,7 @@ +from django.db import models + + +class EventBuffer(models.Model): + event: models.JSONField = models.JSONField(null=True, blank=True) + process_at: models.DateTimeField = models.DateTimeField() + locked: models.BooleanField = models.BooleanField() From 45277c1e85616e5d0253517ad308fbde56081720 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Fri, 8 Jul 2022 11:49:36 +0000 Subject: [PATCH 002/213] fix: only run buffer 2.0 on ingestion server (#10690) --- plugin-server/src/main/pluginsServer.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index 7b21b271e7cfe3..b07a2fb9face44 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -235,12 +235,14 @@ export async function startPluginsServer( } }) - // every 5 seconds process buffer events - schedule.scheduleJob('*/5 * * * * *', async () => { - if (piscina) { - await runBuffer(hub!, piscina) - } - }) + if (hub.capabilities.ingestion) { + // every 5 seconds process buffer events + schedule.scheduleJob('*/5 * * * * *', async () => { + if (piscina) { + await runBuffer(hub!, piscina) + } + }) + } // every 30 minutes clear any locks that may have lingered on the buffer table schedule.scheduleJob('*/30 * * * *', async () => { From c64acbc982b65a5dbdf045395eaee52e8f20b4bd Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 8 Jul 2022 14:25:36 +0200 Subject: [PATCH 003/213] fix(insights): editor panel flagg off layout fix (#10689) --- frontend/src/scenes/insights/EditorFilters/EditorFilters.scss | 1 - frontend/src/scenes/insights/EditorFilters/EditorFilters.tsx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/scenes/insights/EditorFilters/EditorFilters.scss b/frontend/src/scenes/insights/EditorFilters/EditorFilters.scss index 95d8e635cc15fe..80d792c4b0b088 100644 --- a/frontend/src/scenes/insights/EditorFilters/EditorFilters.scss +++ b/frontend/src/scenes/insights/EditorFilters/EditorFilters.scss @@ -16,7 +16,6 @@ @media screen and (min-width: $md) { display: flex; gap: 2rem; - flex-direction: column; .EditorFilterGroup { flex: 1; } diff --git a/frontend/src/scenes/insights/EditorFilters/EditorFilters.tsx b/frontend/src/scenes/insights/EditorFilters/EditorFilters.tsx index ce9406098b5a9c..ed291f2d54b509 100644 --- a/frontend/src/scenes/insights/EditorFilters/EditorFilters.tsx +++ b/frontend/src/scenes/insights/EditorFilters/EditorFilters.tsx @@ -309,7 +309,7 @@ export function EditorFilters({ insightProps, showing }: EditorFiltersProps): JS
From 0f199e333a0dbf903b3d4ddd074bf3d2bd54a9f7 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Fri, 8 Jul 2022 13:02:36 +0000 Subject: [PATCH 004/213] feat: increase buffer 2.0 throughput 200x (#10693) * feat: increase buffer 2.0 throughput 200x * clear interval on shutdown * fix error --- plugin-server/src/main/ingestion-queues/buffer.ts | 2 +- plugin-server/src/main/pluginsServer.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugin-server/src/main/ingestion-queues/buffer.ts b/plugin-server/src/main/ingestion-queues/buffer.ts index 3f0e76f3a26b5b..d31d69a2c56202 100644 --- a/plugin-server/src/main/ingestion-queues/buffer.ts +++ b/plugin-server/src/main/ingestion-queues/buffer.ts @@ -18,7 +18,7 @@ export async function runBuffer(hub: Hub, piscina: Piscina): Promise { SELECT id FROM posthog_eventbuffer WHERE process_at <= now() AND process_at > (now() - INTERVAL '30 minute') AND locked=false ORDER BY id - LIMIT 10 + LIMIT 40 FOR UPDATE SKIP LOCKED ) RETURNING id, event diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index b07a2fb9face44..963d60dbe04482 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -72,6 +72,7 @@ export async function startPluginsServer( let lastActivityCheck: NodeJS.Timeout | undefined let httpServer: Server | undefined let stopEventLoopMetrics: (() => void) | undefined + let bufferInterval: NodeJS.Timer | undefined let shutdownStatus = 0 @@ -87,6 +88,7 @@ export async function startPluginsServer( process.exit() } status.info('💤', ' Shutting down gracefully...') + bufferInterval && clearInterval(bufferInterval) lastActivityCheck && clearInterval(lastActivityCheck) cancelAllScheduledJobs() stopEventLoopMetrics?.() @@ -236,12 +238,12 @@ export async function startPluginsServer( }) if (hub.capabilities.ingestion) { - // every 5 seconds process buffer events - schedule.scheduleJob('*/5 * * * * *', async () => { + bufferInterval = setInterval(async () => { if (piscina) { + status.info('⚙️', 'Processing buffer events') await runBuffer(hub!, piscina) } - }) + }, 100) } // every 30 minutes clear any locks that may have lingered on the buffer table From cc71d4e592e73b9699b994aedc8e5c0df1b2db21 Mon Sep 17 00:00:00 2001 From: Tiina Turban Date: Fri, 8 Jul 2022 15:33:19 +0200 Subject: [PATCH 005/213] fix: Don't log to sentry on bad inputs (#10670) --- posthog/api/capture.py | 32 +++++++++++++++++++++++--------- posthog/api/utils.py | 17 +++++------------ requirements.in | 1 - requirements.txt | 2 -- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/posthog/api/capture.py b/posthog/api/capture.py index e9d58200bfadd2..c669e541868a3b 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -2,8 +2,9 @@ import json import re from datetime import datetime -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple +import structlog from dateutil import parser from django.conf import settings from django.http import JsonResponse @@ -16,7 +17,6 @@ from posthog.api.utils import ( EventIngestionContext, - capture_as_common_error_to_sentry_ratelimited, get_data, get_event_ingestion_context, get_token, @@ -32,6 +32,8 @@ from posthog.settings import KAFKA_EVENTS_PLUGIN_INGESTION_TOPIC from posthog.utils import cors_response, get_ip_address +logger = structlog.get_logger(__name__) + def parse_kafka_event_data( distinct_id: str, @@ -111,7 +113,7 @@ def _datetime_from_seconds_or_millis(timestamp: str) -> datetime: return datetime.fromtimestamp(timestamp_number, timezone.utc) -def _get_sent_at(data, request) -> Optional[datetime]: +def _get_sent_at(data, request) -> Tuple[Optional[datetime], Any]: try: if request.GET.get("_"): # posthog-js sent_at = request.GET["_"] @@ -120,15 +122,24 @@ def _get_sent_at(data, request) -> Optional[datetime]: elif request.POST.get("sent_at"): # when urlencoded body and not JSON (in some test) sent_at = request.POST["sent_at"] else: - return None + return None, None if re.match(r"^\d+(?:\.\d+)?$", sent_at): - return _datetime_from_seconds_or_millis(sent_at) + return _datetime_from_seconds_or_millis(sent_at), None - return parser.isoparse(sent_at) + return parser.isoparse(sent_at), None except Exception as error: - capture_as_common_error_to_sentry_ratelimited("Invalid sent_at value", error) - return None + statsd.incr("capture_endpoint_invalid_sent_at") + logger.exception(f"Invalid sent_at value", error=error) + return ( + None, + cors_response( + request, + generate_exception_response( + "capture", f"Malformed request data, invalid sent at: {error}", code="invalid_payload" + ), + ), + ) def _get_distinct_id(data: Dict[str, Any]) -> str: @@ -170,7 +181,10 @@ def get_event(request): if error_response: return error_response - sent_at = _get_sent_at(data, request) + sent_at, error_response = _get_sent_at(data, request) + + if error_response: + return error_response token = get_token(data, request) diff --git a/posthog/api/utils.py b/posthog/api/utils.py index 6b7e6dac9923f6..ed23ad2baac3ad 100644 --- a/posthog/api/utils.py +++ b/posthog/api/utils.py @@ -4,7 +4,7 @@ from enum import Enum, auto from typing import Any, List, Optional, Tuple, Union, cast -from ratelimit import limits +import structlog from rest_framework import request, status from sentry_sdk import capture_exception from statshog.defaults.django import statsd @@ -18,6 +18,8 @@ from posthog.models.user import User from posthog.utils import cors_response, load_data_from_request +logger = structlog.get_logger(__name__) + class PaginationMode(Enum): next = auto() @@ -141,22 +143,13 @@ def get_project_id(data, request) -> Optional[int]: return None -@limits(calls=10, period=1) -def capture_as_common_error_to_sentry_ratelimited(common_message, error): - try: - raise Exception(common_message) from error - except Exception as error_for_sentry: - capture_exception(error_for_sentry) - - def get_data(request): data = None try: data = load_data_from_request(request) except RequestParsingError as error: - capture_as_common_error_to_sentry_ratelimited( - "Invalid payload", error - ) # We still capture this on Sentry to identify actual potential bugs + statsd.incr("capture_endpoint_invalid_payload") + logger.exception(f"Invalid payload", error=error) return ( None, cors_response( diff --git a/requirements.in b/requirements.in index bd415ca6882d38..7d18006a8b79f7 100644 --- a/requirements.in +++ b/requirements.in @@ -52,7 +52,6 @@ pyjwt==2.4.0 python-dateutil==2.8.1 python3-saml==1.12.0 pytz==2021.1 -ratelimit==2.2.1 redis==3.4.1 requests==2.25.1 requests-oauthlib==1.3.0 diff --git a/requirements.txt b/requirements.txt index 46d49bda79728b..af6f43de4561fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -249,8 +249,6 @@ pytz==2021.1 # tzlocal pyyaml==6.0 # via drf-spectacular -ratelimit==2.2.1 - # via -r requirements.in redis==3.4.1 # via # -r requirements.in From 707b868f09356a16d6e6d0c86b9f3243e12c880a Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Fri, 8 Jul 2022 14:21:19 +0000 Subject: [PATCH 006/213] Revert "feat: increase buffer 2.0 throughput 200x (#10693)" (#10695) This reverts commit 0f199e333a0dbf903b3d4ddd074bf3d2bd54a9f7. --- plugin-server/src/main/ingestion-queues/buffer.ts | 2 +- plugin-server/src/main/pluginsServer.ts | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/plugin-server/src/main/ingestion-queues/buffer.ts b/plugin-server/src/main/ingestion-queues/buffer.ts index d31d69a2c56202..3f0e76f3a26b5b 100644 --- a/plugin-server/src/main/ingestion-queues/buffer.ts +++ b/plugin-server/src/main/ingestion-queues/buffer.ts @@ -18,7 +18,7 @@ export async function runBuffer(hub: Hub, piscina: Piscina): Promise { SELECT id FROM posthog_eventbuffer WHERE process_at <= now() AND process_at > (now() - INTERVAL '30 minute') AND locked=false ORDER BY id - LIMIT 40 + LIMIT 10 FOR UPDATE SKIP LOCKED ) RETURNING id, event diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index 963d60dbe04482..b07a2fb9face44 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -72,7 +72,6 @@ export async function startPluginsServer( let lastActivityCheck: NodeJS.Timeout | undefined let httpServer: Server | undefined let stopEventLoopMetrics: (() => void) | undefined - let bufferInterval: NodeJS.Timer | undefined let shutdownStatus = 0 @@ -88,7 +87,6 @@ export async function startPluginsServer( process.exit() } status.info('💤', ' Shutting down gracefully...') - bufferInterval && clearInterval(bufferInterval) lastActivityCheck && clearInterval(lastActivityCheck) cancelAllScheduledJobs() stopEventLoopMetrics?.() @@ -238,12 +236,12 @@ export async function startPluginsServer( }) if (hub.capabilities.ingestion) { - bufferInterval = setInterval(async () => { + // every 5 seconds process buffer events + schedule.scheduleJob('*/5 * * * * *', async () => { if (piscina) { - status.info('⚙️', 'Processing buffer events') await runBuffer(hub!, piscina) } - }, 100) + }) } // every 30 minutes clear any locks that may have lingered on the buffer table From 66101e5e10df3dc3904d2993b50803bb7bd653f4 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Fri, 8 Jul 2022 17:16:08 +0100 Subject: [PATCH 007/213] fix(trends): Disambiguate when joining multiple (#10697) * fix(trends): Disambiguate when joining multiple * Update snapshots Co-authored-by: neilkakkar --- posthog/queries/test/test_trends.py | 96 +++++++++++++++++++ .../test/__snapshots__/test_breakdowns.ambr | 12 +-- posthog/queries/trends/util.py | 2 +- 3 files changed, 103 insertions(+), 7 deletions(-) diff --git a/posthog/queries/test/test_trends.py b/posthog/queries/test/test_trends.py index f40373fe1cce8d..e0ada62331509a 100644 --- a/posthog/queries/test/test_trends.py +++ b/posthog/queries/test/test_trends.py @@ -392,6 +392,102 @@ def test_trends_with_session_property_single_aggregate_math(self): self.assertEqual(daily_response[0]["aggregated_value"], 7.5) self.assertEqual(daily_response[0]["aggregated_value"], weekly_response[0]["aggregated_value"]) + def test_unique_session_with_session_breakdown(self): + _create_person( + team_id=self.team.pk, distinct_ids=["blabla", "anonymous_id"], properties={"$some_prop": "some_val"} + ) + _create_person(team_id=self.team.pk, distinct_ids=["blabla2"], properties={"$some_prop": "some_val"}) + + _create_event( + team=self.team, + event="sign up before", + distinct_id="blabla", + properties={"$session_id": 1}, + timestamp="2020-01-01 00:06:30", + ) + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$session_id": 1}, + timestamp="2020-01-01 00:06:34", + ) + _create_event( + team=self.team, + event="sign up later", + distinct_id="blabla", + properties={"$session_id": 1}, + timestamp="2020-01-01 00:06:35", + ) + # First session lasted 5 seconds + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla2", + properties={"$session_id": 2}, + timestamp="2020-01-01 00:06:35", + ) + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla2", + properties={"$session_id": 2}, + timestamp="2020-01-01 00:06:45", + ) + # Second session lasted 10 seconds + + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$session_id": 3}, + timestamp="2020-01-01 00:06:45", + ) + # Third session lasted 0 seconds + + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$session_id": 4}, + timestamp="2020-01-02 00:06:30", + ) + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$session_id": 4}, + timestamp="2020-01-02 00:06:45", + ) + # Fourth session lasted 15 seconds + + with freeze_time("2020-01-04T13:00:01Z"): + response = trends().run( + Filter( + data={ + "display": "ActionsLineGraph", + "interval": "day", + "events": [{"id": "sign up", "math": "unique_session",}], + "breakdown": "$session_duration", + "breakdown_type": "session", + "insight": "TRENDS", + "breakdown_histogram_bin_count": 3, + "properties": [{"key": "$some_prop", "value": "some_val", "type": "person"}], + "date_from": "-3d", + } + ), + self.team, + ) + + self.assertEqual( + [(item["breakdown_value"], item["count"], item["data"]) for item in response], + [ + ("[0.0,4.95]", 1.0, [1.0, 0.0, 0.0, 0.0]), + ("[4.95,10.05]", 2.0, [2.0, 0.0, 0.0, 0.0]), + ("[10.05,15.01]", 1.0, [0.0, 1.0, 0.0, 0.0]), + ], + ) + @test_with_materialized_columns(person_properties=["name"], verify_no_jsonextract=False) def test_trends_breakdown_single_aggregate_cohorts(self): _create_person(team_id=self.team.pk, distinct_ids=["Jane"], properties={"name": "Jane"}) diff --git a/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr b/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr index e6982e9d78fab0..7d88aa56e686fe 100644 --- a/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr @@ -4,7 +4,7 @@ SELECT arrayCompact(arrayMap(x -> floor(x, 2), quantiles(0.00, 0.33, 0.67, 1.00)(value))) FROM (SELECT toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) AS value, - count(DISTINCT $session_id) as count + count(DISTINCT e.$session_id) as count FROM events e WHERE team_id = 2 AND event = 'watched movie' @@ -43,7 +43,7 @@ UNION ALL SELECT - count(DISTINCT $session_id) as total, + count(DISTINCT e.$session_id) as total, toStartOfDay(timestamp, 'UTC') as day_start, multiIf(toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) >= 25.0 AND toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) < 66.25,'[25.0,66.25]',toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) >= 66.25 AND toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) < 98.37,'[66.25,98.37]',toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) >= 98.37 AND toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) < 1000.01,'[98.37,1000.01]','["",""]') as breakdown_value FROM events e @@ -354,7 +354,7 @@ SELECT groupArray(value) FROM (SELECT sessions.session_duration AS value, - count(DISTINCT $session_id) as count + count(DISTINCT e.$session_id) as count FROM events e INNER JOIN (SELECT $session_id, @@ -402,7 +402,7 @@ JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start - UNION ALL SELECT count(DISTINCT $session_id) as total, + UNION ALL SELECT count(DISTINCT e.$session_id) as total, toStartOfDay(timestamp, 'UTC') as day_start, sessions.session_duration as breakdown_value FROM events e @@ -436,7 +436,7 @@ SELECT arrayCompact(arrayMap(x -> floor(x, 2), quantiles(0.00, 0.33, 0.67, 1.00)(value))) FROM (SELECT toFloat64OrNull(toString(sessions.session_duration)) AS value, - count(DISTINCT $session_id) as count + count(DISTINCT e.$session_id) as count FROM events e INNER JOIN (SELECT $session_id, @@ -484,7 +484,7 @@ UNION ALL SELECT - count(DISTINCT $session_id) as total, + count(DISTINCT e.$session_id) as total, toStartOfDay(timestamp, 'UTC') as day_start, multiIf(toFloat64OrNull(toString(sessions.session_duration)) >= 0.0 AND toFloat64OrNull(toString(sessions.session_duration)) < 69.92,'[0.0,69.92]',toFloat64OrNull(toString(sessions.session_duration)) >= 69.92 AND toFloat64OrNull(toString(sessions.session_duration)) < 110.72,'[69.92,110.72]',toFloat64OrNull(toString(sessions.session_duration)) >= 110.72 AND toFloat64OrNull(toString(sessions.session_duration)) < 180.01,'[110.72,180.01]','["",""]') as breakdown_value FROM events e diff --git a/posthog/queries/trends/util.py b/posthog/queries/trends/util.py index 3feb65e2ae357b..4e7e6adf3636e1 100644 --- a/posthog/queries/trends/util.py +++ b/posthog/queries/trends/util.py @@ -42,7 +42,7 @@ def process_math( aggregate_operation = f"count(DISTINCT $group_{entity.math_group_type_index})" elif entity.math == "unique_session": - aggregate_operation = "count(DISTINCT $session_id)" + aggregate_operation = f"count(DISTINCT {event_table_alias + '.' if event_table_alias else ''}$session_id)" elif entity.math in MATH_FUNCTIONS: if entity.math_property is None: raise ValidationError({"math_property": "This field is required when `math` is set."}, code="required") From e2f1ee37da957d7916e910a0f1a96ca043c09ef3 Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Fri, 8 Jul 2022 09:27:19 -0700 Subject: [PATCH 008/213] feat(session-analysis): format duration values (#10686) * feat(session-analysis): format duration values * fix tests --- .../components/PropertyFilterButton.tsx | 4 +- .../components/PropertyValue.tsx | 6 +- .../lib/components/SelectGradientOverflow.tsx | 4 +- .../models/propertyDefinitionsModel.test.ts | 53 ++++++++---- .../src/models/propertyDefinitionsModel.ts | 20 +++-- .../src/scenes/funnels/useFunnelTooltip.tsx | 7 +- .../InsightTooltip/InsightTooltip.tsx | 32 +++++--- .../InsightTooltip/insightTooltipUtils.tsx | 19 ++++- .../TaxonomicBreakdownButton.tsx | 7 +- .../TaxonomicBreakdownFilter.tsx | 27 ++++-- .../taxonomicBreakdownFilterUtils.test.ts | 25 ++++-- .../taxonomicBreakdownFilterUtils.ts | 16 +++- frontend/src/scenes/insights/utils.ts | 46 +++++++++++ .../views/Funnels/FunnelStepsTable.tsx | 6 +- .../views/InsightsTable/InsightsTable.tsx | 82 ++++++++++--------- 15 files changed, 247 insertions(+), 107 deletions(-) diff --git a/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx b/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx index 1a738d2204ca46..565830de6ee09a 100644 --- a/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx +++ b/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx @@ -21,12 +21,12 @@ export interface PropertyFilterButtonProps { export function PropertyFilterText({ item }: PropertyFilterButtonProps): JSX.Element { const { cohortsById } = useValues(cohortsModel) - const { formatForDisplay } = useValues(propertyDefinitionsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) return ( <> {formatPropertyLabel(item, cohortsById, keyMapping, (s) => - midEllipsis(formatForDisplay(item.key, s)?.toString() || '', 32) + midEllipsis(formatPropertyValueForDisplay(item.key, s)?.toString() || '', 32) )} ) diff --git a/frontend/src/lib/components/PropertyFilters/components/PropertyValue.tsx b/frontend/src/lib/components/PropertyFilters/components/PropertyValue.tsx index 63f7ac9f961e20..45cf6327fe3c4d 100644 --- a/frontend/src/lib/components/PropertyFilters/components/PropertyValue.tsx +++ b/frontend/src/lib/components/PropertyFilters/components/PropertyValue.tsx @@ -79,7 +79,7 @@ export function PropertyValue({ const [shouldBlur, setShouldBlur] = useState(false) const autoCompleteRef = useRef(null) - const { formatForDisplay, describeProperty } = useValues(propertyDefinitionsModel) + const { formatPropertyValueForDisplay, describeProperty } = useValues(propertyDefinitionsModel) const isMultiSelect = operator && isOperatorMulti(operator) const isDateTimeProperty = operator && isOperatorDate(operator) @@ -220,7 +220,7 @@ export function PropertyValue({ > {input && !displayOptions.some(({ name }) => input.toLowerCase() === toString(name).toLowerCase()) && ( - Specify: {formatForDisplay(propertyKey, input)} + Specify: {formatPropertyValueForDisplay(propertyKey, input)} )} {displayOptions.map(({ name: _name }, index) => { @@ -233,7 +233,7 @@ export function PropertyValue({ className="ph-no-capture" title={name} > - {name === '' ? (empty string) : formatForDisplay(propertyKey, name)} + {name === '' ? (empty string) : formatPropertyValueForDisplay(propertyKey, name)} ) })} diff --git a/frontend/src/lib/components/SelectGradientOverflow.tsx b/frontend/src/lib/components/SelectGradientOverflow.tsx index 8b594d761e1b7c..34d4b888884ce4 100644 --- a/frontend/src/lib/components/SelectGradientOverflow.tsx +++ b/frontend/src/lib/components/SelectGradientOverflow.tsx @@ -53,7 +53,7 @@ export function SelectGradientOverflow({ const containerRef: React.RefObject = useRef(null) const dropdownRef = useRef(null) const [isOpen, setOpen] = useState(false) - const { formatForDisplay } = useValues(propertyDefinitionsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) /** * Ant Design Tag with custom styling in .scss to match default style @@ -61,7 +61,7 @@ export function SelectGradientOverflow({ function CustomTag({ label, onClose, value }: CustomTagProps): JSX.Element { // if this component is displaying a list of PropertyFilterValues it needs to format them for display if (typeof label === 'string' && propertyKey) { - label = formatForDisplay(propertyKey, label) + label = formatPropertyValueForDisplay(propertyKey, label) } return ( diff --git a/frontend/src/models/propertyDefinitionsModel.test.ts b/frontend/src/models/propertyDefinitionsModel.test.ts index aa89a1c70db056..70cf8f1d542c4b 100644 --- a/frontend/src/models/propertyDefinitionsModel.test.ts +++ b/frontend/src/models/propertyDefinitionsModel.test.ts @@ -94,48 +94,71 @@ describe('the property definitions model', () => { }) it('can format a property with no formatting needs for display', () => { - expect(logic.values.formatForDisplay('a string', '1641368752.908')).toEqual('1641368752.908') + expect(logic.values.formatPropertyValueForDisplay('a string', '1641368752.908')).toEqual('1641368752.908') }) it('can format an unknown property for display', () => { - expect(logic.values.formatForDisplay('not a known property type', '1641368752.908')).toEqual('1641368752.908') + expect(logic.values.formatPropertyValueForDisplay('not a known property type', '1641368752.908')).toEqual( + '1641368752.908' + ) }) - it('can format an undefined property key for display', () => { - expect(logic.values.formatForDisplay(undefined, '1641368752.908')).toEqual('1641368752.908') + it('can format an null property key for display', () => { + expect(logic.values.formatPropertyValueForDisplay(null, '1641368752.908')).toEqual('1641368752.908') }) describe('formatting datetime properties', () => { it('can format a unix timestamp as seconds with fractional part for display', () => { - expect(logic.values.formatForDisplay('$timestamp', '1641368752.908')).toEqual('2022-01-05 07:45:52') + expect(logic.values.formatPropertyValueForDisplay('$timestamp', '1641368752.908')).toEqual( + '2022-01-05 07:45:52' + ) }) it('can format a unix timestamp as milliseconds for display', () => { - expect(logic.values.formatForDisplay('$timestamp', '1641368752908')).toEqual('2022-01-05 07:45:52') + expect(logic.values.formatPropertyValueForDisplay('$timestamp', '1641368752908')).toEqual( + '2022-01-05 07:45:52' + ) }) it('can format a unix timestamp as seconds for display', () => { - expect(logic.values.formatForDisplay('$timestamp', '1641368752')).toEqual('2022-01-05 07:45:52') + expect(logic.values.formatPropertyValueForDisplay('$timestamp', '1641368752')).toEqual( + '2022-01-05 07:45:52' + ) }) it('can format a date string for display', () => { - expect(logic.values.formatForDisplay('$timestamp', '2022-01-05')).toEqual('2022-01-05') + expect(logic.values.formatPropertyValueForDisplay('$timestamp', '2022-01-05')).toEqual('2022-01-05') }) it('can format a datetime string for display', () => { - expect(logic.values.formatForDisplay('$timestamp', '2022-01-05 07:45:52')).toEqual('2022-01-05 07:45:52') + expect(logic.values.formatPropertyValueForDisplay('$timestamp', '2022-01-05 07:45:52')).toEqual( + '2022-01-05 07:45:52' + ) }) it('can format an array of datetime string for display', () => { - expect(logic.values.formatForDisplay('$timestamp', ['1641368752.908', 1641368752.908])).toEqual([ - '2022-01-05 07:45:52', - '2022-01-05 07:45:52', - ]) + expect( + logic.values.formatPropertyValueForDisplay('$timestamp', ['1641368752.908', 1641368752.908]) + ).toEqual(['2022-01-05 07:45:52', '2022-01-05 07:45:52']) + }) + }) + + describe('formatting duration properties', () => { + it('can format a number to duration', () => { + expect(logic.values.formatPropertyValueForDisplay('$session_duration', 60)).toEqual('00:01:00') + }) + + it('can format a string to duration', () => { + expect(logic.values.formatPropertyValueForDisplay('$session_duration', '60')).toEqual('00:01:00') + }) + + it('handles non numbers', () => { + expect(logic.values.formatPropertyValueForDisplay('$session_duration', 'blah')).toEqual('blah') }) }) it('can format a null value for display', () => { - expect(logic.values.formatForDisplay('$timestamp', null)).toEqual(null) - expect(logic.values.formatForDisplay('$timestamp', undefined)).toEqual(null) + expect(logic.values.formatPropertyValueForDisplay('$timestamp', null)).toEqual(null) + expect(logic.values.formatPropertyValueForDisplay('$timestamp', undefined)).toEqual(null) }) }) diff --git a/frontend/src/models/propertyDefinitionsModel.ts b/frontend/src/models/propertyDefinitionsModel.ts index dd13e54158b5df..589b19a46e08bf 100644 --- a/frontend/src/models/propertyDefinitionsModel.ts +++ b/frontend/src/models/propertyDefinitionsModel.ts @@ -1,9 +1,10 @@ import { kea } from 'kea' import api from 'lib/api' -import { PropertyDefinition, PropertyFilterValue, PropertyType, SelectOption } from '~/types' +import { BreakdownKeyType, PropertyDefinition, PropertyFilterValue, PropertyType, SelectOption } from '~/types' import type { propertyDefinitionsModelType } from './propertyDefinitionsModelType' import { dayjs } from 'lib/dayjs' import { TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' +import { colonDelimitedDuration } from 'lib/utils' export interface PropertySelectOption extends SelectOption { is_numerical?: boolean @@ -15,6 +16,8 @@ export interface PropertyDefinitionStorage { results: PropertyDefinition[] } +// List of property definitions that are calculated on the backend. These +// are valid properties that do not exist on events. const localPropertyDefinitions: PropertyDefinition[] = [ { id: '$session_duration', @@ -39,9 +42,9 @@ const normaliseToArray = ( } } -export type FormatForDisplayFunction = ( - propertyName: string | undefined, - valueToFormat: PropertyFilterValue | undefined +export type FormatPropertyValueForDisplayFunction = ( + propertyName?: BreakdownKeyType, + valueToFormat?: PropertyFilterValue ) => string | string[] | null export const propertyDefinitionsModel = kea({ @@ -142,10 +145,10 @@ export const propertyDefinitionsModel = kea({ return propertyDefinitions.find((pd) => pd.name === propertyName) ?? null }, ], - formatForDisplay: [ + formatPropertyValueForDisplay: [ (s) => [s.propertyDefinitions], - (propertyDefinitions: PropertyDefinition[]): FormatForDisplayFunction => { - return (propertyName: string | undefined, valueToFormat: PropertyFilterValue | undefined) => { + (propertyDefinitions: PropertyDefinition[]): FormatPropertyValueForDisplayFunction => { + return (propertyName?: BreakdownKeyType, valueToFormat?: PropertyFilterValue | undefined) => { if (valueToFormat === null || valueToFormat === undefined) { return null } @@ -172,6 +175,9 @@ export const propertyDefinitionsModel = kea({ const numericalTimestamp = Number.parseInt(propertyValue) return dayjs(numericalTimestamp).tz().format('YYYY-MM-DD hh:mm:ss') } + } else if (propertyDefinition?.property_type === PropertyType.Duration) { + const numericalDuration = Number.parseFloat(propertyValue) + return isNaN(numericalDuration) ? propertyValue : colonDelimitedDuration(numericalDuration) } return propertyValue diff --git a/frontend/src/scenes/funnels/useFunnelTooltip.tsx b/frontend/src/scenes/funnels/useFunnelTooltip.tsx index ff6014b0fa4aee..7081e38961d164 100644 --- a/frontend/src/scenes/funnels/useFunnelTooltip.tsx +++ b/frontend/src/scenes/funnels/useFunnelTooltip.tsx @@ -11,9 +11,10 @@ import { humanFriendlyDuration, humanFriendlyNumber, percentage } from 'lib/util import { ensureTooltipElement } from 'scenes/insights/views/LineGraph/LineGraph' import { LemonDivider } from 'lib/components/LemonDivider' import { cohortsModel } from '~/models/cohortsModel' -import { formatBreakdownLabel } from 'scenes/insights/views/InsightsTable/InsightsTable' import { ClickToInspectActors } from 'scenes/insights/InsightTooltip/InsightTooltip' import { groupsModel } from '~/models/groupsModel' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' +import { formatBreakdownLabel } from 'scenes/insights/utils' /** The tooltip is offset horizontally by a few pixels from the bar to give it some breathing room. */ const FUNNEL_TOOLTIP_OFFSET_PX = 2 @@ -27,7 +28,7 @@ interface FunnelTooltipProps { function FunnelTooltip({ showPersonsModal, stepIndex, series, groupTypeLabel }: FunnelTooltipProps): JSX.Element { const { cohorts } = useValues(cohortsModel) - + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) return (
} fullWidth> @@ -36,7 +37,7 @@ function FunnelTooltip({ showPersonsModal, stepIndex, series, groupTypeLabel }: filter={getActionFilterFromFunnelStep(series)} style={{ display: 'inline-block' }} />{' '} - • {formatBreakdownLabel(cohorts, series.breakdown_value)} + • {formatBreakdownLabel(cohorts, formatPropertyValueForDisplay, series.breakdown_value)} diff --git a/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.tsx b/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.tsx index 7833cb665d05e0..386c726fb00608 100644 --- a/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.tsx +++ b/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.tsx @@ -16,6 +16,8 @@ import { SeriesLetter } from 'lib/components/SeriesGlyph' import { IconHandClick } from 'lib/components/icons' import { shortTimeZone } from 'lib/utils' import { humanFriendlyNumber } from 'lib/utils' +import { useValues } from 'kea' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' export function ClickToInspectActors({ isTruncated, @@ -51,9 +53,9 @@ export function InsightTooltip({ {value} ), - renderCount = (value: number | React.ReactNode) => ( - <>{typeof value === 'number' ? humanFriendlyNumber(value) : value} - ), + renderCount = (value: number) => { + return <>{typeof value === 'number' ? humanFriendlyNumber(value) : value} + }, hideColorCol = false, hideInspectActorsSection = false, forceEntitiesAsColumns = false, @@ -70,11 +72,12 @@ export function InsightTooltip({ ((seriesData?.length ?? 0) > 1 && (seriesData?.[0]?.breakdown_value !== undefined || seriesData?.[0]?.compare_label !== undefined)) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) + const title: ReactNode | null = getTooltipTitle(seriesData, altTitle, date) || `${getFormattedDate(date, seriesData?.[0]?.filter?.interval)} (${shortTimeZone(timezone)})` const rightTitle: ReactNode | null = getTooltipTitle(seriesData, altRightTitle, date) || null - const renderTable = (): JSX.Element => { if (itemizeEntitiesAsColumns) { const dataSource = invertDataSource(seriesData) @@ -120,11 +123,14 @@ export function InsightTooltip({ colIdx )), render: function renderSeriesColumnData(_, datum) { - return ( -
- {renderCount(datum.seriesData?.[colIdx]?.count ?? 0, datum, colIdx)} -
- ) + const innerValue = datum.seriesData?.[colIdx]?.action?.math_property + ? formatPropertyValueForDisplay( + datum.seriesData?.[colIdx]?.action?.math_property, + datum.seriesData?.[colIdx]?.count + ) + : renderCount(datum.seriesData?.[colIdx]?.count ?? 0) + + return
{innerValue}
}, }) }) @@ -182,8 +188,12 @@ export function InsightTooltip({ width: 50, title: {rightTitle ?? undefined}, align: 'right', - render: function renderDatum(_, datum, rowIdx) { - return
{renderCount(datum?.count ?? 0, datum, rowIdx)}
+ render: function renderDatum(_, datum) { + const innerValue = datum.action?.math_property + ? formatPropertyValueForDisplay(datum.action?.math_property, datum.count) + : renderCount(datum.count ?? 0) + + return
{innerValue}
}, }) diff --git a/frontend/src/scenes/insights/InsightTooltip/insightTooltipUtils.tsx b/frontend/src/scenes/insights/InsightTooltip/insightTooltipUtils.tsx index 8df3e3399c6e88..b4fd96e49bfb11 100644 --- a/frontend/src/scenes/insights/InsightTooltip/insightTooltipUtils.tsx +++ b/frontend/src/scenes/insights/InsightTooltip/insightTooltipUtils.tsx @@ -3,6 +3,10 @@ import React from 'react' import { ActionFilter, CompareLabelType, FilterType, IntervalType } from '~/types' import { Space, Tag, Typography } from 'antd' import { capitalizeFirstLetter, midEllipsis, pluralize } from 'lib/utils' +import { cohortsModel } from '~/models/cohortsModel' +import { useValues } from 'kea' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' +import { formatBreakdownLabel } from '../utils' export interface SeriesDatum { id: number // determines order that series will be displayed in @@ -34,7 +38,7 @@ export interface TooltipConfig { rowCutoff?: number colCutoff?: number renderSeries?: (value: React.ReactNode, seriesDatum: SeriesDatum, idx: number) => React.ReactNode - renderCount?: (value: number, seriesDatum: SeriesDatum | InvertedSeriesDatum, idx: number) => React.ReactNode + renderCount?: (value: number) => React.ReactNode showHeader?: boolean hideColorCol?: boolean } @@ -81,12 +85,23 @@ export function getFormattedDate(dayInput?: string | number, interval?: Interval } export function invertDataSource(seriesData: SeriesDatum[]): InvertedSeriesDatum[] { + const { cohorts } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) const flattenedData: Record = {} seriesData.forEach((s) => { let datumTitle const pillValues = [] if (s.breakdown_value !== undefined) { - pillValues.push(String(s.breakdown_value || 'None')) + pillValues.push( + formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + s.breakdown_value, + s.filter.breakdown, + s.filter.breakdown_type, + s.filter.breakdown_histogram_bin_count !== undefined + ) + ) } if (s.compare_label) { pillValues.push(capitalizeFirstLetter(String(s.compare_label))) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownButton.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownButton.tsx index f8cc0a28475551..8be48203520914 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownButton.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownButton.tsx @@ -10,26 +10,25 @@ import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' import { useValues } from 'kea' import { groupsModel } from '~/models/groupsModel' import { insightLogic } from 'scenes/insights/insightLogic' -import { FEATURE_FLAGS } from 'lib/constants' import { LemonButton } from '@posthog/lemon-ui' import { IconPlusMini } from 'lib/components/icons' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' export interface TaxonomicBreakdownButtonProps { breakdownType?: TaxonomicFilterGroupType onChange: (breakdown: TaxonomicFilterValue, taxonomicGroup: TaxonomicFilterGroup) => void onlyCohorts?: boolean + includeSessions?: boolean } export function TaxonomicBreakdownButton({ breakdownType, onChange, onlyCohorts, + includeSessions, }: TaxonomicBreakdownButtonProps): JSX.Element { const [open, setOpen] = useState(false) const { allEventNames } = useValues(insightLogic) const { groupsTaxonomicTypes } = useValues(groupsModel) - const { featureFlags } = useValues(featureFlagLogic) const taxonomicGroupTypes = onlyCohorts ? [TaxonomicFilterGroupType.CohortsWithAllUsers] @@ -38,7 +37,7 @@ export function TaxonomicBreakdownButton({ TaxonomicFilterGroupType.PersonProperties, ...groupsTaxonomicTypes, TaxonomicFilterGroupType.CohortsWithAllUsers, - ].concat(featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS] ? [TaxonomicFilterGroupType.Sessions] : []) + ].concat(includeSessions ? [TaxonomicFilterGroupType.Sessions] : []) return ( getPropertyDefinition(t)?.is_numerical) - : false - const tags = !breakdown_type ? [] : breakdownArray.map((t, index) => { const key = `${t}-${index}` + const isPropertyHistogramable = featureFlags[FEATURE_FLAGS.HISTOGRAM_INSIGHTS] + ? !useMultiBreakdown && !!getPropertyDefinition(t)?.is_numerical + : false return ( {tags} {onChange && (!hasSelectedBreakdown || useMultiBreakdown) ? ( - + ) : null}
) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts index baf2b42c9dc09a..4c6ea47bb3faa2 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts @@ -16,6 +16,8 @@ const taxonomicGroupFor = ( const setFilters = jest.fn() +const getPropertyDefinition = jest.fn() + describe('taxonomic breakdown filter utils', () => { describe('with multi property breakdown flag on', () => { it('sets breakdowns for events', () => { @@ -23,7 +25,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: true, breakdownParts: ['a', 'b'], setFilters, - isHistogramable: false, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = 'c' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.EventProperties) @@ -47,7 +50,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: true, breakdownParts: ['all', 1], setFilters, - isHistogramable: false, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = 2 const group: TaxonomicFilterGroup = taxonomicGroupFor( @@ -74,7 +78,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: true, breakdownParts: ['country'], setFilters, - isHistogramable: false, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = 'height' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.PersonProperties, undefined) @@ -101,7 +106,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: false, breakdownParts: ['a', 'b'], setFilters, - isHistogramable: true, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = 'c' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.EventProperties, undefined) @@ -111,7 +117,7 @@ describe('taxonomic breakdown filter utils', () => { breakdown: 'c', breakdowns: undefined, breakdown_group_type_index: undefined, - breakdown_histogram_bin_count: 10, + breakdown_histogram_bin_count: undefined, }) }) @@ -120,7 +126,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: false, breakdownParts: ['all', 1], setFilters, - isHistogramable: false, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = 2 const group: TaxonomicFilterGroup = taxonomicGroupFor( @@ -141,7 +148,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: false, breakdownParts: ['country'], setFilters, - isHistogramable: false, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = 'height' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.PersonProperties, undefined) @@ -159,7 +167,8 @@ describe('taxonomic breakdown filter utils', () => { useMultiBreakdown: false, breakdownParts: ['$lib'], setFilters, - isHistogramable: false, + getPropertyDefinition, + histogramFeatureFlag: false, }) const changedBreakdown = '$lib_version' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.GroupsPrefix, 0) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts index 428add9b77d958..f4c34dc42ea15c 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts @@ -1,4 +1,4 @@ -import { BreakdownType, FilterType } from '~/types' +import { BreakdownType, FilterType, PropertyDefinition } from '~/types' import { TaxonomicFilterGroup, TaxonomicFilterGroupType, @@ -10,14 +10,24 @@ interface FilterChange { useMultiBreakdown: string | boolean | undefined breakdownParts: (string | number)[] setFilters: (filters: Partial, mergeFilters?: boolean) => void - isHistogramable: boolean + getPropertyDefinition: (propertyName: string | number) => PropertyDefinition | null + histogramFeatureFlag: boolean } -export function onFilterChange({ useMultiBreakdown, breakdownParts, setFilters, isHistogramable }: FilterChange) { +export function onFilterChange({ + useMultiBreakdown, + breakdownParts, + setFilters, + getPropertyDefinition, + histogramFeatureFlag, +}: FilterChange) { return (changedBreakdown: TaxonomicFilterValue, taxonomicGroup: TaxonomicFilterGroup): void => { const changedBreakdownType = taxonomicFilterTypeToPropertyFilterType(taxonomicGroup.type) as BreakdownType if (changedBreakdownType) { + const isHistogramable = + histogramFeatureFlag && !useMultiBreakdown && !!getPropertyDefinition(changedBreakdown)?.is_numerical + const newFilters: Partial = { breakdown_type: changedBreakdownType, breakdown_group_type_index: taxonomicGroup.groupTypeIndex, diff --git a/frontend/src/scenes/insights/utils.ts b/frontend/src/scenes/insights/utils.ts index 271d535f1fd696..41ed8409f46cb9 100644 --- a/frontend/src/scenes/insights/utils.ts +++ b/frontend/src/scenes/insights/utils.ts @@ -1,5 +1,8 @@ import { ActionFilter, + BreakdownKeyType, + BreakdownType, + CohortType, EntityFilter, FilterType, FunnelVizType, @@ -24,6 +27,7 @@ import { mathsLogicType } from 'scenes/trends/mathsLogicType' import { apiValueToMathType, MathDefinition } from 'scenes/trends/mathsLogic' import { dashboardsModel } from '~/models/dashboardsModel' import { insightLogic } from './insightLogic' +import { FormatPropertyValueForDisplayFunction } from '~/models/propertyDefinitionsModel' export const getDisplayNameFromEntityFilter = ( filter: EntityFilter | ActionFilter | null, @@ -294,3 +298,45 @@ export function summarizeInsightFilters( } return summary } + +export function formatBreakdownLabel( + cohorts?: CohortType[], + formatPropertyValueForDisplay?: FormatPropertyValueForDisplayFunction, + breakdown_value?: BreakdownKeyType, + breakdown?: BreakdownKeyType, + breakdown_type?: BreakdownType | null, + isHistogram?: boolean +): string { + if (isHistogram && typeof breakdown_value === 'string') { + const [bucketStart, bucketEnd] = JSON.parse(breakdown_value) + const formattedBucketStart = formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + bucketStart, + breakdown, + breakdown_type + ) + const formattedBucketEnd = formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + bucketEnd, + breakdown, + breakdown_type + ) + return `${formattedBucketStart} – ${formattedBucketEnd}` + } + if (typeof breakdown_value == 'number') { + if (breakdown_type === 'cohort') { + return cohorts?.filter((c) => c.id == breakdown_value)[0]?.name ?? breakdown_value.toString() + } + return formatPropertyValueForDisplay + ? formatPropertyValueForDisplay(breakdown, breakdown_value)?.toString() ?? 'None' + : breakdown_value.toString() + } else if (typeof breakdown_value == 'string') { + return breakdown_value === 'nan' ? 'Other' : breakdown_value === '' ? 'None' : breakdown_value + } else if (Array.isArray(breakdown_value)) { + return breakdown_value.join('::') + } else { + return '' + } +} diff --git a/frontend/src/scenes/insights/views/Funnels/FunnelStepsTable.tsx b/frontend/src/scenes/insights/views/Funnels/FunnelStepsTable.tsx index 05841dd09d81cd..b06c3dac113254 100644 --- a/frontend/src/scenes/insights/views/Funnels/FunnelStepsTable.tsx +++ b/frontend/src/scenes/insights/views/Funnels/FunnelStepsTable.tsx @@ -7,7 +7,6 @@ import { BreakdownKeyType, FlattenedFunnelStepByBreakdown } from '~/types' import { EntityFilterInfo } from 'lib/components/EntityFilterInfo' import { getVisibilityIndex } from 'scenes/funnels/funnelUtils' import { getActionFilterFromFunnelStep, getSignificanceFromBreakdownStep } from './funnelStepTableUtils' -import { formatBreakdownLabel } from 'scenes/insights/views/InsightsTable/InsightsTable' import { cohortsModel } from '~/models/cohortsModel' import { LemonCheckbox } from 'lib/components/LemonCheckbox' import { Lettermark, LettermarkColor } from 'lib/components/Lettermark/Lettermark' @@ -16,6 +15,8 @@ import { humanFriendlyDuration, humanFriendlyNumber, percentage } from 'lib/util import { ValueInspectorButton } from 'scenes/funnels/ValueInspectorButton' import { getSeriesColor } from 'lib/colors' import { IconFlag } from 'lib/components/icons' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' +import { formatBreakdownLabel } from 'scenes/insights/utils' export function FunnelStepsTable(): JSX.Element | null { const { insightProps } = useValues(insightLogic) @@ -24,6 +25,7 @@ export function FunnelStepsTable(): JSX.Element | null { useValues(logic) const { setHiddenById, toggleVisibilityByBreakdown, openPersonsModalForSeries } = useActions(logic) const { cohorts } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) const isOnlySeries = flattenedBreakdowns.length === 1 const allChecked = flattenedBreakdowns?.every( @@ -71,7 +73,7 @@ export function FunnelStepsTable(): JSX.Element | null { ), dataIndex: 'breakdown_value', render: function RenderBreakdownValue(breakdownValue: BreakdownKeyType | undefined): JSX.Element { - const label = formatBreakdownLabel(cohorts, breakdownValue) + const label = formatBreakdownLabel(cohorts, formatPropertyValueForDisplay, breakdownValue) return isOnlySeries ? ( {label} ) : ( diff --git a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx index 064b3ec8209cc6..71d0fdfcc9018b 100644 --- a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx +++ b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx @@ -1,17 +1,16 @@ import React from 'react' import { Dropdown, Menu } from 'antd' -import { Tooltip } from 'lib/components/Tooltip' import { BindLogic, useActions, useValues } from 'kea' import { trendsLogic } from 'scenes/trends/trendsLogic' import { LemonCheckbox } from 'lib/components/LemonCheckbox' import { getSeriesColor } from 'lib/colors' import { cohortsModel } from '~/models/cohortsModel' -import { BreakdownKeyType, ChartDisplayType, CohortType, IntervalType, TrendResult } from '~/types' +import { ChartDisplayType, IntervalType, TrendResult } from '~/types' import { average, median, capitalizeFirstLetter, humanFriendlyNumber } from 'lib/utils' import { InsightLabel } from 'lib/components/InsightLabel' import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' import { CalcColumnState, insightsTableLogic } from './insightsTableLogic' -import { DownOutlined, InfoCircleOutlined } from '@ant-design/icons' +import { DownOutlined } from '@ant-design/icons' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { DateDisplay } from 'lib/components/DateDisplay' import { SeriesToggleWrapper } from './components/SeriesToggleWrapper' @@ -26,6 +25,8 @@ import { LemonButton } from 'lib/components/LemonButton' import { IconEdit } from 'lib/components/icons' import { countryCodeToName } from '../WorldMap' import { NON_TIME_SERIES_DISPLAY_TYPES } from 'lib/constants' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' +import { formatBreakdownLabel } from 'scenes/insights/utils' interface InsightsTableProps { /** Whether this is just a legend instead of standalone insight viz. Default: false. */ @@ -73,6 +74,8 @@ export function InsightsTable({ const { indexedResults, hiddenLegendKeys, filters, resultsLoading } = useValues(trendsLogic(insightProps)) const { toggleVisibility, setFilters } = useActions(trendsLogic(insightProps)) const { cohorts } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) + const { reportInsightsTableCalcToggled } = useActions(eventUsageLogic) const hasMathUniqueFilter = !!( @@ -86,14 +89,14 @@ export function InsightsTable({ const handleEditClick = (item: IndexedTrendResult): void => { if (canEditSeriesNameInline) { - const entityFitler = entityFilterLogic.findMounted({ + const entityFilter = entityFilterLogic.findMounted({ setFilters, filters, typeKey: filterKey, }) - if (entityFitler) { - entityFitler.actions.selectFilter(item.action) - entityFitler.actions.showModal() + if (entityFilter) { + entityFilter.actions.selectFilter(item.action) + entityFilter.actions.showModal() } } } @@ -182,7 +185,14 @@ export function InsightsTable({ ), render: function RenderBreakdownValue(_, item: IndexedTrendResult) { - const breakdownLabel = formatBreakdownLabel(cohorts, item.breakdown_value) + const breakdownLabel = formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + item.breakdown_value, + item.filter?.breakdown, + item.filter?.breakdown_type, + item.filter?.breakdown_histogram_bin_count !== undefined + ) return ( { - const labelA = formatBreakdownLabel(cohorts, a.breakdown_value) - const labelB = formatBreakdownLabel(cohorts, b.breakdown_value) + const labelA = formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + a.breakdown_value, + a.filter?.breakdown, + a.filter?.breakdown_type, + a.filter?.breakdown_histogram_bin_count !== undefined + ) + const labelB = formatBreakdownLabel( + cohorts, + formatPropertyValueForDisplay, + b.breakdown_value, + b.filter?.breakdown, + b.filter?.breakdown_type, + a.filter?.breakdown_histogram_bin_count !== undefined + ) return labelA.localeCompare(labelB) }, }) @@ -228,7 +252,9 @@ export function InsightsTable({ /> ), render: function RenderPeriod(_, item: IndexedTrendResult) { - return humanFriendlyNumber(item.data[index] ?? NaN) + return item.action.math_property + ? formatPropertyValueForDisplay(item.action.math_property, item.data[index]) + : humanFriendlyNumber(item.data[index] ?? NaN) }, key: `data-${index}`, sorter: (a, b) => (a.data[index] ?? NaN) - (b.data[index] ?? NaN), @@ -251,24 +277,18 @@ export function InsightsTable({ ) : ( CALC_COLUMN_LABELS.total ), - render: function RenderCalc(count: any, item: IndexedTrendResult) { + render: function RenderCalc(_: any, item: IndexedTrendResult) { + let value if (calcColumnState === 'total' || isDisplayModeNonTimeSeries) { - return (item.count || item.aggregated_value || 'Unknown').toLocaleString() + value = item.count || item.aggregated_value } else if (calcColumnState === 'average') { - return average(item.data).toLocaleString() + value = average(item.data) } else if (calcColumnState === 'median') { - return median(item.data).toLocaleString() + value = median(item.data) } - return ( - <> - {count?.toLocaleString?.()} - {item.action && item.action?.math === 'dau' && ( - - - - )} - - ) + return item.action.math_property + ? formatPropertyValueForDisplay(item.action.math_property, value) + : (value ?? 'Unknown').toLocaleString() }, sorter: (a, b) => (a.count || a.aggregated_value) - (b.count || b.aggregated_value), dataIndex: 'count', @@ -292,18 +312,6 @@ export function InsightsTable({ ) } -export function formatBreakdownLabel(cohorts?: CohortType[], breakdown_value?: BreakdownKeyType): string { - if (breakdown_value && typeof breakdown_value == 'number') { - return cohorts?.filter((c) => c.id == breakdown_value)[0]?.name || breakdown_value.toString() - } else if (typeof breakdown_value == 'string') { - return breakdown_value === 'nan' ? 'Other' : breakdown_value === '' ? 'None' : breakdown_value - } else if (Array.isArray(breakdown_value)) { - return breakdown_value.join('::') - } else { - return '' - } -} - export function formatCompareLabel(trendResult: TrendResult): string { // label splitting ensures backwards compatibility for api results that don't contain the new compare_label const labels = trendResult.label.split(' - ') From f4f178e859855a42ff3878242eb9899006721132 Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Fri, 8 Jul 2022 12:49:58 -0700 Subject: [PATCH 009/213] fix(session-analysis): add sessions to the glabal property filters on trends and fix column disambiguation (#10700) * fix(session-analysis): add sessions to the glabal property filters on trends * fix more session_id disambiguation * add test * move to array deconstruction --- .../test/__snapshots__/test_event_query.ambr | 23 ++++++++ .../queries/test/test_event_query.py | 56 +++++++++++++++++++ .../TrendsGlobalAndOrFilters.tsx | 20 ++++--- posthog/queries/trends/trend_event_query.py | 4 +- 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr b/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr index 6686c769627c3a..b64e9a18312129 100644 --- a/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr +++ b/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr @@ -332,3 +332,26 @@ AND team_id = 2)) ' --- +# name: TestEventQuery.test_unique_session_math_filtered_by_session_duration + ' + + SELECT e.timestamp as timestamp, + e."$session_id" as "$session_id", + sessions.session_duration as session_duration + FROM events e + INNER JOIN + (SELECT $session_id, + dateDiff('second', min(timestamp), max(timestamp)) as session_duration + FROM events + WHERE $session_id != '' + AND team_id = 2 + AND timestamp >= toDateTime('2021-05-02 00:00:00') - INTERVAL 24 HOUR + AND timestamp <= toDateTime('2021-05-03 00:00:00') + INTERVAL 24 HOUR + GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id + WHERE team_id = 2 + AND event = '$pageview' + AND timestamp >= toDateTime('2021-05-02 00:00:00') + AND timestamp <= toDateTime('2021-05-03 00:00:00') + AND (sessions.session_duration > 30.0) + ' +--- diff --git a/ee/clickhouse/queries/test/test_event_query.py b/ee/clickhouse/queries/test/test_event_query.py index bd4c0ae32a025e..0e97795397a08a 100644 --- a/ee/clickhouse/queries/test/test_event_query.py +++ b/ee/clickhouse/queries/test/test_event_query.py @@ -558,3 +558,59 @@ def test_entity_filtered_by_multiple_session_duration_filters(self): results, _ = self._run_query(filter) self.assertEqual(len(results), 1) self.assertEqual(results[0][0].strftime("%Y-%m-%d %H:%M:%S"), event_timestamp_str) + + @snapshot_clickhouse_queries + def test_unique_session_math_filtered_by_session_duration(self): + + filter = Filter( + data={ + "date_from": "2021-05-02 00:00:00", + "date_to": "2021-05-03 00:00:00", + "events": [ + { + "id": "$pageview", + "math": "unique_session", + "order": 0, + "properties": [{"key": "$session_duration", "type": "session", "operator": "gt", "value": 30},], + }, + ], + } + ) + + event_timestamp_str = "2021-05-02 00:01:00" + + # Session that should be returned + _create_event( + team=self.team, + event="start", + distinct_id="p1", + timestamp="2021-05-02 00:00:00", + properties={"$session_id": "1abc"}, + ) + _create_event( + team=self.team, + event="$pageview", + distinct_id="p1", + timestamp=event_timestamp_str, + properties={"$session_id": "1abc"}, + ) + + # Session that's too short + _create_event( + team=self.team, + event="$pageview", + distinct_id="p2", + timestamp="2021-05-02 00:02:00", + properties={"$session_id": "2abc"}, + ) + _create_event( + team=self.team, + event="final_event", + distinct_id="p2", + timestamp="2021-05-02 00:02:01", + properties={"$session_id": "2abc"}, + ) + + results, _ = self._run_query(filter) + self.assertEqual(len(results), 1) + self.assertEqual(results[0][0].strftime("%Y-%m-%d %H:%M:%S"), event_timestamp_str) diff --git a/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx b/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx index 838834959d0412..f2313eab351b33 100644 --- a/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx +++ b/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx @@ -7,25 +7,31 @@ import { useActions, useValues } from 'kea' import { trendsLogic } from 'scenes/trends/trendsLogic' import { groupsModel } from '~/models/groupsModel' import { insightLogic } from 'scenes/insights/insightLogic' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { FEATURE_FLAGS } from 'lib/constants' export function TrendsGlobalAndOrFilters({ filters, insightProps }: EditorFilterProps): JSX.Element { const { setFilters } = useActions(trendsLogic(insightProps)) const { allEventNames } = useValues(insightLogic) const { groupsTaxonomicTypes } = useValues(groupsModel) + const { featureFlags } = useValues(featureFlagLogic) + + const taxonomicGroupTypes = [ + TaxonomicFilterGroupType.EventProperties, + TaxonomicFilterGroupType.PersonProperties, + ...groupsTaxonomicTypes, + TaxonomicFilterGroupType.Cohorts, + TaxonomicFilterGroupType.Elements, + ...(featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS] ? [TaxonomicFilterGroupType.Sessions] : []), + ] return ( setFilters({ properties })} - taxonomicGroupTypes={[ - TaxonomicFilterGroupType.EventProperties, - TaxonomicFilterGroupType.PersonProperties, - ...groupsTaxonomicTypes, - TaxonomicFilterGroupType.Cohorts, - TaxonomicFilterGroupType.Elements, - ]} + taxonomicGroupTypes={taxonomicGroupTypes} pageKey="insight-filters" eventNames={allEventNames} filters={filters} diff --git a/posthog/queries/trends/trend_event_query.py b/posthog/queries/trends/trend_event_query.py index fd25e54b35635a..c8894802ee0c3f 100644 --- a/posthog/queries/trends/trend_event_query.py +++ b/posthog/queries/trends/trend_event_query.py @@ -46,7 +46,9 @@ def get_query(self) -> Tuple[str, Dict[str, Any]]: ) + ( f", {self.SESSION_TABLE_ALIAS}.$session_id as $session_id" - if self._should_join_sessions and "$session_id" not in self._extra_event_properties + if self._should_join_sessions + and "$session_id" not in self._extra_event_properties + and "$session_id" not in self._column_optimizer.event_columns_to_query else "" ) + (f", {self.EVENT_TABLE_ALIAS}.distinct_id as distinct_id" if self._aggregate_users_by_distinct_id else "") From 5b22dc819a7b949e4fe48baf551eee84fae620e5 Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Fri, 8 Jul 2022 16:04:10 -0700 Subject: [PATCH 010/213] chore(session-analysis): add $session_duration property key info (#10705) * chore(session-analysis): add $session_duration property key info * message about format * little bug fix --- frontend/src/lib/components/PropertyKeyInfo.tsx | 16 ++++++++++++++++ .../views/InsightsTable/InsightsTable.tsx | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/PropertyKeyInfo.tsx b/frontend/src/lib/components/PropertyKeyInfo.tsx index 30ae965178d3eb..4651e69c656c7d 100644 --- a/frontend/src/lib/components/PropertyKeyInfo.tsx +++ b/frontend/src/lib/components/PropertyKeyInfo.tsx @@ -508,6 +508,22 @@ export const keyMapping: KeyMappingInterface = { label: 'Subdivision 3 Code', description: `Code of the third subdivision matched to this event's IP address.`, }, + // NOTE: This is a hack. $session_duration is a session property, not an event property + // but we don't do a good job of tracking property types, so making it a session property + // would require a large refactor, and this works (because all properties are treated as + // event properties if they're not elements) + $session_duration: { + label: 'Session duration', + description: ( + + The duration of the session being tracked. Learn more about how PostHog tracks sessions in{' '} + our documentation. +

+ Note, if the duration is formatted as a single number (not 'HH:MM:SS'), it's in seconds. + + ), + examples: ['01:04:12'], + }, }, element: { tag_name: { diff --git a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx index 71d0fdfcc9018b..bf2641474f158d 100644 --- a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx +++ b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx @@ -252,7 +252,7 @@ export function InsightsTable({ /> ), render: function RenderPeriod(_, item: IndexedTrendResult) { - return item.action.math_property + return item.action?.math_property ? formatPropertyValueForDisplay(item.action.math_property, item.data[index]) : humanFriendlyNumber(item.data[index] ?? NaN) }, @@ -286,8 +286,8 @@ export function InsightsTable({ } else if (calcColumnState === 'median') { value = median(item.data) } - return item.action.math_property - ? formatPropertyValueForDisplay(item.action.math_property, value) + return item.action?.math_property + ? formatPropertyValueForDisplay(item.action?.math_property, value) : (value ?? 'Unknown').toLocaleString() }, sorter: (a, b) => (a.count || a.aggregated_value) - (b.count || b.aggregated_value), From 45e1d6c90c78389c7f6e0e3d922f5a6bbdeb87e3 Mon Sep 17 00:00:00 2001 From: "posthog-contributions-bot[bot]" <80958034+posthog-contributions-bot[bot]@users.noreply.github.com> Date: Sun, 10 Jul 2022 19:34:19 +0200 Subject: [PATCH 011/213] =?UTF-8?q?chore(contributors):=20=F0=9F=A4=96=20-?= =?UTF-8?q?=20Add=20hakubo=20as=20a=20contributor=20=F0=9F=8E=89=20(#10707?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .all-contributorsrc | 9 +++++++++ README.md | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index a402aef6866ced..185d1aece9bc08 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1926,6 +1926,15 @@ "contributions": [ "code" ] + }, + { + "login": "hakubo", + "name": "Jakub Olek", + "avatar_url": "https://avatars.githubusercontent.com/u/1018759?v=4", + "profile": "https://github.com/hakubo", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 61e46b3bdcc73e..030156160be64e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- + PRs Welcome Join Slack Community @@ -104,7 +104,7 @@ Premium features (contained in the `ee` directory) require a PostHog license. Co - + From 46e9fee496a7677a490bef01dfe87d8374863fef Mon Sep 17 00:00:00 2001 From: Cory Watilo Date: Mon, 11 Jul 2022 04:53:24 -0400 Subject: [PATCH 012/213] change label of button for authorized domain to URL (#10702) --- frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx b/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx index 95d77da84df222..4b7212eb34475f 100644 --- a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx +++ b/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx @@ -147,7 +147,7 @@ export function AuthorizedUrlsTable({ pageKey, actionId }: AuthorizedUrlsTableIn />

- Add{pageKey === 'toolbar-launch' && ' authorized domain'} + Add{pageKey === 'toolbar-launch' && ' authorized URL'}
Date: Mon, 11 Jul 2022 10:20:35 +0100 Subject: [PATCH 013/213] feat: write csv exports to object storage when enabled (#10708) * Revert "Revert "feat: write csv exports to object storage when enabled (#10666)" (#10684)" This reverts commit b898b60f6d03d52ec70cf21508812e3e394c8355. * fallback to writing to postgres if object storage write fails * obey mypy * instead of a long templated string --- posthog/api/test/test_exports.py | 3 +- posthog/models/exported_asset.py | 4 +- posthog/tasks/exporter.py | 9 +- posthog/tasks/exports/csv_exporter.py | 48 +++- .../tasks/exports/test/test_csv_exporter.py | 232 ++++++++++-------- 5 files changed, 169 insertions(+), 127 deletions(-) diff --git a/posthog/api/test/test_exports.py b/posthog/api/test/test_exports.py index 9174adee1d9b74..96d8ab5295777c 100644 --- a/posthog/api/test/test_exports.py +++ b/posthog/api/test/test_exports.py @@ -299,7 +299,8 @@ def requests_side_effect(*args, **kwargs): # pass the root in because django/celery refused to override it otherwise # limit the query to force it to page against the API - exporter.export_asset(instance.id, TEST_ROOT_BUCKET, limit=1) + with self.settings(OBJECT_STORAGE_ENABLED=False): + exporter.export_asset(instance.id, limit=1) response: Optional[HttpResponse] = None attempt_count = 0 diff --git a/posthog/models/exported_asset.py b/posthog/models/exported_asset.py index 244636ba065d62..d35bb76dc6ecd3 100644 --- a/posthog/models/exported_asset.py +++ b/posthog/models/exported_asset.py @@ -1,4 +1,3 @@ -import gzip import secrets from datetime import timedelta from typing import Optional @@ -95,8 +94,7 @@ def asset_for_token(token: str) -> ExportedAsset: def get_content_response(asset: ExportedAsset, download: bool = False): content = asset.content if not content and asset.content_location: - content_bytes = object_storage.read_bytes(asset.content_location) - content = gzip.decompress(content_bytes) + content = object_storage.read(asset.content_location) res = HttpResponse(content, content_type=asset.export_format) if download: diff --git a/posthog/tasks/exporter.py b/posthog/tasks/exporter.py index ecb94c366170f9..fe7bc305968f83 100644 --- a/posthog/tasks/exporter.py +++ b/posthog/tasks/exporter.py @@ -1,16 +1,11 @@ from typing import Optional -from posthog import settings from posthog.celery import app from posthog.models import ExportedAsset @app.task(retries=3) -def export_asset( - exported_asset_id: int, - storage_root_bucket: str = settings.OBJECT_STORAGE_EXPORTS_FOLDER, - limit: Optional[int] = None, -) -> None: +def export_asset(exported_asset_id: int, limit: Optional[int] = None,) -> None: from statshog.defaults.django import statsd from posthog.tasks.exports import csv_exporter, image_exporter @@ -19,7 +14,7 @@ def export_asset( is_csv_export = exported_asset.export_format == ExportedAsset.ExportFormat.CSV if is_csv_export: - csv_exporter.export_csv(exported_asset, storage_root_bucket, limit=limit) + csv_exporter.export_csv(exported_asset, limit=limit) statsd.incr("csv_exporter.queued", tags={"team_id": str(exported_asset.team_id)}) else: image_exporter.export_image(exported_asset) diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index 61051273a34b5a..c185a2bee12d41 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -4,14 +4,17 @@ import requests import structlog +from django.conf import settings from rest_framework_csv import renderers as csvrenderers from sentry_sdk import capture_exception, push_scope from statshog.defaults.django import statsd -from posthog import settings from posthog.jwt import PosthogJwtAudience, encode_jwt from posthog.logging.timing import timed from posthog.models.exported_asset import ExportedAsset +from posthog.models.utils import UUIDT +from posthog.storage import object_storage +from posthog.storage.object_storage import ObjectStorageError from posthog.utils import absolute_uri logger = structlog.get_logger(__name__) @@ -117,9 +120,7 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: return [] -def _export_to_csv( - exported_asset: ExportedAsset, root_bucket: str, limit: int = 1000, max_limit: int = 10_000, -) -> None: +def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: int = 10_000,) -> None: resource = exported_asset.export_context path: str = resource["path"] @@ -144,6 +145,7 @@ def _export_to_csv( # Figure out how to handle funnel polling.... data = response.json() + csv_rows = _convert_response_to_csv_data(data) all_csv_rows = all_csv_rows + csv_rows @@ -162,23 +164,45 @@ def _export_to_csv( # If values are serialised then keep the order of the keys, else allow it to be unordered renderer.header = all_csv_rows[0].keys() - exported_asset.content = renderer.render(all_csv_rows) + rendered_csv_content = renderer.render(all_csv_rows) + + try: + if settings.OBJECT_STORAGE_ENABLED: + _write_to_object_storage(exported_asset, rendered_csv_content) + else: + _write_to_exported_asset(exported_asset, rendered_csv_content) + except ObjectStorageError as ose: + logger.error("csv_exporter.object-storage-error", exception=ose, exc_info=True) + _write_to_exported_asset(exported_asset, rendered_csv_content) + + +def _write_to_exported_asset(exported_asset: ExportedAsset, rendered_csv_content: bytes) -> None: + exported_asset.content = rendered_csv_content exported_asset.save(update_fields=["content"]) +def _write_to_object_storage(exported_asset: ExportedAsset, rendered_csv_content: bytes) -> None: + path_parts: List[str] = [ + settings.OBJECT_STORAGE_EXPORTS_FOLDER, + "csvs", + f"team-{exported_asset.team.id}", + f"task-{exported_asset.id}", + str(UUIDT()), + ] + object_path = f'/{"/".join(path_parts)}' + object_storage.write(object_path, rendered_csv_content) + exported_asset.content_location = object_path + exported_asset.save(update_fields=["content_location"]) + + @timed("csv_exporter") -def export_csv( - exported_asset: ExportedAsset, - root_bucket: str = settings.OBJECT_STORAGE_EXPORTS_FOLDER, - limit: Optional[int] = None, - max_limit: int = 10_000, -) -> None: +def export_csv(exported_asset: ExportedAsset, limit: Optional[int] = None, max_limit: int = 10_000,) -> None: if not limit: limit = 1000 try: if exported_asset.export_format == "text/csv": - _export_to_csv(exported_asset, root_bucket, limit, max_limit) + _export_to_csv(exported_asset, limit, max_limit) statsd.incr("csv_exporter.succeeded", tags={"team_id": exported_asset.team.id}) else: statsd.incr("csv_exporter.unknown_asset", tags={"team_id": exported_asset.team.id}) diff --git a/posthog/tasks/exports/test/test_csv_exporter.py b/posthog/tasks/exports/test/test_csv_exporter.py index fdf57c51e58f90..db495d498d2320 100644 --- a/posthog/tasks/exports/test/test_csv_exporter.py +++ b/posthog/tasks/exports/test/test_csv_exporter.py @@ -1,122 +1,146 @@ from unittest.mock import Mock, patch -from rest_framework_csv import renderers as csvrenderers +import pytest +from boto3 import resource +from botocore.client import Config from posthog.models import ExportedAsset +from posthog.settings import ( + OBJECT_STORAGE_ACCESS_KEY_ID, + OBJECT_STORAGE_BUCKET, + OBJECT_STORAGE_ENDPOINT, + OBJECT_STORAGE_SECRET_ACCESS_KEY, +) +from posthog.storage import object_storage +from posthog.storage.object_storage import ObjectStorageError from posthog.tasks.exports import csv_exporter from posthog.test.base import APIBaseTest +TEST_BUCKET = "Test-Exports" + class TestCSVExporter(APIBaseTest): - @patch("posthog.tasks.exports.csv_exporter.requests.request") - def test_can_render_known_responses(self, patched_request) -> None: - """ - regression test to triangulate a test that passes locally but fails in CI - """ + @pytest.fixture(autouse=True) + def patched_request(self): + with patch("posthog.tasks.exports.csv_exporter.requests.request") as patched_request: + mock_response = Mock() + # API responses copied from https://github.com/PostHog/posthog/runs/7221634689?check_suite_focus=true + mock_response.json.side_effect = [ + { + "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095295%2B00%3A00", + "results": [ + { + "id": "e9ca132e-400f-4854-a83c-16c151b2f145", + "distinct_id": "2", + "properties": {"$browser": "Safari"}, + "event": "event_name", + "timestamp": "2022-07-06T19:37:43.095295+00:00", + "person": None, + "elements": [], + "elements_chain": "", + } + ], + }, + { + "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095279%2B00%3A00", + "results": [ + { + "id": "1624228e-a4f1-48cd-aabc-6baa3ddb22e4", + "distinct_id": "2", + "properties": {"$browser": "Safari"}, + "event": "event_name", + "timestamp": "2022-07-06T19:37:43.095279+00:00", + "person": None, + "elements": [], + "elements_chain": "", + } + ], + }, + { + "next": None, + "results": [ + { + "id": "66d45914-bdf5-4980-a54a-7dc699bdcce9", + "distinct_id": "2", + "properties": {"$browser": "Safari"}, + "event": "event_name", + "timestamp": "2022-07-06T19:37:43.095262+00:00", + "person": None, + "elements": [], + "elements_chain": "", + } + ], + }, + ] + patched_request.return_value = mock_response + yield patched_request + + exported_asset: ExportedAsset + + def setup_method(self, method): asset = ExportedAsset( team=self.team, export_format=ExportedAsset.ExportFormat.CSV, export_context={"path": "/api/literally/anything"}, ) asset.save() + self.exported_asset = asset - mock_response = Mock() - # API responses copied from https://github.com/PostHog/posthog/runs/7221634689?check_suite_focus=true - mock_response.json.side_effect = [ - { - "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095295%2B00%3A00", - "results": [ - { - "id": "e9ca132e-400f-4854-a83c-16c151b2f145", - "distinct_id": "2", - "properties": {"$browser": "Safari"}, - "event": "event_name", - "timestamp": "2022-07-06T19:37:43.095295+00:00", - "person": None, - "elements": [], - "elements_chain": "", - } - ], - }, - { - "next": "http://testserver/api/projects/169/events?orderBy=%5B%22-timestamp%22%5D&properties=%5B%7B%22key%22%3A%22%24browser%22%2C%22value%22%3A%5B%22Safari%22%5D%2C%22operator%22%3A%22exact%22%2C%22type%22%3A%22event%22%7D%5D&after=2022-07-06T19%3A27%3A43.206326&limit=1&before=2022-07-06T19%3A37%3A43.095279%2B00%3A00", - "results": [ - { - "id": "1624228e-a4f1-48cd-aabc-6baa3ddb22e4", - "distinct_id": "2", - "properties": {"$browser": "Safari"}, - "event": "event_name", - "timestamp": "2022-07-06T19:37:43.095279+00:00", - "person": None, - "elements": [], - "elements_chain": "", - } - ], - }, - { - "next": None, - "results": [ - { - "id": "66d45914-bdf5-4980-a54a-7dc699bdcce9", - "distinct_id": "2", - "properties": {"$browser": "Safari"}, - "event": "event_name", - "timestamp": "2022-07-06T19:37:43.095262+00:00", - "person": None, - "elements": [], - "elements_chain": "", - } - ], - }, - ] - patched_request.return_value = mock_response - csv_exporter.export_csv(asset) - - assert ( - asset.content - == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" + def teardown_method(self, method): + s3 = resource( + "s3", + endpoint_url=OBJECT_STORAGE_ENDPOINT, + aws_access_key_id=OBJECT_STORAGE_ACCESS_KEY_ID, + aws_secret_access_key=OBJECT_STORAGE_SECRET_ACCESS_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1", ) + bucket = s3.Bucket(OBJECT_STORAGE_BUCKET) + bucket.objects.filter(Prefix=TEST_BUCKET).delete() - def test_can_render_known_response_using_renderer(self) -> None: - """ - regression test to triangulate a test that passes locally but fails in CI - """ - csv_data_gathered_in_ci = [ - { - "id": "e9ca132e-400f-4854-a83c-16c151b2f145", - "distinct_id": "2", - "properties": {"$browser": "Safari"}, - "event": "event_name", - "timestamp": "2022-07-06T19:37:43.095295+00:00", - "person": None, - "elements": [], - "elements_chain": "", - }, - { - "id": "1624228e-a4f1-48cd-aabc-6baa3ddb22e4", - "distinct_id": "2", - "properties": {"$browser": "Safari"}, - "event": "event_name", - "timestamp": "2022-07-06T19:37:43.095279+00:00", - "person": None, - "elements": [], - "elements_chain": "", - }, - { - "id": "66d45914-bdf5-4980-a54a-7dc699bdcce9", - "distinct_id": "2", - "properties": {"$browser": "Safari"}, - "event": "event_name", - "timestamp": "2022-07-06T19:37:43.095262+00:00", - "person": None, - "elements": [], - "elements_chain": "", - }, - ] - - renderer = csvrenderers.CSVRenderer() - - assert ( - renderer.render(csv_data_gathered_in_ci) - == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" - ) + def test_csv_exporter_writes_to_asset_when_object_storage_is_disabled(self) -> None: + with self.settings(OBJECT_STORAGE_ENABLED=False): + csv_exporter.export_csv(self.exported_asset) + + assert ( + self.exported_asset.content + == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" + ) + assert self.exported_asset.content_location is None + + @patch("posthog.tasks.exports.csv_exporter.UUIDT") + def test_csv_exporter_writes_to_object_storage_when_object_storage_is_enabled(self, mocked_uuidt) -> None: + mocked_uuidt.return_value = "a-guid" + with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): + csv_exporter.export_csv(self.exported_asset) + + assert ( + self.exported_asset.content_location + == f"/{TEST_BUCKET}/csvs/team-{self.team.id}/task-{self.exported_asset.id}/a-guid" + ) + + content = object_storage.read(self.exported_asset.content_location) + assert ( + content + == "distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" + ) + + assert self.exported_asset.content is None + + @patch("posthog.tasks.exports.csv_exporter.UUIDT") + @patch("posthog.tasks.exports.csv_exporter.object_storage.write") + def test_csv_exporter_writes_to_object_storage_when_object_storage_write_fails( + self, mocked_uuidt, mocked_object_storage_write + ) -> None: + mocked_uuidt.return_value = "a-guid" + mocked_object_storage_write.side_effect = ObjectStorageError("mock write failed") + + with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): + csv_exporter.export_csv(self.exported_asset) + + assert self.exported_asset.content_location is None + + assert ( + self.exported_asset.content + == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" + ) From 1e7cc28f2611bf6bad46f40c075fa49f5e28c8e2 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 11 Jul 2022 11:47:46 +0100 Subject: [PATCH 014/213] chore: capture object storage error to sentry (#10710) * chore: capture object storage error to sentry * capture all object storage errors to sentry --- posthog/storage/object_storage.py | 3 +++ posthog/tasks/exports/csv_exporter.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/posthog/storage/object_storage.py b/posthog/storage/object_storage.py index 301386ce253c06..5a48a70f5b8593 100644 --- a/posthog/storage/object_storage.py +++ b/posthog/storage/object_storage.py @@ -5,6 +5,7 @@ from boto3 import client from botocore.client import Config from django.conf import settings +from sentry_sdk import capture_exception logger = structlog.get_logger(__name__) @@ -72,6 +73,7 @@ def read_bytes(self, bucket: str, key: str) -> Optional[bytes]: return s3_response["Body"].read() except Exception as e: logger.error("object_storage.read_failed", bucket=bucket, file_name=key, error=e, s3_response=s3_response) + capture_exception(e) raise ObjectStorageError("read failed") from e def write(self, bucket: str, key: str, content: Union[str, bytes]) -> None: @@ -80,6 +82,7 @@ def write(self, bucket: str, key: str, content: Union[str, bytes]) -> None: s3_response = self.aws_client.put_object(Bucket=bucket, Body=content, Key=key) except Exception as e: logger.error("object_storage.write_failed", bucket=bucket, file_name=key, error=e, s3_response=s3_response) + capture_exception(e) raise ObjectStorageError("write failed") from e diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index c185a2bee12d41..2f78c8498aadc9 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -172,6 +172,9 @@ def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: else: _write_to_exported_asset(exported_asset, rendered_csv_content) except ObjectStorageError as ose: + with push_scope() as scope: + scope.set_tag("celery_task", "csv_export") + capture_exception(ose) logger.error("csv_exporter.object-storage-error", exception=ose, exc_info=True) _write_to_exported_asset(exported_asset, rendered_csv_content) From 4d314a7c27fa34a9b9aea1507c72415a35c5eb7d Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 11 Jul 2022 12:34:50 +0100 Subject: [PATCH 015/213] chore: configure celery logging to use correct structlog config (#10614) * chore: need to run config to have logs in same format as django * wip * condfigure logging in celery * try printing config errors * poke with stick * don't need to catch exception in signal handler --- posthog/celery.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/posthog/celery.py b/posthog/celery.py index a787273b4dcf38..9008a48a2cc012 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -4,7 +4,7 @@ from celery import Celery from celery.schedules import crontab -from celery.signals import task_postrun, task_prerun +from celery.signals import setup_logging, task_postrun, task_prerun from django.conf import settings from django.db import connection from django.utils import timezone @@ -43,6 +43,17 @@ UPDATE_CACHED_DASHBOARD_ITEMS_INTERVAL_SECONDS = settings.UPDATE_CACHED_DASHBOARD_ITEMS_INTERVAL_SECONDS +@setup_logging.connect +def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs) -> None: + import logging + + from posthog.settings import logs + + # following instructions from here https://django-structlog.readthedocs.io/en/latest/celery.html + # mypy thinks that there is no `logging.config` but there is ¯\_(ツ)_/¯ + logging.config.dictConfig(logs.LOGGING) # type: ignore + + @app.on_after_configure.connect def setup_periodic_tasks(sender: Celery, **kwargs): # Monitoring tasks From 7f0756eedd6d0f991e6f82b2cea63b8f4a216b1b Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 11 Jul 2022 13:44:28 +0100 Subject: [PATCH 016/213] chore: updates the chromium version (#10712) * chore: updates the chromium version * spedicy exact version for alpine 3.14 --- production.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production.Dockerfile b/production.Dockerfile index de741a3c24bb71..09b683646a0c84 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -69,8 +69,8 @@ RUN apk --update --no-cache add \ "libpq~=13" \ "libxslt~=1.1" \ "nodejs-current~=16" \ - "chromium~=93" \ - "chromium-chromedriver~=93" \ + "chromium=93.0.4577.82-r0" \ + "chromium-chromedriver=93.0.4577.82-r0" \ "xmlsec~=1.2" From 133f4ff0901212afde60dcf0cc9b0d664f2527a6 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Mon, 11 Jul 2022 13:31:18 +0000 Subject: [PATCH 017/213] feat(plugins-ui): immediate feedback on enable/disable switch (#10714) --- .../components/LemonSwitch/LemonSwitch.tsx | 8 +++-- .../src/scenes/plugins/plugin/PluginCard.tsx | 32 +++++++++---------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/frontend/src/lib/components/LemonSwitch/LemonSwitch.tsx b/frontend/src/lib/components/LemonSwitch/LemonSwitch.tsx index f1fd161dc61e8d..ccf55b58c4fce7 100644 --- a/frontend/src/lib/components/LemonSwitch/LemonSwitch.tsx +++ b/frontend/src/lib/components/LemonSwitch/LemonSwitch.tsx @@ -5,7 +5,7 @@ import { Spinner } from '../Spinner/Spinner' import './LemonSwitch.scss' export interface LemonSwitchProps extends Omit, 'alt' | 'label' | 'onChange' | 'outlined'> { - onChange: (newChecked: boolean) => void + onChange?: (newChecked: boolean) => void checked: boolean label?: string | JSX.Element /** Whether the switch should use the alternative primary color. */ @@ -53,7 +53,11 @@ export function LemonSwitch({ className="LemonSwitch__button" type="button" role="switch" - onClick={() => onChange(!checked)} + onClick={() => { + if (onChange) { + onChange(!checked) + } + }} onMouseDown={() => setIsActive(true)} onMouseUp={() => setIsActive(false)} onMouseOut={() => setIsActive(false)} diff --git a/frontend/src/scenes/plugins/plugin/PluginCard.tsx b/frontend/src/scenes/plugins/plugin/PluginCard.tsx index 94a64d0079c5e5..31b06f59d34fe6 100644 --- a/frontend/src/scenes/plugins/plugin/PluginCard.tsx +++ b/frontend/src/scenes/plugins/plugin/PluginCard.tsx @@ -1,4 +1,4 @@ -import { Button, Card, Col, Popconfirm, Row, Space, Switch, Tag } from 'antd' +import { Button, Card, Col, Row, Space, Tag } from 'antd' import { useActions, useValues } from 'kea' import React from 'react' import { pluginsLogic } from 'scenes/plugins/pluginsLogic' @@ -28,6 +28,7 @@ import { canInstallPlugins } from '../access' import { LinkButton } from 'lib/components/LinkButton' import { PluginUpdateButton } from './PluginUpdateButton' import { Tooltip } from 'lib/components/Tooltip' +import { LemonSwitch } from '@posthog/lemon-ui' export function PluginAboutButton({ url, disabled = false }: { url: string; disabled?: boolean }): JSX.Element { return ( @@ -128,22 +129,19 @@ export function PluginCard({ ) : null} {pluginConfig && ( - - pluginConfig.id - ? toggleEnabled({ id: pluginConfig.id, enabled: !pluginConfig.enabled }) - : editPlugin(pluginId || null, { __enabled: true }) - } - okText="Yes" - cancelText="No" - disabled={rearranging} - > - - + {pluginConfig.id ? ( + + toggleEnabled({ id: pluginConfig.id, enabled: !pluginConfig.enabled }) + } + /> + ) : ( + + + + )} )} From 77ff6b319325ec53bd68b8c65419364d3c8d6075 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 11 Jul 2022 15:02:26 +0100 Subject: [PATCH 018/213] Revert "chore: updates the chromium version (#10712)" (#10715) This reverts commit 7f0756eedd6d0f991e6f82b2cea63b8f4a216b1b. --- production.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production.Dockerfile b/production.Dockerfile index 09b683646a0c84..de741a3c24bb71 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -69,8 +69,8 @@ RUN apk --update --no-cache add \ "libpq~=13" \ "libxslt~=1.1" \ "nodejs-current~=16" \ - "chromium=93.0.4577.82-r0" \ - "chromium-chromedriver=93.0.4577.82-r0" \ + "chromium~=93" \ + "chromium-chromedriver~=93" \ "xmlsec~=1.2" From 950ccc7429cedb2310c3e1c1b5419fb6a090fa25 Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Mon, 11 Jul 2022 07:51:04 -0700 Subject: [PATCH 019/213] chore(session-analysis): add session analytics and session warning (#10704) * chore(session-analysis): add session analytics and session warning * fix test * add tests for isUsingSessionAnalysis * add docs link * fix issue --- frontend/src/lib/utils/eventUsageLogic.ts | 6 +- .../src/scenes/insights/InsightContainer.tsx | 11 ++ .../src/scenes/insights/insightLogic.test.ts | 133 +++++++++++++++++- frontend/src/scenes/insights/insightLogic.ts | 33 ++++- frontend/src/types.ts | 2 +- 5 files changed, 180 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/utils/eventUsageLogic.ts b/frontend/src/lib/utils/eventUsageLogic.ts index d9f5578c6ae524..eb1f30dcc2edc4 100644 --- a/frontend/src/lib/utils/eventUsageLogic.ts +++ b/frontend/src/lib/utils/eventUsageLogic.ts @@ -213,7 +213,8 @@ export const eventUsageLogic = kea({ isFirstLoad: boolean, fromDashboard: boolean, delay?: number, - changedFilters?: Record + changedFilters?: Record, + isUsingSessionAnalysis?: boolean ) => ({ insightModel, filters, @@ -222,6 +223,7 @@ export const eventUsageLogic = kea({ fromDashboard, delay, changedFilters, + isUsingSessionAnalysis, }), reportPersonsModalViewed: (params: PersonsModalParams, count: number, hasNext: boolean) => ({ params, @@ -516,6 +518,7 @@ export const eventUsageLogic = kea({ fromDashboard, delay, changedFilters, + isUsingSessionAnalysis, }) => { const { insight } = filters @@ -543,6 +546,7 @@ export const eventUsageLogic = kea({ if (insight === 'TRENDS') { properties.breakdown_type = filters.breakdown_type properties.breakdown = filters.breakdown + properties.using_session_analysis = isUsingSessionAnalysis } else if (insight === 'RETENTION') { properties.period = filters.period properties.date_to = filters.date_to diff --git a/frontend/src/scenes/insights/InsightContainer.tsx b/frontend/src/scenes/insights/InsightContainer.tsx index 7594d7d2b9d728..892e9fd9c7f59c 100644 --- a/frontend/src/scenes/insights/InsightContainer.tsx +++ b/frontend/src/scenes/insights/InsightContainer.tsx @@ -34,6 +34,7 @@ import { AnimationType } from 'lib/animations/animations' import { FunnelCorrelation } from './views/Funnels/FunnelCorrelation' import { FunnelInsight } from './views/Funnels/FunnelInsight' import { ExportButton } from 'lib/components/ExportButton/ExportButton' +import { AlertMessage } from 'lib/components/AlertMessage' const VIEW_MAP = { [`${InsightType.TRENDS}`]: , @@ -68,6 +69,7 @@ export function InsightContainer( showErrorMessage, csvExportUrl, exporterResourceParams, + isUsingSessionAnalysis, } = useValues(insightLogic) const { areFiltersValid, isValidFunnel, areExclusionFiltersValid, correlationAnalysisAvailable } = useValues( funnelLogic(insightProps) @@ -183,6 +185,15 @@ export function InsightContainer( return ( <> + {isUsingSessionAnalysis ? ( +
+ + When using sessions and session properties, events without session IDs will be excluded from the + set of results.{' '} + Learn more about sessions. + +
+ ) : null} {/* These are filters that are reused between insight features. They each have generic logic that updates the url */} { 0, { changed_insight: InsightType.TRENDS, - } + }, + false ), ]) }) @@ -787,4 +789,133 @@ describe('insightLogic', () => { }).toNotHaveDispatchedActions(['loadResults']) }) }) + describe('isUsingSessionAnalysis selector', () => { + it('is false by default', async () => { + const insight = { + filters: { insight: InsightType.TRENDS }, + } + logic = insightLogic({ + dashboardItemId: undefined, + cachedInsight: insight, + }) + logic.mount() + expectLogic(logic).toMatchValues({ isUsingSessionAnalysis: false }) + }) + it('setting session breakdown sets it true', async () => { + const insight = { + filters: { insight: InsightType.TRENDS, breakdown_type: 'session' as BreakdownType }, + } + logic = insightLogic({ + dashboardItemId: undefined, + cachedInsight: insight, + }) + logic.mount() + expectLogic(logic).toMatchValues({ isUsingSessionAnalysis: true }) + }) + it('setting global session property filters sets it true', async () => { + const insight = { + filters: { + insight: InsightType.TRENDS, + properties: { + type: FilterLogicalOperator.And, + values: [ + { + type: FilterLogicalOperator.And, + values: [ + { + key: '$session_duration', + value: 1, + operator: PropertyOperator.GreaterThan, + type: 'session', + }, + ], + }, + ], + }, + }, + } + logic = insightLogic({ + dashboardItemId: undefined, + cachedInsight: insight, + }) + logic.mount() + expectLogic(logic).toMatchValues({ isUsingSessionAnalysis: true }) + }) + + it('setting entity session property filters sets it true', async () => { + const insight = { + filters: { + events: [ + { + id: '$pageview', + name: '$pageview', + type: 'events', + order: 0, + properties: [ + { + key: '$session_duration', + value: 1, + operator: PropertyOperator.GreaterThan, + type: 'session', + }, + ], + }, + ], + }, + } + logic = insightLogic({ + dashboardItemId: undefined, + cachedInsight: insight, + }) + logic.mount() + expectLogic(logic).toMatchValues({ isUsingSessionAnalysis: true }) + }) + + it('setting math to unique_session sets it true', async () => { + const insight = { + filters: { + events: [ + { + id: '$pageview', + name: '$pageview', + type: 'events', + order: 0, + properties: [], + math: 'unique_session', + }, + ], + }, + } + logic = insightLogic({ + dashboardItemId: undefined, + cachedInsight: insight, + }) + logic.mount() + expectLogic(logic).toMatchValues({ isUsingSessionAnalysis: true }) + }) + + it('setting math to use session property sets it true', async () => { + const insight = { + filters: { + events: [ + { + id: '$pageview', + name: '$pageview', + type: 'events', + order: 0, + properties: [], + math: 'median', + math_property: '$session_duration', + }, + ], + }, + } + logic = insightLogic({ + dashboardItemId: undefined, + cachedInsight: insight, + }) + logic.mount() + expectLogic(logic).toMatchValues({ isUsingSessionAnalysis: true }) + }) + }) }) diff --git a/frontend/src/scenes/insights/insightLogic.ts b/frontend/src/scenes/insights/insightLogic.ts index 81e07ba6f88f8f..8a6a1c97295343 100644 --- a/frontend/src/scenes/insights/insightLogic.ts +++ b/frontend/src/scenes/insights/insightLogic.ts @@ -14,6 +14,7 @@ import { InsightType, ItemMode, SetInsightOptions, + PropertyFilter, } from '~/types' import { captureInternalMetric } from 'lib/internalMetrics' import { router } from 'kea-router' @@ -40,6 +41,7 @@ import { mathsLogic } from 'scenes/trends/mathsLogic' import { insightSceneLogic } from 'scenes/insights/insightSceneLogic' import { mergeWithDashboardTile } from 'scenes/insights/utils/dashboardTiles' import { TriggerExportProps } from 'lib/components/ExportButton/exporter' +import { parseProperties } from 'lib/components/PropertyFilters/utils' const IS_TEST_MODE = process.env.NODE_ENV === 'test' const SHOW_TIMEOUT_MESSAGE_AFTER = 15000 @@ -617,6 +619,31 @@ export const insightLogic = kea({ } }, ], + isUsingSessionAnalysis: [ + (s) => [s.filters], + (filters: Partial): boolean => { + const entities = (filters.events || []).concat(filters.actions ?? []) + const using_session_breakdown = filters.breakdown_type === 'session' + const using_session_math = entities.some((entity) => entity.math === 'unique_session') + const using_session_property_math = entities.some((entity) => { + // Should be made more generic is we ever add more session properties + return entity.math_property === '$session_duration' + }) + const using_entity_session_property_filter = entities.some((entity) => { + return entity.properties?.some((property: PropertyFilter) => property.type === 'session') + }) + const using_global_session_property_filter = parseProperties(filters.properties).some( + (property) => property.type === 'session' + ) + return ( + using_session_breakdown || + using_session_math || + using_session_property_math || + using_entity_session_property_filter || + using_global_session_property_filter + ) + }, + ], }, listeners: ({ actions, selectors, values }) => ({ setFilters: async ({ filters }, _, __, previousState) => { @@ -687,7 +714,8 @@ export const insightLogic = kea({ values.isFirstLoad, Boolean(fromDashboard), 0, - changedKeysObj + changedKeysObj, + values.isUsingSessionAnalysis ) actions.setNotFirstLoad() @@ -700,7 +728,8 @@ export const insightLogic = kea({ values.isFirstLoad, Boolean(fromDashboard), 10, - changedKeysObj + changedKeysObj, + values.isUsingSessionAnalysis ) } }, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 72984ad3f4207b..7c331f1af1ae92 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -981,7 +981,7 @@ export enum ChartDisplayType { WorldMap = 'WorldMap', } -export type BreakdownType = 'cohort' | 'person' | 'event' | 'group' +export type BreakdownType = 'cohort' | 'person' | 'event' | 'group' | 'session' export type IntervalType = 'hour' | 'day' | 'week' | 'month' export type SmoothingType = number From cfc9acf557e103bd150514b40221da9ed6ca596d Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Mon, 11 Jul 2022 08:16:43 -0700 Subject: [PATCH 020/213] chore(session-analysis): merge histogram and session analysis feature flags (#10717) --- frontend/src/lib/constants.tsx | 1 - .../filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 14d8272846bdf3..a9484312802b25 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -122,7 +122,6 @@ export const FEATURE_FLAGS = { FEATURE_FLAG_EXPERIENCE_CONTINUITY: 'feature-flag-experience-continuity', // owner: @neilkakkar EMBED_INSIGHTS: 'embed-insights', // owner: @mariusandra ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS: 'ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS', // owner: @pauldambra - HISTOGRAM_INSIGHTS: 'histogram-insights', // owner: @rcmarron } /** Which self-hosted plan's features are available with Cloud's "Standard" plan (aka card attached). */ diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx index 970cd3a2c91593..bcecfc96be76e8 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx @@ -86,7 +86,7 @@ export function BreakdownFilter({ ? [] : breakdownArray.map((t, index) => { const key = `${t}-${index}` - const isPropertyHistogramable = featureFlags[FEATURE_FLAGS.HISTOGRAM_INSIGHTS] + const isPropertyHistogramable = featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS] ? !useMultiBreakdown && !!getPropertyDefinition(t)?.is_numerical : false return ( @@ -108,7 +108,7 @@ export function BreakdownFilter({ breakdownParts, setFilters, getPropertyDefinition: getPropertyDefinition, - histogramFeatureFlag: !!featureFlags[FEATURE_FLAGS.HISTOGRAM_INSIGHTS], + histogramFeatureFlag: !!featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS], }) : undefined From 25152334f9133dbbc26196d10c6ec92c3c277043 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Mon, 11 Jul 2022 15:34:36 +0000 Subject: [PATCH 021/213] refactor(plugin-jobs): make job queues extensible (#10713) * refactor(plugin-jobs): make job queues extensible * fix enqueue callsites * final fix * use enum for job names * test fix * fix tests --- .../job-queues/concurrent/graphile-queue.ts | 12 +++------ .../src/main/job-queues/job-queue-base.ts | 18 +++++++------ .../src/main/job-queues/job-queue-consumer.ts | 15 +++++------ .../src/main/job-queues/job-queue-manager.ts | 11 ++++---- .../src/main/job-queues/local/fs-queue.ts | 25 ++++++++++++------- plugin-server/src/types.ts | 10 +++++--- plugin-server/src/worker/tasks.ts | 4 +-- .../src/worker/vm/extensions/jobs.ts | 3 ++- plugin-server/tests/jobs.test.ts | 6 ++--- plugin-server/tests/worker/vm.test.ts | 10 ++++---- 10 files changed, 63 insertions(+), 51 deletions(-) diff --git a/plugin-server/src/main/job-queues/concurrent/graphile-queue.ts b/plugin-server/src/main/job-queues/concurrent/graphile-queue.ts index dbf76cbb06e9bf..688f59018e3eb0 100644 --- a/plugin-server/src/main/job-queues/concurrent/graphile-queue.ts +++ b/plugin-server/src/main/job-queues/concurrent/graphile-queue.ts @@ -33,10 +33,10 @@ export class GraphileQueue extends JobQueueBase { await this.migrate() } - async enqueue(retry: EnqueuedJob): Promise { + async enqueue(jobName: string, job: EnqueuedJob): Promise { const workerUtils = await this.getWorkerUtils() - await workerUtils.addJob('pluginJob', retry, { - runAt: new Date(retry.timestamp), + await workerUtils.addJob(jobName, job, { + runAt: new Date(job.timestamp), maxAttempts: 1, }) } @@ -78,11 +78,7 @@ export class GraphileQueue extends JobQueueBase { noHandleSignals: false, pollInterval: 100, // you can set the taskList or taskDirectory but not both - taskList: { - pluginJob: (payload) => { - void this.onJob?.([payload as EnqueuedJob]) - }, - }, + taskList: this.jobHandlers, }) } } else { diff --git a/plugin-server/src/main/job-queues/job-queue-base.ts b/plugin-server/src/main/job-queues/job-queue-base.ts index 23230827c16d36..0ea025128714dd 100644 --- a/plugin-server/src/main/job-queues/job-queue-base.ts +++ b/plugin-server/src/main/job-queues/job-queue-base.ts @@ -1,16 +1,18 @@ -import { EnqueuedJob, JobQueue, OnJobCallback } from '../../types' +import { TaskList } from 'graphile-worker' + +import { EnqueuedJob, JobQueue } from '../../types' export class JobQueueBase implements JobQueue { started: boolean paused: boolean - onJob: OnJobCallback | null + jobHandlers: TaskList timeout: NodeJS.Timeout | null intervalSeconds: number constructor() { this.started = false this.paused = false - this.onJob = null + this.jobHandlers = {} this.timeout = null this.intervalSeconds = 10 } @@ -21,9 +23,9 @@ export class JobQueueBase implements JobQueue { throw new Error('connectProducer() not implemented for job queue!') } - enqueue(retry: EnqueuedJob): void + enqueue(jobName: string, job: EnqueuedJob): void // eslint-disable-next-line @typescript-eslint/require-await, @typescript-eslint/no-unused-vars - async enqueue(retry: EnqueuedJob): Promise { + async enqueue(jobName: string, job: EnqueuedJob): Promise { throw new Error('enqueue() not implemented for job queue!') } @@ -33,9 +35,9 @@ export class JobQueueBase implements JobQueue { throw new Error('disconnectProducer() not implemented for job queue!') } - startConsumer(onJob: OnJobCallback): void - async startConsumer(onJob: OnJobCallback): Promise { - this.onJob = onJob + startConsumer(jobHandlers: TaskList): void + async startConsumer(jobHandlers: TaskList): Promise { + this.jobHandlers = jobHandlers if (!this.started) { this.started = true await this.syncState() diff --git a/plugin-server/src/main/job-queues/job-queue-consumer.ts b/plugin-server/src/main/job-queues/job-queue-consumer.ts index 7fe9bbe8d6205a..a41911766480b0 100644 --- a/plugin-server/src/main/job-queues/job-queue-consumer.ts +++ b/plugin-server/src/main/job-queues/job-queue-consumer.ts @@ -1,6 +1,7 @@ import Piscina from '@posthog/piscina' +import { TaskList } from 'graphile-worker' -import { Hub, JobQueueConsumerControl, OnJobCallback } from '../../types' +import { EnqueuedJob, Hub, JobQueueConsumerControl } from '../../types' import { killProcess } from '../../utils/kill' import { status } from '../../utils/status' import { logOrThrowJobQueueError } from '../../utils/utils' @@ -9,19 +10,19 @@ import { pauseQueueIfWorkerFull } from '../ingestion-queues/queue' export async function startJobQueueConsumer(server: Hub, piscina: Piscina): Promise { status.info('🔄', 'Starting job queue consumer, trying to get lock...') - const onJob: OnJobCallback = async (jobs) => { - pauseQueueIfWorkerFull(() => server.jobQueueManager.pauseConsumer(), server, piscina) - for (const job of jobs) { + const jobHandlers: TaskList = { + pluginJob: async (job) => { + pauseQueueIfWorkerFull(() => server.jobQueueManager.pauseConsumer(), server, piscina) server.statsd?.increment('triggered_job', { instanceId: server.instanceId.toString(), }) - await piscina.run({ task: 'runJob', args: { job } }) - } + await piscina.run({ task: 'runJob', args: { job: job as EnqueuedJob } }) + }, } status.info('🔄', 'Job queue consumer starting') try { - await server.jobQueueManager.startConsumer(onJob) + await server.jobQueueManager.startConsumer(jobHandlers) } catch (error) { try { logOrThrowJobQueueError(server, error, `Cannot start job queue consumer!`) diff --git a/plugin-server/src/main/job-queues/job-queue-manager.ts b/plugin-server/src/main/job-queues/job-queue-manager.ts index 286d43509ad230..f9532c846ba091 100644 --- a/plugin-server/src/main/job-queues/job-queue-manager.ts +++ b/plugin-server/src/main/job-queues/job-queue-manager.ts @@ -1,6 +1,7 @@ import * as Sentry from '@sentry/node' +import { TaskList } from 'graphile-worker' -import { EnqueuedJob, Hub, JobQueue, JobQueueType, OnJobCallback } from '../../types' +import { EnqueuedJob, Hub, JobQueue, JobQueueType } from '../../types' import { status } from '../../utils/status' import { logOrThrowJobQueueError } from '../../utils/utils' import { jobQueueMap } from './job-queues' @@ -49,10 +50,10 @@ export class JobQueueManager implements JobQueue { } } - async enqueue(job: EnqueuedJob): Promise { + async enqueue(jobName: string, job: EnqueuedJob): Promise { for (const jobQueue of this.jobQueues) { try { - await jobQueue.enqueue(job) + await jobQueue.enqueue(jobName, job) return } catch (error) { // if one fails, take the next queue @@ -72,8 +73,8 @@ export class JobQueueManager implements JobQueue { await Promise.all(this.jobQueues.map((r) => r.disconnectProducer())) } - async startConsumer(onJob: OnJobCallback): Promise { - await Promise.all(this.jobQueues.map((r) => r.startConsumer(onJob))) + async startConsumer(jobHandlers: TaskList): Promise { + await Promise.all(this.jobQueues.map((r) => r.startConsumer(jobHandlers))) } async stopConsumer(): Promise { diff --git a/plugin-server/src/main/job-queues/local/fs-queue.ts b/plugin-server/src/main/job-queues/local/fs-queue.ts index 30c52aed84105b..cd5826f264497a 100644 --- a/plugin-server/src/main/job-queues/local/fs-queue.ts +++ b/plugin-server/src/main/job-queues/local/fs-queue.ts @@ -1,10 +1,15 @@ -import { EnqueuedJob, OnJobCallback } from '../../../types' +import { JobHelpers, TaskList } from 'graphile-worker' + +import { EnqueuedJob } from '../../../types' import Timeout = NodeJS.Timeout import * as fs from 'fs' import * as path from 'path' import { JobQueueBase } from '../job-queue-base' +interface FsJob extends EnqueuedJob { + jobName: string +} export class FsQueue extends JobQueueBase { paused: boolean started: boolean @@ -32,12 +37,12 @@ export class FsQueue extends JobQueueBase { // nothing to do } - enqueue(job: EnqueuedJob): Promise | void { - fs.appendFileSync(this.filename, `${JSON.stringify(job)}\n`) + enqueue(jobName: string, job: EnqueuedJob): Promise | void { + fs.appendFileSync(this.filename, `${JSON.stringify({ jobName, ...job })}\n`) } - startConsumer(onJob: OnJobCallback): void { - super.startConsumer(onJob) + startConsumer(jobHandlers: TaskList): void { + super.startConsumer(jobHandlers) fs.writeFileSync(this.filename, '') } @@ -48,15 +53,17 @@ export class FsQueue extends JobQueueBase { .toString() .split('\n') .filter((a) => a) - .map((s) => JSON.parse(s) as EnqueuedJob) + .map((s) => JSON.parse(s) as FsJob) - const newQueue = queue.filter((element) => element.timestamp < timestamp) + const jobsQueue = queue.filter((element) => element.timestamp < timestamp) - if (newQueue.length > 0) { + if (jobsQueue.length > 0) { const oldQueue = queue.filter((element) => element.timestamp >= timestamp) fs.writeFileSync(this.filename, `${oldQueue.map((q) => JSON.stringify(q)).join('\n')}\n`) - await this.onJob?.(newQueue) + for (const job of jobsQueue) { + await this.jobHandlers[job.jobName](job, {} as JobHelpers) + } return true } diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 58fcfe54e31aab..20c0514ca76829 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -8,6 +8,7 @@ import { Properties, } from '@posthog/plugin-scaffold' import { Pool as GenericPool } from 'generic-pool' +import { TaskList } from 'graphile-worker' import { StatsD } from 'hot-shots' import { Redis } from 'ioredis' import { Kafka } from 'kafkajs' @@ -205,7 +206,6 @@ export interface PluginServerCapabilities { http?: boolean } -export type OnJobCallback = (queue: EnqueuedJob[]) => Promise | void export interface EnqueuedJob { type: string payload: Record @@ -214,15 +214,19 @@ export interface EnqueuedJob { pluginConfigTeam: number } +export enum JobName { + PLUGIN_JOB = 'pluginJob', +} + export interface JobQueue { - startConsumer: (onJob: OnJobCallback) => Promise | void + startConsumer: (jobHandlers: TaskList) => Promise | void stopConsumer: () => Promise | void pauseConsumer: () => Promise | void resumeConsumer: () => Promise | void isConsumerPaused: () => boolean connectProducer: () => Promise | void - enqueue: (job: EnqueuedJob) => Promise | void + enqueue: (jobName: string, job: EnqueuedJob) => Promise | void disconnectProducer: () => Promise | void } diff --git a/plugin-server/src/worker/tasks.ts b/plugin-server/src/worker/tasks.ts index 3ce2979d324f87..0e94a0971ac8dd 100644 --- a/plugin-server/src/worker/tasks.ts +++ b/plugin-server/src/worker/tasks.ts @@ -1,6 +1,6 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { Action, EnqueuedJob, Hub, IngestionEvent, PluginTaskType, Team } from '../types' +import { Action, EnqueuedJob, Hub, IngestionEvent, JobName, PluginTaskType, Team } from '../types' import { convertToProcessedPluginEvent } from '../utils/event' import { EventPipelineRunner } from './ingestion/event-pipeline/runner' import { runPluginTask, runProcessEvent } from './plugins/run' @@ -62,7 +62,7 @@ export const workerTasks: Record = { await hub.kafkaProducer.flush() }, enqueueJob: async (hub, { job }: { job: EnqueuedJob }) => { - await hub.jobQueueManager.enqueue(job) + await hub.jobQueueManager.enqueue(JobName.PLUGIN_JOB, job) }, // Exported only for tests _testsRunProcessEvent: async (hub, args: { event: PluginEvent }) => { diff --git a/plugin-server/src/worker/vm/extensions/jobs.ts b/plugin-server/src/worker/vm/extensions/jobs.ts index 7dd9e19c21d6a8..66ee09f1d59c2d 100644 --- a/plugin-server/src/worker/vm/extensions/jobs.ts +++ b/plugin-server/src/worker/vm/extensions/jobs.ts @@ -1,4 +1,5 @@ import { Hub, PluginConfig } from '../../../types' +import { JobName } from './../../../types' type JobRunner = { runAt: (date: Date) => Promise @@ -39,7 +40,7 @@ export function durationToMs(duration: number, unit: string): number { export function createJobs(server: Hub, pluginConfig: PluginConfig): Jobs { const runJob = async (type: string, payload: Record, timestamp: number) => { - await server.jobQueueManager.enqueue({ + await server.jobQueueManager.enqueue(JobName.PLUGIN_JOB, { type, payload, timestamp, diff --git a/plugin-server/tests/jobs.test.ts b/plugin-server/tests/jobs.test.ts index 3c4a9b679d2937..d089958f981752 100644 --- a/plugin-server/tests/jobs.test.ts +++ b/plugin-server/tests/jobs.test.ts @@ -2,7 +2,7 @@ import { gzipSync } from 'zlib' import { defaultConfig } from '../src/config/config' import { ServerInstance, startPluginsServer } from '../src/main/pluginsServer' -import { EnqueuedJob, Hub, LogLevel, PluginsServerConfig } from '../src/types' +import { EnqueuedJob, Hub, JobName, LogLevel, PluginsServerConfig } from '../src/types' import { createHub } from '../src/utils/db/hub' import { killProcess } from '../src/utils/kill' import { delay } from '../src/utils/utils' @@ -157,7 +157,7 @@ describe.skip('job queues', () => { pluginConfigTeam: 3, } - server.hub.jobQueueManager.enqueue(job) + server.hub.jobQueueManager.enqueue(JobName.PLUGIN_JOB, job) const consumedJob: EnqueuedJob = await new Promise((resolve) => { server.hub.jobQueueManager.startConsumer((consumedJob) => { resolve(consumedJob[0]) @@ -266,7 +266,7 @@ describe.skip('job queues', () => { pluginConfigId: 2, pluginConfigTeam: 3, } - await hub.jobQueueManager.enqueue(job) + await hub.jobQueueManager.enqueue(JobName.PLUGIN_JOB, job) expect(mS3WrapperInstance.upload).toBeCalledWith({ Body: gzipSync(Buffer.from(JSON.stringify(job), 'utf8')), diff --git a/plugin-server/tests/worker/vm.test.ts b/plugin-server/tests/worker/vm.test.ts index fb94eb71bf291f..9d491a5d9bd16a 100644 --- a/plugin-server/tests/worker/vm.test.ts +++ b/plugin-server/tests/worker/vm.test.ts @@ -1132,28 +1132,28 @@ describe('vm tests', () => { const mockJobQueueInstance = (JobQueueManager as any).mock.instances[0] const mockEnqueue = mockJobQueueInstance.enqueue expect(mockEnqueue).toHaveBeenCalledTimes(1) - expect(mockEnqueue).toHaveBeenCalledWith({ + expect(mockEnqueue).toHaveBeenCalledWith('pluginJob', { payload: { batch: [event, event, event], batchId: expect.any(Number), retriesPerformedSoFar: 1 }, pluginConfigId: 39, pluginConfigTeam: 2, timestamp: expect.any(Number), type: 'exportEventsWithRetry', }) - const jobPayload = mockEnqueue.mock.calls[0][0].payload + const jobPayload = mockEnqueue.mock.calls[0][1].payload // run the job directly await vm.tasks.job['exportEventsWithRetry'].exec(jobPayload) // enqueued again expect(mockEnqueue).toHaveBeenCalledTimes(2) - expect(mockEnqueue).toHaveBeenLastCalledWith({ + expect(mockEnqueue).toHaveBeenLastCalledWith('pluginJob', { payload: { batch: jobPayload.batch, batchId: jobPayload.batchId, retriesPerformedSoFar: 2 }, pluginConfigId: 39, pluginConfigTeam: 2, timestamp: expect.any(Number), type: 'exportEventsWithRetry', }) - const jobPayload2 = mockEnqueue.mock.calls[1][0].payload + const jobPayload2 = mockEnqueue.mock.calls[1][1].payload // run the job a second time await vm.tasks.job['exportEventsWithRetry'].exec(jobPayload2) @@ -1195,7 +1195,7 @@ describe('vm tests', () => { // won't retry after the nth time where n = MAXIMUM_RETRIES for (let i = 2; i < 20; i++) { - const lastPayload = mockEnqueue.mock.calls[mockEnqueue.mock.calls.length - 1][0].payload + const lastPayload = mockEnqueue.mock.calls[mockEnqueue.mock.calls.length - 1][1].payload await vm.tasks.job['exportEventsWithRetry'].exec(lastPayload) expect(mockEnqueue).toHaveBeenCalledTimes(i > MAXIMUM_RETRIES ? MAXIMUM_RETRIES : i) } From cca14f696fa74f0f764d0937a181bfff0b4fccec Mon Sep 17 00:00:00 2001 From: Ben White Date: Mon, 11 Jul 2022 19:35:21 +0200 Subject: [PATCH 022/213] fix: Breakdown csv exports (#10711) * fix: CSV exports --- posthog/tasks/exports/csv_exporter.py | 57 +- .../exports/test/csv_renders/funnels.json | 38 + .../test/csv_renders/funnels_breakdown.json | 112 ++ .../exports/test/csv_renders/people.json | 70 ++ .../exports/test/csv_renders/retention.json | 364 ++++++ .../test/csv_renders/retention_breakdown.json | 118 ++ .../exports/test/csv_renders/trends.json | 1120 +++++++++++++++++ .../test/csv_renders/trends_formula.json | 36 + .../exports/test/test_csv_exporter_renders.py | 46 + 9 files changed, 1938 insertions(+), 23 deletions(-) create mode 100644 posthog/tasks/exports/test/csv_renders/funnels.json create mode 100644 posthog/tasks/exports/test/csv_renders/funnels_breakdown.json create mode 100644 posthog/tasks/exports/test/csv_renders/people.json create mode 100644 posthog/tasks/exports/test/csv_renders/retention.json create mode 100644 posthog/tasks/exports/test/csv_renders/retention_breakdown.json create mode 100644 posthog/tasks/exports/test/csv_renders/trends.json create mode 100644 posthog/tasks/exports/test/csv_renders/trends_formula.json create mode 100644 posthog/tasks/exports/test/test_csv_exporter_renders.py diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index 2f78c8498aadc9..7522eb23f7e197 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -61,7 +61,28 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: items = data["result"] first_result = items[0] - if first_result.get("appearances") and first_result.get("person"): + if isinstance(first_result, list) or first_result.get("action_id"): + csv_rows = [] + multiple_items = items if isinstance(first_result, list) else [items] + # FUNNELS LIKE + + for items in multiple_items: + csv_rows.extend( + [ + { + "name": x["custom_name"] or x["action_id"], + "breakdown_value": "::".join(x.get("breakdown_value", [])), + "action_id": x["action_id"], + "count": x["count"], + "median_conversion_time (seconds)": x["median_conversion_time"], + "average_conversion_time (seconds)": x["average_conversion_time"], + } + for x in items + ] + ) + + return csv_rows + elif first_result.get("appearances") and first_result.get("person"): # RETENTION PERSONS LIKE csv_rows = [] for item in items: @@ -78,9 +99,16 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: csv_rows = [] # RETENTION LIKE for item in items: - line = {"cohort": item["date"], "cohort size": item["values"][0]["count"]} - for index, data in enumerate(item["values"]): - line[items[index]["label"]] = data["count"] + if item.get("date"): + # Dated means we create a grid + line = {"cohort": item["date"], "cohort size": item["values"][0]["count"]} + for index, data in enumerate(item["values"]): + line[items[index]["label"]] = data["count"] + else: + # Otherwise we just specify "Period" for titles + line = {"cohort": item["label"], "cohort size": item["values"][0]["count"]} + for index, data in enumerate(item["values"]): + line[f"Period {index}"] = data["count"] csv_rows.append(line) return csv_rows @@ -89,7 +117,7 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: # TRENDS LIKE for item in items: line = {"series": item["label"]} - if item.get("action").get("custom_name"): + if item.get("action", {}).get("custom_name"): line["custom name"] = item.get("action").get("custom_name") if item.get("aggregated_value"): line["total count"] = item.get("aggregated_value") @@ -99,20 +127,6 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: csv_rows.append(line) - return csv_rows - elif first_result.get("action_id"): - # FUNNELS LIKE - csv_rows = [ - { - "name": x["custom_name"] or x["action_id"], - "action_id": x["action_id"], - "count": x["count"], - "median_conversion_time (seconds)": x["median_conversion_time"], - "average_conversion_time (seconds)": x["average_conversion_time"], - } - for x in items - ] - return csv_rows else: return items @@ -141,11 +155,8 @@ def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: response = requests.request( method=method.lower(), url=url, json=body, headers={"Authorization": f"Bearer {access_token}"}, ) - # Figure out how to handle funnel polling.... - data = response.json() - csv_rows = _convert_response_to_csv_data(data) all_csv_rows = all_csv_rows + csv_rows @@ -220,6 +231,6 @@ def export_csv(exported_asset: ExportedAsset, limit: Optional[int] = None, max_l scope.set_tag("celery_task", "csv_export") capture_exception(e) - logger.error("csv_exporter.failed", exception=e) + logger.error("csv_exporter.failed", exception=e, exc_info=True) statsd.incr("csv_exporter.failed", tags={"team_id": team_id}) raise e diff --git a/posthog/tasks/exports/test/csv_renders/funnels.json b/posthog/tasks/exports/test/csv_renders/funnels.json new file mode 100644 index 00000000000000..8604bea44fc063 --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/funnels.json @@ -0,0 +1,38 @@ +{ + "csv_rows": [ + "name,breakdown_value,action_id,count,median_conversion_time (seconds),average_conversion_time (seconds)", + "$pageview,,$pageview,5193,,", + "2671,,2671,88,30,473.5640151515152", + "" + ], + "response": { + "result": [ + { + "action_id": "$pageview", + "name": "$pageview", + "custom_name": null, + "order": 0, + "people": [], + "count": 5193, + "type": "events", + "average_conversion_time": null, + "median_conversion_time": null, + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": null + }, + { + "action_id": "2671", + "name": "User Signed Up", + "custom_name": null, + "order": 1, + "people": [], + "count": 88, + "type": "actions", + "average_conversion_time": 473.5640151515152, + "median_conversion_time": 30, + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": "/api/person/funnel/?foo=bar" + } + ] + } +} diff --git a/posthog/tasks/exports/test/csv_renders/funnels_breakdown.json b/posthog/tasks/exports/test/csv_renders/funnels_breakdown.json new file mode 100644 index 00000000000000..d62dfc4d4af301 --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/funnels_breakdown.json @@ -0,0 +1,112 @@ +{ + "csv_rows": [ + "name,breakdown_value,action_id,count,median_conversion_time (seconds),average_conversion_time (seconds)", + "$pageview,slider_on,$pageview,5193,,", + "2671,slider_on,2671,88,30,473.5640151515152", + "$pageview,,$pageview,746,,", + "2671,,2671,1,1283,1283.25", + "$pageview,control,$pageview,5109,,", + "2671,control,2671,75,32,4309.385604617603", + "" + ], + "response": { + "result": [ + [ + { + "action_id": "$pageview", + "name": "$pageview", + "custom_name": null, + "order": 0, + "people": [], + "count": 5193, + "type": "events", + "average_conversion_time": null, + "median_conversion_time": null, + "breakdown": ["slider_on"], + "breakdown_value": ["slider_on"], + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": null + }, + { + "action_id": "2671", + "name": "User Signed Up", + "custom_name": null, + "order": 1, + "people": [], + "count": 88, + "type": "actions", + "average_conversion_time": 473.5640151515152, + "median_conversion_time": 30, + "breakdown": ["slider_on"], + "breakdown_value": ["slider_on"], + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": "/api/person/funnel/?foo=bar" + } + ], + [ + { + "action_id": "$pageview", + "name": "$pageview", + "custom_name": null, + "order": 0, + "people": [], + "count": 746, + "type": "events", + "average_conversion_time": null, + "median_conversion_time": null, + "breakdown": [""], + "breakdown_value": [""], + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": null + }, + { + "action_id": "2671", + "name": "User Signed Up", + "custom_name": null, + "order": 1, + "people": [], + "count": 1, + "type": "actions", + "average_conversion_time": 1283.25, + "median_conversion_time": 1283, + "breakdown": [""], + "breakdown_value": [""], + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": "/api/person/funnel/?foo=bar" + } + ], + [ + { + "action_id": "$pageview", + "name": "$pageview", + "custom_name": null, + "order": 0, + "people": [], + "count": 5109, + "type": "events", + "average_conversion_time": null, + "median_conversion_time": null, + "breakdown": ["control"], + "breakdown_value": ["control"], + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": null + }, + { + "action_id": "2671", + "name": "User Signed Up", + "custom_name": null, + "order": 1, + "people": [], + "count": 75, + "type": "actions", + "average_conversion_time": 4309.385604617603, + "median_conversion_time": 32, + "breakdown": ["control"], + "breakdown_value": ["control"], + "converted_people_url": "/api/person/funnel/?foo=bar", + "dropped_people_url": "/api/person/funnel/?foo=bar" + } + ] + ] + } +} diff --git a/posthog/tasks/exports/test/csv_renders/people.json b/posthog/tasks/exports/test/csv_renders/people.json new file mode 100644 index 00000000000000..3ac596640b8902 --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/people.json @@ -0,0 +1,70 @@ +{ + "csv_rows": [ + "created_at,distinct_ids.0,id,name,properties.$browser,properties.$browser_version,properties.$geoip_city_name,properties.$geoip_continent_code,properties.$geoip_continent_name,properties.$geoip_country_code,properties.$geoip_country_name,properties.$geoip_latitude,properties.$geoip_longitude,properties.$geoip_postal_code,properties.$geoip_subdivision_1_code,properties.$geoip_subdivision_1_name,properties.$geoip_time_zone,properties.$initial_browser,properties.$initial_browser_version,properties.$initial_current_url,properties.$initial_device_type,properties.$initial_geoip_city_name,properties.$initial_geoip_continent_code,properties.$initial_geoip_continent_name,properties.$initial_geoip_country_code,properties.$initial_geoip_country_name,properties.$initial_geoip_latitude,properties.$initial_geoip_longitude,properties.$initial_geoip_postal_code,properties.$initial_geoip_subdivision_1_code,properties.$initial_geoip_subdivision_1_name,properties.$initial_geoip_time_zone,properties.$initial_os,properties.$initial_pathname,properties.$initial_referrer,properties.$initial_referring_domain,properties.$os,properties.anonymize_data,properties.billing_plan,properties.completed_onboarding_once,properties.email,properties.email_opt_in,properties.has_password_set,properties.has_social_auth,properties.is_signed_up,properties.joined_at,properties.organization_count,properties.organization_id,properties.project_count,properties.project_id,properties.project_setup_complete,properties.realm,properties.team_member_count_all,uuid", + "2022-07-07T09:33:23.447000Z,1234,1,person@posthog.com,Chrome,103,Sydney,OC,Oceania,AU,Australia,-33.8715,151.2006,2000,NSW,New South Wales,Australia/Sydney,Chrome,103,http://localhost:8000/ingestion,Desktop,Sydney,OC,Oceania,AU,Australia,-33.8715,151.2006,2000,NSW,New South Wales,Australia/Sydney,Mac OS X,/ingestion,http://localhost:8000/signup,localhost:8000,Mac OS X,False,,True,person@posthog.com,True,True,False,True,2022-07-07T09:33:21.934719+00:00,1,0181d801-ba46-0000-734e-ab2e7d2617d9,1,0181d801-bed6-0000-8f2b-e4969cf5a6a3,True,hosted-clickhouse,1,0181d801-c717-0000-ba5e-05262063a1e0", + "" + ], + "response": { + "results": [ + { + "id": 1, + "name": "person@posthog.com", + "distinct_ids": ["1234"], + "properties": { + "$os": "Mac OS X", + "email": "person@posthog.com", + "realm": "hosted-clickhouse", + "$browser": "Chrome", + "joined_at": "2022-07-07T09:33:21.934719+00:00", + "project_id": "0181d801-bed6-0000-8f2b-e4969cf5a6a3", + "$initial_os": "Mac OS X", + "billing_plan": null, + "email_opt_in": true, + "is_signed_up": true, + "project_count": 1, + "anonymize_data": false, + "$geoip_latitude": -33.8715, + "has_social_auth": false, + "organization_id": "0181d801-ba46-0000-734e-ab2e7d2617d9", + "$browser_version": 103, + "$geoip_city_name": "Sydney", + "$geoip_longitude": 151.2006, + "$geoip_time_zone": "Australia/Sydney", + "$initial_browser": "Chrome", + "has_password_set": true, + "social_providers": [], + "$initial_pathname": "/ingestion", + "$initial_referrer": "http://localhost:8000/signup", + "$geoip_postal_code": "2000", + "organization_count": 1, + "$geoip_country_code": "AU", + "$geoip_country_name": "Australia", + "$initial_current_url": "http://localhost:8000/ingestion", + "$initial_device_type": "Desktop", + "$geoip_continent_code": "OC", + "$geoip_continent_name": "Oceania", + "team_member_count_all": 1, + "project_setup_complete": true, + "$initial_geoip_latitude": -33.8715, + "$initial_browser_version": 103, + "$initial_geoip_city_name": "Sydney", + "$initial_geoip_longitude": 151.2006, + "$initial_geoip_time_zone": "Australia/Sydney", + "$geoip_subdivision_1_code": "NSW", + "$geoip_subdivision_1_name": "New South Wales", + "$initial_referring_domain": "localhost:8000", + "completed_onboarding_once": true, + "$initial_geoip_postal_code": "2000", + "$initial_geoip_country_code": "AU", + "$initial_geoip_country_name": "Australia", + "$initial_geoip_continent_code": "OC", + "$initial_geoip_continent_name": "Oceania", + "$initial_geoip_subdivision_1_code": "NSW", + "$initial_geoip_subdivision_1_name": "New South Wales" + }, + "created_at": "2022-07-07T09:33:23.447000Z", + "uuid": "0181d801-c717-0000-ba5e-05262063a1e0" + } + ] + } +} diff --git a/posthog/tasks/exports/test/csv_renders/retention.json b/posthog/tasks/exports/test/csv_renders/retention.json new file mode 100644 index 00000000000000..fc763eadbbdfd5 --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/retention.json @@ -0,0 +1,364 @@ +{ + "csv_rows": [ + "cohort,cohort size,Week 0,Week 1,Week 2,Week 3,Week 4,Week 5,Week 6,Week 7,Week 8,Week 9,Week 10", + "2022-05-01T00:00:00Z,0,0,0,0,0,0,0,0,0,0,0,0", + "2022-05-08T00:00:00Z,0,0,0,0,0,0,0,0,0,0,0,", + "2022-05-15T00:00:00Z,0,0,0,0,0,0,0,0,0,0,,", + "2022-05-22T00:00:00Z,0,0,0,0,0,0,0,0,0,,,", + "2022-05-29T00:00:00Z,0,0,0,0,0,0,0,0,,,,", + "2022-06-05T00:00:00Z,0,0,0,0,0,0,0,,,,,", + "2022-06-12T00:00:00Z,0,0,0,0,0,0,,,,,,", + "2022-06-19T00:00:00Z,0,0,0,0,0,,,,,,,", + "2022-06-26T00:00:00Z,0,0,0,0,,,,,,,,", + "2022-07-03T00:00:00Z,1,1,1,,,,,,,,,", + "2022-07-10T00:00:00Z,0,0,,,,,,,,,,", + "" + ], + "response": { + "result": [ + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 0", + "date": "2022-05-01T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 1", + "date": "2022-05-08T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 2", + "date": "2022-05-15T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 3", + "date": "2022-05-22T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 4", + "date": "2022-05-29T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 5", + "date": "2022-06-05T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 6", + "date": "2022-06-12T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 7", + "date": "2022-06-19T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Week 8", + "date": "2022-06-26T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 1, + "people": [], + "people_url": "/api/person/retention?foo=bar" + }, + { + "count": 1, + "people": [], + "people_url": "/api/person/retention?foo=bar" + } + ], + "label": "Week 9", + "date": "2022-07-03T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + }, + { + "values": [ + { + "count": 0, + "people": [] + } + ], + "label": "Week 10", + "date": "2022-07-10T00:00:00Z", + "people_url": "/api/person/retention?foo=bar" + } + ] + } +} diff --git a/posthog/tasks/exports/test/csv_renders/retention_breakdown.json b/posthog/tasks/exports/test/csv_renders/retention_breakdown.json new file mode 100644 index 00000000000000..6eb90851ceaf1a --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/retention_breakdown.json @@ -0,0 +1,118 @@ +{ + "csv_rows": [ + "cohort,cohort size,Period 0,Period 1,Period 2,Period 3,Period 4,Period 5,Period 6,Period 7,Period 8,Period 9,Period 10", + "Chrome::103,1,1,1,0,0,0,0,0,0,0,0,0", + "Chrome::102,1,1,1,0,0,0,0,0,0,0,0,0", + "" + ], + "response": { + "result": [ + { + "values": [ + { + "count": 1, + "people": [], + "people_url": "/api/person/retention/foo=bar" + }, + { + "count": 1, + "people": [], + "people_url": "/api/person/retention/foo=bar" + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Chrome::103", + "breakdown_values": ["Chrome", "103"], + "people_url": "/api/person/retention/?foo=bar" + }, + { + "values": [ + { + "count": 1, + "people": [], + "people_url": "/api/person/retention/foo=bar" + }, + { + "count": 1, + "people": [], + "people_url": "/api/person/retention/foo=bar" + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + }, + { + "count": 0, + "people": [] + } + ], + "label": "Chrome::102", + "breakdown_values": ["Chrome", "102"], + "people_url": "/api/person/retention/?foo=bar" + } + ] + } +} diff --git a/posthog/tasks/exports/test/csv_renders/trends.json b/posthog/tasks/exports/test/csv_renders/trends.json new file mode 100644 index 00000000000000..15b98a16012b58 --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/trends.json @@ -0,0 +1,1120 @@ +{ + "csv_rows": [ + "series,custom name,1-Apr-2022,1-May-2022,1-Jun-2022,1-Jul-2022", + "pineapple_on_pizza_survey - Belarus,Yes,0,0,0,1", + "pineapple_on_pizza_survey - Canada,Yes,1,1,0,0", + "pineapple_on_pizza_survey - Denmark,Yes,0,2,0,0", + "pineapple_on_pizza_survey - Georgia,Yes,0,0,1,0", + "pineapple_on_pizza_survey - Germany,Yes,3,0,0,0", + "pineapple_on_pizza_survey - India,Yes,1,4,3,1", + "pineapple_on_pizza_survey - Ireland,Yes,0,1,1,0", + "pineapple_on_pizza_survey - Luxembourg,Yes,0,0,1,0", + "pineapple_on_pizza_survey - Netherlands,Yes,0,0,1,0", + "pineapple_on_pizza_survey - Norway,Yes,0,0,1,0", + "pineapple_on_pizza_survey - Poland,Yes,0,1,0,0", + "pineapple_on_pizza_survey - Russia,Yes,1,2,0,0", + "pineapple_on_pizza_survey - Slovenia,Yes,1,1,0,0", + "pineapple_on_pizza_survey - Switzerland,Yes,0,1,0,0", + "pineapple_on_pizza_survey - United Kingdom,Yes,1,2,1,2", + "pineapple_on_pizza_survey - United States,Yes,3,6,4,1", + "pineapple_on_pizza_survey - Australia,No,0,2,0,0", + "pineapple_on_pizza_survey - Canada,No,1,0,0,0", + "pineapple_on_pizza_survey - Denmark,No,1,1,1,0", + "pineapple_on_pizza_survey - Estonia,No,0,0,1,0", + "pineapple_on_pizza_survey - France,No,2,0,0,0", + "pineapple_on_pizza_survey - Germany,No,2,1,1,0", + "pineapple_on_pizza_survey - India,No,0,2,1,2", + "pineapple_on_pizza_survey - Ireland,No,0,0,1,0", + "pineapple_on_pizza_survey - Italy,No,1,1,0,0", + "pineapple_on_pizza_survey - Netherlands,No,0,0,2,0", + "pineapple_on_pizza_survey - New Zealand,No,0,0,1,0", + "pineapple_on_pizza_survey - Nigeria,No,0,0,2,0", + "pineapple_on_pizza_survey - North Macedonia,No,1,0,0,0", + "pineapple_on_pizza_survey - Norway,No,0,0,0,1", + "pineapple_on_pizza_survey - Romania,No,0,0,0,1", + "pineapple_on_pizza_survey - Spain,No,0,1,1,0", + "pineapple_on_pizza_survey - Sweden,No,1,0,0,0", + "pineapple_on_pizza_survey - Switzerland,No,1,1,0,0", + "pineapple_on_pizza_survey - Tunisia,No,0,1,0,0", + "pineapple_on_pizza_survey - United Kingdom,No,1,2,1,1", + "pineapple_on_pizza_survey - United States,No,0,2,1,1", + "" + ], + "response": { + "result": [ + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Belarus", + "count": 1, + "data": [0, 0, 0, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Belarus" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Canada", + "count": 2, + "data": [1, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Canada" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Denmark", + "count": 2, + "data": [0, 2, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Denmark" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Georgia", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Georgia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Germany", + "count": 3, + "data": [3, 0, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Germany" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - India", + "count": 9, + "data": [1, 4, 3, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "India" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Ireland", + "count": 2, + "data": [0, 1, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Ireland" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Luxembourg", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Luxembourg" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Netherlands", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Netherlands" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Norway", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Norway" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Poland", + "count": 1, + "data": [0, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Poland" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Russia", + "count": 3, + "data": [1, 2, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Russia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Slovenia", + "count": 2, + "data": [1, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Slovenia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Switzerland", + "count": 1, + "data": [0, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Switzerland" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - United Kingdom", + "count": 6, + "data": [1, 2, 1, 2], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "United Kingdom" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 0, + "name": "pineapple_on_pizza_survey", + "custom_name": "Yes", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["true"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - United States", + "count": 14, + "data": [3, 6, 4, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "United States" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Australia", + "count": 2, + "data": [0, 2, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Australia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Canada", + "count": 1, + "data": [1, 0, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Canada" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Denmark", + "count": 3, + "data": [1, 1, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Denmark" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Estonia", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Estonia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - France", + "count": 2, + "data": [2, 0, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "France" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Germany", + "count": 4, + "data": [2, 1, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Germany" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - India", + "count": 5, + "data": [0, 2, 1, 2], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "India" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Ireland", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Ireland" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Italy", + "count": 2, + "data": [1, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Italy" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Netherlands", + "count": 2, + "data": [0, 0, 2, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Netherlands" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - New Zealand", + "count": 1, + "data": [0, 0, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "New Zealand" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Nigeria", + "count": 2, + "data": [0, 0, 2, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Nigeria" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - North Macedonia", + "count": 1, + "data": [1, 0, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "North Macedonia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Norway", + "count": 1, + "data": [0, 0, 0, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Norway" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Romania", + "count": 1, + "data": [0, 0, 0, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Romania" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Spain", + "count": 2, + "data": [0, 1, 1, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Spain" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Sweden", + "count": 1, + "data": [1, 0, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Sweden" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Switzerland", + "count": 2, + "data": [1, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Switzerland" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - Tunisia", + "count": 1, + "data": [0, 1, 0, 0], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "Tunisia" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - United Kingdom", + "count": 5, + "data": [1, 2, 1, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "United Kingdom" + }, + { + "action": { + "id": "pineapple_on_pizza_survey", + "type": "events", + "order": 1, + "name": "pineapple_on_pizza_survey", + "custom_name": "No", + "math": "dau", + "math_property": null, + "math_group_type_index": null, + "properties": { + "type": "AND", + "values": [ + { + "key": "does_pineapple_go_on_pizza", + "operator": "exact", + "type": "event", + "value": ["false"] + } + ] + } + }, + "label": "pineapple_on_pizza_survey - United States", + "count": 4, + "data": [0, 2, 1, 1], + "labels": ["1-Apr-2022", "1-May-2022", "1-Jun-2022", "1-Jul-2022"], + "days": ["2022-04-01", "2022-05-01", "2022-06-01", "2022-07-01"], + "breakdown_value": "United States" + } + ] + } +} diff --git a/posthog/tasks/exports/test/csv_renders/trends_formula.json b/posthog/tasks/exports/test/csv_renders/trends_formula.json new file mode 100644 index 00000000000000..3ae145475a518e --- /dev/null +++ b/posthog/tasks/exports/test/csv_renders/trends_formula.json @@ -0,0 +1,36 @@ +{ + "csv_rows": [ + "series,4-Jul-2022,5-Jul-2022,6-Jul-2022,7-Jul-2022,8-Jul-2022,9-Jul-2022,10-Jul-2022,11-Jul-2022", + "Formula (A-B),0,0,0,-12,0,0,0,23", + "" + ], + "response": { + "result": [ + { + "data": [0, 0, 0, -12, 0, 0, 0, 23], + "count": 11, + "labels": [ + "4-Jul-2022", + "5-Jul-2022", + "6-Jul-2022", + "7-Jul-2022", + "8-Jul-2022", + "9-Jul-2022", + "10-Jul-2022", + "11-Jul-2022" + ], + "days": [ + "2022-07-04", + "2022-07-05", + "2022-07-06", + "2022-07-07", + "2022-07-08", + "2022-07-09", + "2022-07-10", + "2022-07-11" + ], + "label": "Formula (A-B)" + } + ] + } +} diff --git a/posthog/tasks/exports/test/test_csv_exporter_renders.py b/posthog/tasks/exports/test/test_csv_exporter_renders.py new file mode 100644 index 00000000000000..417c8643992b25 --- /dev/null +++ b/posthog/tasks/exports/test/test_csv_exporter_renders.py @@ -0,0 +1,46 @@ +import json +import os +from unittest.mock import Mock, patch + +import pytest + +from posthog.models import ExportedAsset +from posthog.models.organization import Organization +from posthog.models.team.team import Team +from posthog.tasks.exports import csv_exporter + +TEST_BUCKET = "Test-Exports" + +directory = os.path.join(os.path.abspath(os.path.dirname(__file__)), "./csv_renders") +fixtures = [] + +for file in os.listdir(directory): + filename = os.fsdecode(file) + if filename.endswith(".json"): + fixtures.append(filename) + + +@pytest.mark.parametrize("filename", fixtures) +@pytest.mark.django_db +@patch("posthog.tasks.exports.csv_exporter.requests.request") +@patch("posthog.tasks.exports.csv_exporter.settings") +def test_csv_rendering(mock_settings, mock_request, filename): + mock_settings.OBJECT_STORAGE_ENABLED = False + org = Organization.objects.create(name="org") + team = Team.objects.create(organization=org, name="team") + + with open(os.path.join(directory, filename)) as f: + fixture = json.loads(f.read()) + + asset = ExportedAsset( + team=team, export_format=ExportedAsset.ExportFormat.CSV, export_context={"path": "/api/literally/anything"}, + ) + asset.save() + + mock = Mock() + mock.json.return_value = fixture["response"] + mock_request.return_value = mock + csv_exporter.export_csv(asset) + csv_rows = asset.content.decode("utf-8").split("\r\n") + + assert csv_rows == fixture["csv_rows"] From 8f50e7b0a104f370e32ff05bfe3a268be750ad77 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 11 Jul 2022 19:15:55 +0100 Subject: [PATCH 023/213] Revert "chore: configure celery logging to use correct structlog config (#10614)" (#10721) This reverts commit 4d314a7c27fa34a9b9aea1507c72415a35c5eb7d. --- posthog/celery.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/posthog/celery.py b/posthog/celery.py index 9008a48a2cc012..a787273b4dcf38 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -4,7 +4,7 @@ from celery import Celery from celery.schedules import crontab -from celery.signals import setup_logging, task_postrun, task_prerun +from celery.signals import task_postrun, task_prerun from django.conf import settings from django.db import connection from django.utils import timezone @@ -43,17 +43,6 @@ UPDATE_CACHED_DASHBOARD_ITEMS_INTERVAL_SECONDS = settings.UPDATE_CACHED_DASHBOARD_ITEMS_INTERVAL_SECONDS -@setup_logging.connect -def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs) -> None: - import logging - - from posthog.settings import logs - - # following instructions from here https://django-structlog.readthedocs.io/en/latest/celery.html - # mypy thinks that there is no `logging.config` but there is ¯\_(ツ)_/¯ - logging.config.dictConfig(logs.LOGGING) # type: ignore - - @app.on_after_configure.connect def setup_periodic_tasks(sender: Celery, **kwargs): # Monitoring tasks From f67a813d3696c0fd010320464ef5b4fbb413b3cc Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 11 Jul 2022 21:15:07 +0100 Subject: [PATCH 024/213] chore: configure celery logging to use correct structlog config (#10722) This reverts commit 8f50e7b0a104f370e32ff05bfe3a268be750ad77. Reimplementing correct structlog config for celery --- posthog/celery.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/posthog/celery.py b/posthog/celery.py index a787273b4dcf38..9008a48a2cc012 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -4,7 +4,7 @@ from celery import Celery from celery.schedules import crontab -from celery.signals import task_postrun, task_prerun +from celery.signals import setup_logging, task_postrun, task_prerun from django.conf import settings from django.db import connection from django.utils import timezone @@ -43,6 +43,17 @@ UPDATE_CACHED_DASHBOARD_ITEMS_INTERVAL_SECONDS = settings.UPDATE_CACHED_DASHBOARD_ITEMS_INTERVAL_SECONDS +@setup_logging.connect +def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs) -> None: + import logging + + from posthog.settings import logs + + # following instructions from here https://django-structlog.readthedocs.io/en/latest/celery.html + # mypy thinks that there is no `logging.config` but there is ¯\_(ツ)_/¯ + logging.config.dictConfig(logs.LOGGING) # type: ignore + + @app.on_after_configure.connect def setup_periodic_tasks(sender: Celery, **kwargs): # Monitoring tasks From 2d72a7a4abff434ce90eab62ad3efc6d215bd56e Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Tue, 12 Jul 2022 09:56:36 +0100 Subject: [PATCH 025/213] chore: only notify sentry of releases that actually happen (#10720) --- .github/workflows/build-and-deploy-prod.yml | 41 +++++++++------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml index f57bafa6af02f5..aa2a9c7b7acd49 100644 --- a/.github/workflows/build-and-deploy-prod.yml +++ b/.github/workflows/build-and-deploy-prod.yml @@ -189,14 +189,6 @@ jobs: # SLACK_USERNAME: Max Hedgehog # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - name: Notify Grafana of deploy for annotations - uses: frankie567/grafana-annotation-action@v1.0.2 - with: - apiHost: https://metrics.posthog.net - apiToken: ${{ secrets.GRAFANA_API_KEY }} - text: Prod deployment of ${{ github.sha }} - ${{ github.event.head_commit.message }} - tags: deployment,github - - name: Trigger PostHog Cloud deployment uses: mvasigh/dispatch-action@main with: @@ -213,6 +205,23 @@ jobs: "context": ${{ toJson(github) }} } + - name: Notify Grafana of deploy for annotations + uses: frankie567/grafana-annotation-action@v1.0.2 + with: + apiHost: https://metrics.posthog.net + apiToken: ${{ secrets.GRAFANA_API_KEY }} + text: Prod deployment of ${{ github.sha }} - ${{ github.event.head_commit.message }} + tags: deployment,github + + - name: Notify Sentry of a production release + uses: getsentry/action-release@v1 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: posthog2 + SENTRY_PROJECT: posthog + with: + environment: production + # TODO: Bring back once https://github.com/rtCamp/action-slack-notify/issues/126 is resolved # slack: # name: Notify Slack of start of deploy @@ -229,19 +238,3 @@ jobs: # SLACK_TITLE: Message # SLACK_USERNAME: Max Hedgehog # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - sentry: - name: Notify Sentry of a production release - runs-on: ubuntu-20.04 - if: github.repository == 'PostHog/posthog' - steps: - - name: Checkout master - uses: actions/checkout@v2 - - name: Notify Sentry - uses: getsentry/action-release@v1 - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_ORG: posthog2 - SENTRY_PROJECT: posthog - with: - environment: production From 9d83892a3604e460e983503872ff910ff5a655ba Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Tue, 12 Jul 2022 10:37:31 +0100 Subject: [PATCH 026/213] Revert "chore: only notify sentry of releases that actually happen (#10720)" (#10726) This reverts commit 2d72a7a4abff434ce90eab62ad3efc6d215bd56e. --- .github/workflows/build-and-deploy-prod.yml | 41 ++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml index aa2a9c7b7acd49..f57bafa6af02f5 100644 --- a/.github/workflows/build-and-deploy-prod.yml +++ b/.github/workflows/build-and-deploy-prod.yml @@ -189,6 +189,14 @@ jobs: # SLACK_USERNAME: Max Hedgehog # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + - name: Notify Grafana of deploy for annotations + uses: frankie567/grafana-annotation-action@v1.0.2 + with: + apiHost: https://metrics.posthog.net + apiToken: ${{ secrets.GRAFANA_API_KEY }} + text: Prod deployment of ${{ github.sha }} - ${{ github.event.head_commit.message }} + tags: deployment,github + - name: Trigger PostHog Cloud deployment uses: mvasigh/dispatch-action@main with: @@ -205,23 +213,6 @@ jobs: "context": ${{ toJson(github) }} } - - name: Notify Grafana of deploy for annotations - uses: frankie567/grafana-annotation-action@v1.0.2 - with: - apiHost: https://metrics.posthog.net - apiToken: ${{ secrets.GRAFANA_API_KEY }} - text: Prod deployment of ${{ github.sha }} - ${{ github.event.head_commit.message }} - tags: deployment,github - - - name: Notify Sentry of a production release - uses: getsentry/action-release@v1 - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} - SENTRY_ORG: posthog2 - SENTRY_PROJECT: posthog - with: - environment: production - # TODO: Bring back once https://github.com/rtCamp/action-slack-notify/issues/126 is resolved # slack: # name: Notify Slack of start of deploy @@ -238,3 +229,19 @@ jobs: # SLACK_TITLE: Message # SLACK_USERNAME: Max Hedgehog # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + sentry: + name: Notify Sentry of a production release + runs-on: ubuntu-20.04 + if: github.repository == 'PostHog/posthog' + steps: + - name: Checkout master + uses: actions/checkout@v2 + - name: Notify Sentry + uses: getsentry/action-release@v1 + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: posthog2 + SENTRY_PROJECT: posthog + with: + environment: production From 89c17e6e2f6b7b81c3e3b9552dd4ae6fae27ba22 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Tue, 12 Jul 2022 11:09:16 +0100 Subject: [PATCH 027/213] chore: only run sentry notification if build succeeds (#10728) --- .github/workflows/build-and-deploy-prod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml index f57bafa6af02f5..99d2ae8f594c0f 100644 --- a/.github/workflows/build-and-deploy-prod.yml +++ b/.github/workflows/build-and-deploy-prod.yml @@ -234,6 +234,7 @@ jobs: name: Notify Sentry of a production release runs-on: ubuntu-20.04 if: github.repository == 'PostHog/posthog' + needs: build steps: - name: Checkout master uses: actions/checkout@v2 From 7b271ac89bd16a781a066f2c6f3a1f6c5b65e3f9 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Tue, 12 Jul 2022 11:25:10 +0100 Subject: [PATCH 028/213] chore: re-enable slack notification of deploy start (#10729) --- .github/workflows/build-and-deploy-prod.yml | 31 ++++++++++----------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml index 99d2ae8f594c0f..fc56b8af2f0bf1 100644 --- a/.github/workflows/build-and-deploy-prod.yml +++ b/.github/workflows/build-and-deploy-prod.yml @@ -213,22 +213,21 @@ jobs: "context": ${{ toJson(github) }} } - # TODO: Bring back once https://github.com/rtCamp/action-slack-notify/issues/126 is resolved - # slack: - # name: Notify Slack of start of deploy - # runs-on: ubuntu-20.04 - # if: github.repository == 'posthog/posthog' - # steps: - # - name: Notify Platform team on slack - # uses: rtCamp/action-slack-notify@v2 - # env: - # SLACK_CHANNEL: platform-bots - # SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff' - # SLACK_ICON: https://github.com/posthog.png?size=48 - # SLACK_MESSAGE: 'Production Cloud Deploy Beginning :rocket: - ${{ github.event.head_commit.message }}' - # SLACK_TITLE: Message - # SLACK_USERNAME: Max Hedgehog - # SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + slack: + name: Notify Slack of start of deploy + runs-on: ubuntu-20.04 + if: github.repository == 'posthog/posthog' + steps: + - name: Notify Platform team on slack + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_CHANNEL: platform-bots + SLACK_COLOR: ${{ job.status }} # or a specific color like 'good' or '#ff00ff' + SLACK_ICON: https://github.com/posthog.png?size=48 + SLACK_MESSAGE: 'Production Cloud Deploy Beginning :rocket: - ${{ github.event.head_commit.message }}' + SLACK_TITLE: Message + SLACK_USERNAME: Max Hedgehog + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} sentry: name: Notify Sentry of a production release From 4d0e00460f3322f63796af44e5ce6582e198c3a2 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Tue, 12 Jul 2022 12:12:59 +0100 Subject: [PATCH 029/213] chore: upgrade celery past 4.4.4 (#10732) --- requirements.in | 6 +++--- requirements.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.in b/requirements.in index 7d18006a8b79f7..56b21636937ef1 100644 --- a/requirements.in +++ b/requirements.in @@ -5,9 +5,9 @@ # - `pip-compile --rebuild requirements-dev.in` # django-rest-hooks@ git+https://github.com/zapier/django-rest-hooks.git@v1.6.0 -amqp==2.5.2 +amqp==2.6.0 boto3==1.21.29 -celery==4.4.2 +celery==4.4.7 celery-redbeat==2.0.0 clickhouse-driver==0.2.1 clickhouse-pool==0.5.3 @@ -39,7 +39,7 @@ importlib-metadata==1.6.0 infi-clickhouse-orm@ git+https://github.com/PostHog/infi.clickhouse_orm@37722f350f3b449bbcd6564917c436b0d93e796f kafka-python==2.0.2 kafka-helper==0.2 -kombu==4.6.8 +kombu==4.6.10 lzstring==1.0.4 numpy==1.21.4 parso==0.8.1 diff --git a/requirements.txt b/requirements.txt index af6f43de4561fa..100c033f830f3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile requirements.in # -amqp==2.5.2 +amqp==2.6.0 # via # -r requirements.in # kombu @@ -29,7 +29,7 @@ botocore==1.24.46 # via # boto3 # s3transfer -celery==4.4.2 +celery==4.4.7 # via # -r requirements.in # celery-redbeat @@ -166,7 +166,7 @@ kafka-helper==0.2 # via -r requirements.in kafka-python==2.0.2 # via -r requirements.in -kombu==4.6.8 +kombu==4.6.10 # via # -r requirements.in # celery From 7fdb2cac9738dfeb42f65406d816d5d1ccc6f97c Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:20:11 +0000 Subject: [PATCH 030/213] fix: show plugin jobs on the UI (#10718) * fix: show plugin jobs on the UI * Update frontend/src/scenes/plugins/edit/PluginDrawer.tsx * fix type error --- frontend/src/scenes/plugins/edit/PluginDrawer.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/scenes/plugins/edit/PluginDrawer.tsx b/frontend/src/scenes/plugins/edit/PluginDrawer.tsx index 21b08b644e9678..c7e0d541753fe4 100644 --- a/frontend/src/scenes/plugins/edit/PluginDrawer.tsx +++ b/frontend/src/scenes/plugins/edit/PluginDrawer.tsx @@ -300,7 +300,8 @@ export function PluginDrawer(): JSX.Element { {!!( editingPlugin.pluginConfig.id && editingPlugin.capabilities?.jobs?.length && - editingPlugin.public_jobs?.length + editingPlugin.public_jobs && + Object.keys(editingPlugin.public_jobs).length ) && ( Date: Tue, 12 Jul 2022 11:09:37 -0400 Subject: [PATCH 031/213] fix: session analysis check (#10738) --- frontend/src/scenes/insights/insightLogic.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/scenes/insights/insightLogic.ts b/frontend/src/scenes/insights/insightLogic.ts index 8a6a1c97295343..e889d456715e71 100644 --- a/frontend/src/scenes/insights/insightLogic.ts +++ b/frontend/src/scenes/insights/insightLogic.ts @@ -14,7 +14,6 @@ import { InsightType, ItemMode, SetInsightOptions, - PropertyFilter, } from '~/types' import { captureInternalMetric } from 'lib/internalMetrics' import { router } from 'kea-router' @@ -630,7 +629,7 @@ export const insightLogic = kea({ return entity.math_property === '$session_duration' }) const using_entity_session_property_filter = entities.some((entity) => { - return entity.properties?.some((property: PropertyFilter) => property.type === 'session') + return parseProperties(entity.properties).some((property) => property.type === 'session') }) const using_global_session_property_filter = parseProperties(filters.properties).some( (property) => property.type === 'session' From 49169a6c421b7f0912b4c4a2fe9d4e8f111fd80e Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Tue, 12 Jul 2022 15:46:13 +0000 Subject: [PATCH 032/213] fix: connect to correct db for graphile when enqueuing UI jobs from django (#10736) * fix: connect to correct db for graphile when enqueuing UI jobs from django * handle graphile url missing * handle connections later * fix * again * fix test --- posthog/api/plugin.py | 3 ++- posthog/api/test/test_plugin.py | 6 +++--- posthog/settings/data_stores.py | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index 12ed82d1facbdd..93595eb085c295 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -6,7 +6,7 @@ from dateutil.relativedelta import relativedelta from django.core.exceptions import ObjectDoesNotExist from django.core.files.uploadedfile import UploadedFile -from django.db import connection +from django.db import connections from django.db.models import Q from django.http import HttpResponse from django.utils.encoding import smart_str @@ -589,6 +589,7 @@ def job(self, request: request.Request, **kwargs): sql = f"SELECT graphile_worker.add_job('pluginJob', %s)" params = [payload_json] try: + connection = connections["graphile"] if "graphile" in connections else connections["default"] with connection.cursor() as cursor: cursor.execute(sql, params) except Exception as e: diff --git a/posthog/api/test/test_plugin.py b/posthog/api/test/test_plugin.py index 61d9413a5337f5..45c5cdbe067639 100644 --- a/posthog/api/test/test_plugin.py +++ b/posthog/api/test/test_plugin.py @@ -1026,8 +1026,8 @@ def test_create_plugin_config_with_secrets(self, mock_get, mock_reload): plugin_config = PluginConfig.objects.get(plugin=plugin_id) self.assertEqual(plugin_config.config, {"bar": "a new very secret value"}) - @patch("posthog.api.plugin.connection") - def test_job_trigger(self, db_connection, mock_get, mock_reload): + @patch("posthog.api.plugin.connections") + def test_job_trigger(self, db_connections, mock_get, mock_reload): response = self.client.post( "/api/organizations/@current/plugins/", {"url": "https://github.com/PostHog/helloworldplugin"} ) @@ -1044,7 +1044,7 @@ def test_job_trigger(self, db_connection, mock_get, mock_reload): format="json", ) self.assertEqual(response.status_code, 200) - execute_fn = db_connection.cursor().__enter__().execute + execute_fn = db_connections["default"].cursor().__enter__().execute self.assertEqual(execute_fn.call_count, 1) expected_sql = "SELECT graphile_worker.add_job('pluginJob', %s)" expected_params = [ diff --git a/posthog/settings/data_stores.py b/posthog/settings/data_stores.py index 91fd6ee2252a33..dbc2d8c0ad99fa 100644 --- a/posthog/settings/data_stores.py +++ b/posthog/settings/data_stores.py @@ -18,6 +18,8 @@ # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases +JOB_QUEUE_GRAPHILE_URL = os.getenv("JOB_QUEUE_GRAPHILE_URL") + if TEST or DEBUG: PG_HOST = os.getenv("PGHOST", "localhost") PG_USER = os.getenv("PGUSER", "posthog") @@ -76,6 +78,9 @@ f'The environment vars "DATABASE_URL" or "POSTHOG_DB_NAME" are absolutely required to run this software' ) +if JOB_QUEUE_GRAPHILE_URL: + DATABASES["graphile"] = dj_database_url.config(default=JOB_QUEUE_GRAPHILE_URL, conn_max_age=600) + # Clickhouse Settings CLICKHOUSE_TEST_DB = "posthog_test" From 465343102bedca089f76d65c3f519f4b644be084 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Tue, 12 Jul 2022 17:54:06 +0000 Subject: [PATCH 033/213] feat: make TimeoutError retriable (#10747) --- plugin-server/src/worker/vm/vm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-server/src/worker/vm/vm.ts b/plugin-server/src/worker/vm/vm.ts index e855694c464662..0d87cae1df3455 100644 --- a/plugin-server/src/worker/vm/vm.ts +++ b/plugin-server/src/worker/vm/vm.ts @@ -16,7 +16,7 @@ import { transformCode } from './transforms' import { upgradeExportEvents } from './upgrades/export-events' import { addHistoricalEventsExportCapability } from './upgrades/historical-export/export-historical-events' -export class TimeoutError extends Error { +export class TimeoutError extends RetryError { name = 'TimeoutError' caller?: string = undefined From c29504ed0138b02cf103291f50828f5ec97eab67 Mon Sep 17 00:00:00 2001 From: Eric Duong Date: Tue, 12 Jul 2022 13:58:40 -0400 Subject: [PATCH 034/213] fix(cohort): cohort performed event multiple overwriting param (#10746) * fix: cohort performed event multiple overwriting param * fix: change test name --- .../queries/test/test_cohort_query.py | 77 +++++++++++++++++++ posthog/queries/foss_cohort_query.py | 5 +- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/ee/clickhouse/queries/test/test_cohort_query.py b/ee/clickhouse/queries/test/test_cohort_query.py index 35492be1c5bbbd..a1c5dff04acdfd 100644 --- a/ee/clickhouse/queries/test/test_cohort_query.py +++ b/ee/clickhouse/queries/test/test_cohort_query.py @@ -325,6 +325,83 @@ def test_performed_event_lte_1_times(self): self.assertEqual(set([p2.uuid]), set([r[0] for r in res])) + def test_can_handle_many_performed_multiple_filters(self): + p1 = _create_person( + team_id=self.team.pk, distinct_ids=["p1"], properties={"name": "test", "email": "test@posthog.com"} + ) + _create_event( + team=self.team, + event="$pageview", + properties={}, + distinct_id="p1", + timestamp=datetime.now() - timedelta(hours=9), + ) + + p2 = _create_person( + team_id=self.team.pk, distinct_ids=["p2"], properties={"name": "test", "email": "test@posthog.com"} + ) + _create_event( + team=self.team, + event="$pageview", + properties={}, + distinct_id="p2", + timestamp=datetime.now() - timedelta(hours=9), + ) + + p3 = _create_person( + team_id=self.team.pk, distinct_ids=["p3"], properties={"name": "test3", "email": "test3@posthog.com"} + ) + _create_event( + team=self.team, + event="$pageview", + properties={}, + distinct_id="p3", + timestamp=datetime.now() - timedelta(hours=9), + ) + _create_event( + team=self.team, + event="$pageview", + properties={}, + distinct_id="p3", + timestamp=datetime.now() - timedelta(hours=8), + ) + flush_persons_and_events() + + filter = Filter( + data={ + "properties": { + "type": "OR", + "values": [ + { + "key": "$pageview", + "event_type": "events", + "operator": "eq", + "operator_value": 1, + "time_value": 1, + "time_interval": "week", + "value": "performed_event_multiple", + "type": "behavioral", + }, + { + "key": "$pageview", + "event_type": "events", + "operator": "eq", + "operator_value": 2, + "time_value": 1, + "time_interval": "week", + "value": "performed_event_multiple", + "type": "behavioral", + }, + ], + }, + } + ) + + q, params = CohortQuery(filter=filter, team=self.team).get_query() + res = sync_execute(q, params) + + self.assertEqual(set([p1.uuid, p2.uuid, p3.uuid]), set([r[0] for r in res])) + def test_performed_event_zero_times_(self): filter = Filter( data={ diff --git a/posthog/queries/foss_cohort_query.py b/posthog/queries/foss_cohort_query.py index 26f54c95f10c00..c25fc92454699d 100644 --- a/posthog/queries/foss_cohort_query.py +++ b/posthog/queries/foss_cohort_query.py @@ -460,16 +460,17 @@ def get_performed_event_multiple(self, prop: Property, prepend: str, idx: int) - date_value = parse_and_validate_positive_integer(prop.time_value, "time_value") date_interval = validate_interval(prop.time_interval) date_param = f"{prepend}_date_{idx}" + operator_value_param = f"{prepend}_operator_value_{idx}" self._check_earliest_date((date_value, date_interval)) - field = f"countIf(timestamp > now() - INTERVAL %({date_param})s {date_interval} AND timestamp < now() AND {entity_query}) {get_count_operator(prop.operator)} %(operator_value)s AS {column_name}" + field = f"countIf(timestamp > now() - INTERVAL %({date_param})s {date_interval} AND timestamp < now() AND {entity_query}) {get_count_operator(prop.operator)} %({operator_value_param})s AS {column_name}" self._fields.append(field) # Negation is handled in the where clause to ensure the right result if a full join occurs where the joined person did not perform the event return ( f"{'NOT' if prop.negation else ''} {column_name}", - {"operator_value": count, f"{date_param}": date_value, **entity_params}, + {f"{operator_value_param}": count, f"{date_param}": date_value, **entity_params}, ) def _determine_should_join_distinct_ids(self) -> None: From b7461b146f690c29f8c3921bd57cb2fe112eb472 Mon Sep 17 00:00:00 2001 From: Chris Clark Date: Tue, 12 Jul 2022 11:23:26 -0700 Subject: [PATCH 035/213] Fixes typo in edit annotation modal (#10748) * Fix typo * chore: empty Co-authored-by: eric --- frontend/src/scenes/annotations/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/scenes/annotations/index.tsx b/frontend/src/scenes/annotations/index.tsx index 61ffa340b1da8e..66e59f32b65cbe 100644 --- a/frontend/src/scenes/annotations/index.tsx +++ b/frontend/src/scenes/annotations/index.tsx @@ -231,7 +231,7 @@ function CreateAnnotationModal(props: CreateAnnotationModalProps): JSX.Element { closable={false} visible={props.visible} onCancel={props.onCancel} - title={modalMode === ModalMode.CREATE ? 'Create annotation' : 'Edit ennotation'} + title={modalMode === ModalMode.CREATE ? 'Create annotation' : 'Edit annotation'} > {modalMode === ModalMode.CREATE ? ( From 763d9d824809bf90aea875c9452ad42a3731d50b Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Tue, 12 Jul 2022 16:54:20 -0700 Subject: [PATCH 036/213] fix(recordings): update default date limit (#10749) --- .../scenes/session-recordings/sessionRecordingsTableLogic.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/scenes/session-recordings/sessionRecordingsTableLogic.ts b/frontend/src/scenes/session-recordings/sessionRecordingsTableLogic.ts index 90348bc70ebee1..b08ff70328b000 100644 --- a/frontend/src/scenes/session-recordings/sessionRecordingsTableLogic.ts +++ b/frontend/src/scenes/session-recordings/sessionRecordingsTableLogic.ts @@ -184,7 +184,7 @@ export const sessionRecordingsTableLogic = kea( }, ], fromDate: [ - dayjs().subtract(7, 'days').format('YYYY-MM-DD') as null | string, + dayjs().subtract(21, 'days').format('YYYY-MM-DD') as null | string, { setDateRange: (_, { incomingFromDate }) => incomingFromDate ?? null, }, From 3f1f2418c2d726d62b542b9613a7c20e0d63c868 Mon Sep 17 00:00:00 2001 From: Ben White Date: Wed, 13 Jul 2022 09:23:40 +0200 Subject: [PATCH 037/213] fix: Links to new Slack documentation (#10752) --- .../lib/components/Subscriptions/views/EditSubscription.tsx | 4 ++-- frontend/src/scenes/project/Settings/SlackIntegration.tsx | 2 +- frontend/src/scenes/project/Settings/WebhookIntegration.tsx | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx b/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx index 308178c6525eb9..1bd4f058552a58 100644 --- a/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx +++ b/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx @@ -280,7 +280,7 @@ export function EditSubscription({
Private channels are only shown if you have{' '} @@ -301,7 +301,7 @@ export function EditSubscription({ to the channel otherwise Subscriptions will fail to be delivered.{' '} diff --git a/frontend/src/scenes/project/Settings/SlackIntegration.tsx b/frontend/src/scenes/project/Settings/SlackIntegration.tsx index c0644cde166a65..de55db1e9b85bb 100644 --- a/frontend/src/scenes/project/Settings/SlackIntegration.tsx +++ b/frontend/src/scenes/project/Settings/SlackIntegration.tsx @@ -35,7 +35,7 @@ export function SlackIntegration(): JSX.Element { Integrate with Slack directly to get more advanced options such as sending webhook events to{' '} different channels and subscribing to an Insight or Dashboard for regular reports to Slack channels of your choice. Guidance on integrating with Slack available{' '} - in our docs. + in our docs.

diff --git a/frontend/src/scenes/project/Settings/WebhookIntegration.tsx b/frontend/src/scenes/project/Settings/WebhookIntegration.tsx index 9cf1397a1b9c99..dba71e86bedf7f 100644 --- a/frontend/src/scenes/project/Settings/WebhookIntegration.tsx +++ b/frontend/src/scenes/project/Settings/WebhookIntegration.tsx @@ -22,7 +22,7 @@ export function WebhookIntegration(): JSX.Element { Send notifications when selected actions are performed by users.
Guidance on integrating with webhooks available in our docs,{' '} - for Slack and{' '} + for Slack and{' '} for Microsoft Teams. Discord is also supported.

From 8915785f0f9d9828c56bcf02720cf60d6d2ff012 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 13 Jul 2022 09:12:02 +0100 Subject: [PATCH 038/213] chore: removes unnecessary references to shared_dashboard.html (#10742) --- .github/actions/run-backend-tests/action.yml | 1 - .github/workflows/ci-async-migrations.yml | 1 - .github/workflows/ci-backend.yml | 1 - ee/bin/docker-ch-test | 1 - 4 files changed, 4 deletions(-) diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index 765f8b7d2fcaf5..40d9bcc7b05d73 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -75,7 +75,6 @@ runs: mkdir -p frontend/dist touch frontend/dist/index.html touch frontend/dist/layout.html - touch frontend/dist/shared_dashboard.html touch frontend/dist/exporter.html - name: Wait for Clickhouse & Kafka diff --git a/.github/workflows/ci-async-migrations.yml b/.github/workflows/ci-async-migrations.yml index e34ef9e35e92aa..35d72cff6aa8d6 100644 --- a/.github/workflows/ci-async-migrations.yml +++ b/.github/workflows/ci-async-migrations.yml @@ -71,7 +71,6 @@ jobs: mkdir -p frontend/dist touch frontend/dist/index.html touch frontend/dist/layout.html - touch frontend/dist/shared_dashboard.html touch frontend/dist/exporter.html - name: Wait for Clickhouse & Kafka diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index c5ac22f49d11d7..3d5a4103f00c3e 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -398,7 +398,6 @@ jobs: python manage.py collectstatic --noinput touch frontend/dist/index.html touch frontend/dist/layout.html - touch frontend/dist/shared_dashboard.html touch frontend/dist/exporter.html - name: Run cloud tests (posthog-cloud) diff --git a/ee/bin/docker-ch-test b/ee/bin/docker-ch-test index f1ffb1d45515aa..9bf7adda87882c 100755 --- a/ee/bin/docker-ch-test +++ b/ee/bin/docker-ch-test @@ -6,6 +6,5 @@ set -e mkdir -p frontend/dist touch frontend/dist/index.html touch frontend/dist/layout.html -touch frontend/dist/shared_dashboard.html touch frontend/dist/exporter.html pytest ee From 0c65195a9d1f23690a64457130f4ca1709ded4fc Mon Sep 17 00:00:00 2001 From: Guido Iaquinti <4038041+guidoiaquinti@users.noreply.github.com> Date: Wed, 13 Jul 2022 10:15:21 +0200 Subject: [PATCH 039/213] chore: upgrade sentry-sdk to 1.7.0 (#10743) --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 56b21636937ef1..8f3b4120e6649f 100644 --- a/requirements.in +++ b/requirements.in @@ -56,7 +56,7 @@ redis==3.4.1 requests==2.25.1 requests-oauthlib==1.3.0 selenium==4.1.5 -sentry-sdk==1.3.1 +sentry-sdk==1.7.0 semantic_version==2.8.5 slack_sdk==3.17.1 social-auth-app-django==5.0.0 diff --git a/requirements.txt b/requirements.txt index 100c033f830f3b..2c191b527feb70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -273,7 +273,7 @@ selenium==4.1.5 # via -r requirements.in semantic-version==2.8.5 # via -r requirements.in -sentry-sdk==1.3.1 +sentry-sdk==1.7.0 # via -r requirements.in six==1.14.0 # via From 599da6d99dade3426a49511d3bd343630c8af42c Mon Sep 17 00:00:00 2001 From: Guido Iaquinti <4038041+guidoiaquinti@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:17:42 +0200 Subject: [PATCH 040/213] chore(sentry-sdk): add traces (#10745) --- posthog/settings/sentry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/posthog/settings/sentry.py b/posthog/settings/sentry.py index 3c032a040b629a..f0036296502c3d 100644 --- a/posthog/settings/sentry.py +++ b/posthog/settings/sentry.py @@ -16,8 +16,10 @@ sentry_logging = sentry_logging = LoggingIntegration(level=logging.INFO, event_level=None) sentry_sdk.init( dsn=os.environ["SENTRY_DSN"], + environment=os.getenv("SENTRY_ENVIRONMENT", "production"), integrations=[DjangoIntegration(), CeleryIntegration(), RedisIntegration(), sentry_logging], request_bodies="always", + sample_rate=1.0, # Configures the sample rate for error events, in the range of 0.0 to 1.0. The default is 1.0 which means that 100% of error events are sent. If set to 0.1 only 10% of error events will be sent. Events are picked randomly. send_default_pii=True, - environment=os.getenv("SENTRY_ENVIRONMENT", "production"), + traces_sample_rate=0.001, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. ) From 50deeae355af35b3512097ebdc592c9fadd84bb4 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 13 Jul 2022 13:47:13 +0300 Subject: [PATCH 041/213] feat(async-migration): release 0005 async migration (#10754) * Remove MULTI_TENANCY override * Improve progress indicator --- .../migrations/0005_person_replacing_by_version.py | 10 +++------- .../test/test_0005_person_replacing_by_version.py | 2 -- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/posthog/async_migrations/migrations/0005_person_replacing_by_version.py b/posthog/async_migrations/migrations/0005_person_replacing_by_version.py index d0ec210da41457..897cf9506b4375 100644 --- a/posthog/async_migrations/migrations/0005_person_replacing_by_version.py +++ b/posthog/async_migrations/migrations/0005_person_replacing_by_version.py @@ -70,12 +70,6 @@ class Migration(AsyncMigrationDefinition): depends_on = "0004_replicated_schema" - def precheck(self): - if not settings.MULTI_TENANCY: - return False, "This async migration is not yet ready for self-hosted users" - - return True, None - def is_required(self) -> bool: person_table_engine = sync_execute( "SELECT engine_full FROM system.tables WHERE database = %(database)s AND name = %(name)s", @@ -257,6 +251,8 @@ def progress(self, migration_instance: AsyncMigrationType) -> int: result = 0.5 * migration_instance.current_operation_index / len(self.operations) if migration_instance.current_operation_index == len(self.operations) - 1: - result += 0.5 * (self.get_pg_copy_highwatermark() / self.pg_copy_target_person_id) + result = 0.5 + 0.5 * (self.get_pg_copy_highwatermark() / self.pg_copy_target_person_id) + else: + result = 0.5 * migration_instance.current_operation_index / (len(self.operations) - 1) return int(100 * result) diff --git a/posthog/async_migrations/test/test_0005_person_replacing_by_version.py b/posthog/async_migrations/test/test_0005_person_replacing_by_version.py index e45099edac49d8..c940e137591da0 100644 --- a/posthog/async_migrations/test/test_0005_person_replacing_by_version.py +++ b/posthog/async_migrations/test/test_0005_person_replacing_by_version.py @@ -3,7 +3,6 @@ import pytest from django.conf import settings -from django.test import override_settings from posthog.async_migrations.runner import start_async_migration from posthog.async_migrations.setup import ( @@ -42,7 +41,6 @@ @pytest.mark.async_migrations -@override_settings(MULTI_TENANCY=True) class Test0005PersonCollapsedByVersion(AsyncMigrationBaseTest, ClickhouseTestMixin): def setUp(self): self.recreate_person_table() From fce85484ff8a3f32663bd2611647dcad2201e2e4 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Wed, 13 Jul 2022 11:54:06 +0100 Subject: [PATCH 042/213] feat(lifecycle): Lifecycle persons on events (#9880) --- ee/clickhouse/queries/test/test_lifecycle.py | 1 - posthog/queries/trends/lifecycle.py | 32 +++++++--- posthog/test/test_journeys.py | 66 +++++++++++++++----- 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/ee/clickhouse/queries/test/test_lifecycle.py b/ee/clickhouse/queries/test/test_lifecycle.py index cc3c66f233e006..bfb16c421d737b 100644 --- a/ee/clickhouse/queries/test/test_lifecycle.py +++ b/ee/clickhouse/queries/test/test_lifecycle.py @@ -55,7 +55,6 @@ def test_test_account_filters_with_groups(self): }, self.team, ) - result = Trends().run( Filter( data={ diff --git a/posthog/queries/trends/lifecycle.py b/posthog/queries/trends/lifecycle.py index 0a0b09b7818031..21ed15fc576cd7 100644 --- a/posthog/queries/trends/lifecycle.py +++ b/posthog/queries/trends/lifecycle.py @@ -11,6 +11,7 @@ from posthog.models.filters.mixins.utils import cached_property from posthog.models.person.util import get_persons_by_uuids from posthog.models.team import Team +from posthog.models.utils import PersonPropertiesMode from posthog.queries.event_query import EventQuery from posthog.queries.person_query import PersonQuery from posthog.queries.trends.sql import LIFECYCLE_PEOPLE_SQL, LIFECYCLE_SQL @@ -30,7 +31,9 @@ class Lifecycle: def _format_lifecycle_query(self, entity: Entity, filter: Filter, team: Team) -> Tuple[str, Dict, Callable]: - event_query, event_params = LifecycleEventQuery(team=team, filter=filter).get_query() + event_query, event_params = LifecycleEventQuery( + team=team, filter=filter, using_person_on_events=team.actor_on_events_querying_enabled + ).get_query() return ( LIFECYCLE_SQL.format(events_query=event_query, interval_expr=filter.interval), @@ -60,7 +63,9 @@ def get_people( request: Request, limit: int = 100, ): - event_query, event_params = LifecycleEventQuery(team=team, filter=filter).get_query() + event_query, event_params = LifecycleEventQuery( + team=team, filter=filter, using_person_on_events=team.actor_on_events_querying_enabled + ).get_query() result = sync_execute( LIFECYCLE_PEOPLE_SQL.format(events_query=event_query, interval_expr=filter.interval), @@ -88,7 +93,11 @@ def get_query(self): self.params.update(date_params) prop_query, prop_params = self._get_prop_groups( - self._filter.property_groups, person_id_joined_alias=f"{self.DISTINCT_ID_TABLE_ALIAS}.person_id" + self._filter.property_groups, + person_properties_mode=PersonPropertiesMode.DIRECT_ON_EVENTS + if self._using_person_on_events + else PersonPropertiesMode.USING_PERSON_PROPERTIES_COLUMN, + person_id_joined_alias=f"{self.DISTINCT_ID_TABLE_ALIAS if not self._using_person_on_events else self.EVENT_TABLE_ALIAS}.person_id", ) self.params.update(prop_params) @@ -100,16 +109,23 @@ def get_query(self): self.params.update(groups_params) entity_params, entity_format_params = get_entity_filtering_params( - entity=self._filter.entities[0], team_id=self._team_id, table_name=self.EVENT_TABLE_ALIAS + entity=self._filter.entities[0], + team_id=self._team_id, + table_name=self.EVENT_TABLE_ALIAS, + person_properties_mode=PersonPropertiesMode.DIRECT_ON_EVENTS + if self._using_person_on_events + else PersonPropertiesMode.USING_PERSON_PROPERTIES_COLUMN, ) self.params.update(entity_params) + created_at_clause = "person.created_at" if not self._using_person_on_events else "person_created_at" + return ( f""" SELECT DISTINCT - {self.DISTINCT_ID_TABLE_ALIAS}.person_id as person_id, + {self.DISTINCT_ID_TABLE_ALIAS if not self._using_person_on_events else self.EVENT_TABLE_ALIAS}.person_id as person_id, dateTrunc(%(interval)s, toDateTime(events.timestamp, %(timezone)s)) AS period, - toDateTime(person.created_at, %(timezone)s) AS created_at + toDateTime({created_at_clause}, %(timezone)s) AS created_at FROM events AS {self.EVENT_TABLE_ALIAS} {self._get_distinct_id_query()} {person_query} @@ -139,7 +155,7 @@ def _get_date_filter(self): ) def _determine_should_join_distinct_ids(self) -> None: - self._should_join_distinct_ids = True + self._should_join_distinct_ids = True if not self._using_person_on_events else False def _determine_should_join_persons(self) -> None: - self._should_join_persons = True + self._should_join_persons = True if not self._using_person_on_events else False diff --git a/posthog/test/test_journeys.py b/posthog/test/test_journeys.py index 7135aa0013ef8a..0b48e84e3e5531 100644 --- a/posthog/test/test_journeys.py +++ b/posthog/test/test_journeys.py @@ -54,22 +54,19 @@ def _create_event_from_args(**event): for event in events: # Populate group properties as well - group_props = {} + group_mapping = {} for property_key, value in (event.get("properties") or {}).items(): if property_key.startswith("$group_"): group_type_index = property_key[-1] try: group = Group.objects.get(team_id=team.pk, group_type_index=group_type_index, group_key=value) - group_property_key = f"group{group_type_index}_properties" - group_props = { - group_property_key: {**group.group_properties, **event.get(group_property_key, {})}, - } + group_mapping[f"group{group_type_index}"] = group except Group.DoesNotExist: continue if "timestamp" not in event: - event["timestamp"] = datetime.now() + event["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") events_to_create.append( _create_event_from_args( @@ -80,11 +77,27 @@ def _create_event_from_args(**event): properties=event.get("properties", {}), person_id=people[distinct_id].uuid, person_properties=people[distinct_id].properties or {}, - group0_properties=event.get("group0_properties", {}) or group_props.get("group0_properties", {}), - group1_properties=event.get("group1_properties", {}) or group_props.get("group1_properties", {}), - group2_properties=event.get("group2_properties", {}) or group_props.get("group2_properties", {}), - group3_properties=event.get("group3_properties", {}) or group_props.get("group3_properties", {}), - group4_properties=event.get("group4_properties", {}) or group_props.get("group4_properties", {}), + person_created_at=people[distinct_id].created_at, + group0_properties=event.get("group0_properties", {}) + or getattr(group_mapping.get("group0", {}), "group_properties", {}), + group1_properties=event.get("group1_properties", {}) + or getattr(group_mapping.get("group1", {}), "group_properties", {}), + group2_properties=event.get("group2_properties", {}) + or getattr(group_mapping.get("group2", {}), "group_properties", {}), + group3_properties=event.get("group3_properties", {}) + or getattr(group_mapping.get("group3", {}), "group_properties", {}), + group4_properties=event.get("group4_properties", {}) + or getattr(group_mapping.get("group4", {}), "group_properties", {}), + group0_created_at=event.get("group0_created_at") + or getattr(group_mapping.get("group0", {}), "created_at", None), + group1_created_at=event.get("group1_created_at") + or getattr(group_mapping.get("group1", {}), "created_at", None), + group2_created_at=event.get("group2_created_at") + or getattr(group_mapping.get("group2", {}), "created_at", None), + group3_created_at=event.get("group3_created_at") + or getattr(group_mapping.get("group3", {}), "created_at", None), + group4_created_at=event.get("group4_created_at") + or getattr(group_mapping.get("group4", {}), "created_at", None), ) ) @@ -96,9 +109,10 @@ def _create_event_from_args(**event): def _create_all_events_raw(all_events: List[Dict]): parsed = "" for event in all_events: + timestamp = timezone.now() data: Dict[str, Any] = { "properties": {}, - "timestamp": timezone.now().strftime("%Y-%m-%d %H:%M:%S.%f"), + "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"), "person_id": str(uuid4()), "person_properties": {}, "group0_properties": {}, @@ -106,16 +120,34 @@ def _create_all_events_raw(all_events: List[Dict]): "group2_properties": {}, "group3_properties": {}, "group4_properties": {}, + "person_created_at": timestamp, + "group0_created_at": timestamp, + "group1_created_at": timestamp, + "group2_created_at": timestamp, + "group3_created_at": timestamp, + "group4_created_at": timestamp, } data.update(event) + + # Remove nulls from created_at + for key in [ + "person_created_at", + "group0_created_at", + "group1_created_at", + "group2_created_at", + "group3_created_at", + "group4_created_at", + ]: + if not data[key]: + data[key] = timestamp in_memory_event = InMemoryEvent(**data) parsed += f""" - ('{str(uuid4())}', '{in_memory_event.event}', '{json.dumps(in_memory_event.properties)}', '{in_memory_event.timestamp}', {in_memory_event.team.pk}, '{in_memory_event.distinct_id}', '', '{in_memory_event.person_id}', '{json.dumps(in_memory_event.person_properties)}', '{json.dumps(in_memory_event.group0_properties)}', '{json.dumps(in_memory_event.group1_properties)}', '{json.dumps(in_memory_event.group2_properties)}', '{json.dumps(in_memory_event.group3_properties)}', '{json.dumps(in_memory_event.group4_properties)}', '{timezone.now().strftime("%Y-%m-%d %H:%M:%S.%f")}', now(), 0) + ('{str(uuid4())}', '{in_memory_event.event}', '{json.dumps(in_memory_event.properties)}', '{in_memory_event.timestamp}', {in_memory_event.team.pk}, '{in_memory_event.distinct_id}', '', '{in_memory_event.person_id}', '{json.dumps(in_memory_event.person_properties)}', '{in_memory_event.person_created_at.strftime("%Y-%m-%d %H:%M:%S.%f")}', '{json.dumps(in_memory_event.group0_properties)}', '{json.dumps(in_memory_event.group1_properties)}', '{json.dumps(in_memory_event.group2_properties)}', '{json.dumps(in_memory_event.group3_properties)}', '{json.dumps(in_memory_event.group4_properties)}', '{in_memory_event.group0_created_at.strftime("%Y-%m-%d %H:%M:%S.%f")}', '{in_memory_event.group1_created_at.strftime("%Y-%m-%d %H:%M:%S.%f")}', '{in_memory_event.group2_created_at.strftime("%Y-%m-%d %H:%M:%S.%f")}', '{in_memory_event.group3_created_at.strftime("%Y-%m-%d %H:%M:%S.%f")}', '{in_memory_event.group4_created_at.strftime("%Y-%m-%d %H:%M:%S.%f")}', '{timezone.now().strftime("%Y-%m-%d %H:%M:%S.%f")}', now(), 0) """ sync_execute( f""" - INSERT INTO {EVENTS_DATA_TABLE()} (uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, person_id, person_properties, group0_properties, group1_properties, group2_properties, group3_properties, group4_properties, created_at, _timestamp, _offset) VALUES + INSERT INTO {EVENTS_DATA_TABLE()} (uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, person_id, person_properties, person_created_at, group0_properties, group1_properties, group2_properties, group3_properties, group4_properties, group0_created_at, group1_created_at, group2_created_at, group3_created_at, group4_created_at, created_at, _timestamp, _offset) VALUES {parsed} """ ) @@ -135,12 +167,18 @@ class InMemoryEvent: timestamp: str properties: Dict person_id: str + person_created_at: datetime person_properties: Dict group0_properties: Dict group1_properties: Dict group2_properties: Dict group3_properties: Dict group4_properties: Dict + group0_created_at: datetime + group1_created_at: datetime + group2_created_at: datetime + group3_created_at: datetime + group4_created_at: datetime def update_or_create_person(distinct_ids: List[str], team_id: int, **kwargs): From 9a71bda9c45a67e553e21a8d2674201c653b272c Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 13 Jul 2022 13:54:15 +0300 Subject: [PATCH 043/213] perf: make zstd(3) the standard compression for event.properties column (#10753) --- .../migrations/0031_event_properties_zstd.py | 9 +++++++++ .../clickhouse/test/__snapshots__/test_schema.ambr | 14 +++++++------- posthog/models/event/sql.py | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 posthog/clickhouse/migrations/0031_event_properties_zstd.py diff --git a/posthog/clickhouse/migrations/0031_event_properties_zstd.py b/posthog/clickhouse/migrations/0031_event_properties_zstd.py new file mode 100644 index 00000000000000..c0020f06299d80 --- /dev/null +++ b/posthog/clickhouse/migrations/0031_event_properties_zstd.py @@ -0,0 +1,9 @@ +from infi.clickhouse_orm import migrations + +from posthog.settings import CLICKHOUSE_CLUSTER + +operations = [ + migrations.RunSQL( + f"ALTER TABLE sharded_events ON CLUSTER '{CLICKHOUSE_CLUSTER}' MODIFY COLUMN properties VARCHAR CODEC(ZSTD(3))" + ), +] diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index 905043e2eeb7d1..1ebbf52b57e85e 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -5,7 +5,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, @@ -63,7 +63,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, @@ -212,7 +212,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, @@ -411,7 +411,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, @@ -770,7 +770,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, @@ -845,7 +845,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, @@ -1078,7 +1078,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, diff --git a/posthog/models/event/sql.py b/posthog/models/event/sql.py index d5c6a330926282..6a52698a1ac23b 100644 --- a/posthog/models/event/sql.py +++ b/posthog/models/event/sql.py @@ -23,7 +23,7 @@ ( uuid UUID, event VARCHAR, - properties VARCHAR, + properties VARCHAR CODEC(ZSTD(3)), timestamp DateTime64(6, 'UTC'), team_id Int64, distinct_id VARCHAR, From 0760834a4aa01a341006aa726d802a4424281845 Mon Sep 17 00:00:00 2001 From: Guido Iaquinti <4038041+guidoiaquinti@users.noreply.github.com> Date: Wed, 13 Jul 2022 13:18:39 +0200 Subject: [PATCH 044/213] chore(sentry-sdk): reduce traces sample rate (#10761) --- posthog/settings/sentry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/settings/sentry.py b/posthog/settings/sentry.py index f0036296502c3d..b965dad8a21e64 100644 --- a/posthog/settings/sentry.py +++ b/posthog/settings/sentry.py @@ -21,5 +21,5 @@ request_bodies="always", sample_rate=1.0, # Configures the sample rate for error events, in the range of 0.0 to 1.0. The default is 1.0 which means that 100% of error events are sent. If set to 0.1 only 10% of error events will be sent. Events are picked randomly. send_default_pii=True, - traces_sample_rate=0.001, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. + traces_sample_rate=0.00001, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. ) From 9a2a9046cba852ecca17dd254c96cbfd5da2fa43 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Wed, 13 Jul 2022 11:32:00 +0000 Subject: [PATCH 045/213] feat: buffer 3.0 (graphile) (#10735) * feat: buffer 3.0 (graphile) * fixes * test * address review * add test for buffer processAt --- plugin-server/src/capabilities.ts | 9 ++- .../src/main/ingestion-queues/buffer.ts | 75 ------------------- plugin-server/src/main/job-queues/buffer.ts | 9 +++ .../src/main/job-queues/job-queue-consumer.ts | 43 ++++++++--- .../src/main/job-queues/local/fs-queue.ts | 8 +- plugin-server/src/main/pluginsServer.ts | 17 +---- plugin-server/src/types.ts | 11 ++- plugin-server/src/utils/db/db.ts | 9 --- .../event-pipeline/1-emitToBufferStep.ts | 11 ++- plugin-server/src/worker/tasks.ts | 7 +- plugin-server/src/worker/vm/capabilities.ts | 2 +- plugin-server/src/worker/worker.ts | 2 +- plugin-server/tests/jobs.test.ts | 1 - plugin-server/tests/main/capabilities.test.ts | 66 ++++++++++++---- plugin-server/tests/main/db.test.ts | 15 ---- .../main/ingestion-queues/buffer.test.ts | 50 ------------- plugin-server/tests/server.test.ts | 15 +++- .../tests/worker/capabilities.test.ts | 15 ++-- .../event-pipeline/emitToBufferStep.test.ts | 21 ++++-- plugin-server/tests/worker/vm.lazy.test.ts | 7 +- 20 files changed, 172 insertions(+), 221 deletions(-) delete mode 100644 plugin-server/src/main/ingestion-queues/buffer.ts create mode 100644 plugin-server/src/main/job-queues/buffer.ts delete mode 100644 plugin-server/tests/main/ingestion-queues/buffer.test.ts diff --git a/plugin-server/src/capabilities.ts b/plugin-server/src/capabilities.ts index d88b5d14d7e26d..91ac6aa87027bb 100644 --- a/plugin-server/src/capabilities.ts +++ b/plugin-server/src/capabilities.ts @@ -10,13 +10,18 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin return { ingestion: true, pluginScheduledTasks: true, - processJobs: true, + processPluginJobs: true, processAsyncHandlers: true, ...sharedCapabilities, } case 'ingestion': return { ingestion: true, ...sharedCapabilities } case 'async': - return { pluginScheduledTasks: true, processJobs: true, processAsyncHandlers: true, ...sharedCapabilities } + return { + pluginScheduledTasks: true, + processPluginJobs: true, + processAsyncHandlers: true, + ...sharedCapabilities, + } } } diff --git a/plugin-server/src/main/ingestion-queues/buffer.ts b/plugin-server/src/main/ingestion-queues/buffer.ts deleted file mode 100644 index 3f0e76f3a26b5b..00000000000000 --- a/plugin-server/src/main/ingestion-queues/buffer.ts +++ /dev/null @@ -1,75 +0,0 @@ -import Piscina from '@posthog/piscina' -import { PluginEvent } from '@posthog/plugin-scaffold' - -import { Hub } from '../../types' -import { runInstrumentedFunction } from '../utils' - -export function runBufferEventPipeline(hub: Hub, piscina: Piscina, event: PluginEvent) { - hub.lastActivity = new Date().valueOf() - hub.lastActivityType = 'runBufferEventPipeline' - return piscina.run({ task: 'runBufferEventPipeline', args: { event } }) -} - -export async function runBuffer(hub: Hub, piscina: Piscina): Promise { - let eventRows: { id: number; event: PluginEvent }[] = [] - await hub.db.postgresTransaction(async (client) => { - const eventsResult = await client.query(` - UPDATE posthog_eventbuffer SET locked=true WHERE id IN ( - SELECT id FROM posthog_eventbuffer - WHERE process_at <= now() AND process_at > (now() - INTERVAL '30 minute') AND locked=false - ORDER BY id - LIMIT 10 - FOR UPDATE SKIP LOCKED - ) - RETURNING id, event - `) - eventRows = eventsResult.rows - }) - - const idsToDelete: number[] = [] - - // We don't indiscriminately delete all IDs to prevent the case when we don't trigger `runInstrumentedFunction` - // Once that runs, events will either go to the events table or the dead letter queue - const processBufferEvent = async (event: PluginEvent, id: number) => { - await runInstrumentedFunction({ - server: hub, - event: event, - func: () => runBufferEventPipeline(hub, piscina, event), - statsKey: `kafka_queue.ingest_buffer_event`, - timeoutMessage: 'After 30 seconds still running runBufferEventPipeline', - }) - idsToDelete.push(id) - } - - await Promise.all(eventRows.map((eventRow) => processBufferEvent(eventRow.event, eventRow.id))) - - if (idsToDelete.length > 0) { - await hub.db.postgresQuery( - `DELETE FROM posthog_eventbuffer WHERE id IN (${idsToDelete.join(',')})`, - [], - 'completeBufferEvent' - ) - hub.statsd?.increment('events_deleted_from_buffer', idsToDelete.length) - } -} - -export async function clearBufferLocks(hub: Hub): Promise { - /* - * If we crash during runBuffer we may end up with 2 scenarios: - * 1. "locked" rows with events that were never processed (crashed after fetching and before running the pipeline) - * 2. "locked" rows with events that were processed (crashed after the pipeline and before deletion) - * This clears any old locks such that the events are processed again. If there are any duplicates ClickHouse should collapse them. - */ - const recordsUpdated = await hub.db.postgresQuery( - `UPDATE posthog_eventbuffer - SET locked=false, process_at=now() - WHERE locked=true AND process_at < (now() - INTERVAL '30 minute') - RETURNING 1`, - [], - 'clearBufferLocks' - ) - - if (recordsUpdated.rowCount > 0 && hub.statsd) { - hub.statsd.increment('buffer_locks_cleared', recordsUpdated.rowCount) - } -} diff --git a/plugin-server/src/main/job-queues/buffer.ts b/plugin-server/src/main/job-queues/buffer.ts new file mode 100644 index 00000000000000..1c52b5eefe6e88 --- /dev/null +++ b/plugin-server/src/main/job-queues/buffer.ts @@ -0,0 +1,9 @@ +import Piscina from '@posthog/piscina' +import { PluginEvent } from '@posthog/plugin-scaffold' +import { Hub } from 'types' + +export function runBufferEventPipeline(hub: Hub, piscina: Piscina, event: PluginEvent): Promise { + hub.lastActivity = new Date().valueOf() + hub.lastActivityType = 'runBufferEventPipeline' + return piscina.run({ task: 'runBufferEventPipeline', args: { event } }) +} diff --git a/plugin-server/src/main/job-queues/job-queue-consumer.ts b/plugin-server/src/main/job-queues/job-queue-consumer.ts index a41911766480b0..f18fc1ea02fc90 100644 --- a/plugin-server/src/main/job-queues/job-queue-consumer.ts +++ b/plugin-server/src/main/job-queues/job-queue-consumer.ts @@ -1,31 +1,52 @@ import Piscina from '@posthog/piscina' import { TaskList } from 'graphile-worker' -import { EnqueuedJob, Hub, JobQueueConsumerControl } from '../../types' +import { EnqueuedBufferJob, EnqueuedPluginJob, Hub, JobQueueConsumerControl } from '../../types' import { killProcess } from '../../utils/kill' import { status } from '../../utils/status' import { logOrThrowJobQueueError } from '../../utils/utils' import { pauseQueueIfWorkerFull } from '../ingestion-queues/queue' +import { runInstrumentedFunction } from '../utils' +import { runBufferEventPipeline } from './buffer' -export async function startJobQueueConsumer(server: Hub, piscina: Piscina): Promise { +export async function startJobQueueConsumer(hub: Hub, piscina: Piscina): Promise { status.info('🔄', 'Starting job queue consumer, trying to get lock...') - const jobHandlers: TaskList = { + const ingestionJobHandlers: TaskList = { + bufferJob: async (job) => { + const eventPayload = (job as EnqueuedBufferJob).eventPayload + await runInstrumentedFunction({ + server: hub, + event: eventPayload, + func: () => runBufferEventPipeline(hub, piscina, eventPayload), + statsKey: `kafka_queue.ingest_buffer_event`, + timeoutMessage: 'After 30 seconds still running runBufferEventPipeline', + }) + hub.statsd?.increment('events_deleted_from_buffer') + }, + } + + const pluginJobHandlers: TaskList = { pluginJob: async (job) => { - pauseQueueIfWorkerFull(() => server.jobQueueManager.pauseConsumer(), server, piscina) - server.statsd?.increment('triggered_job', { - instanceId: server.instanceId.toString(), + pauseQueueIfWorkerFull(() => hub.jobQueueManager.pauseConsumer(), hub, piscina) + hub.statsd?.increment('triggered_job', { + instanceId: hub.instanceId.toString(), }) - await piscina.run({ task: 'runJob', args: { job: job as EnqueuedJob } }) + await piscina.run({ task: 'runPluginJob', args: { job: job as EnqueuedPluginJob } }) }, } + const jobHandlers: TaskList = { + ...(hub.capabilities.ingestion ? ingestionJobHandlers : {}), + ...(hub.capabilities.processPluginJobs ? pluginJobHandlers : {}), + } + status.info('🔄', 'Job queue consumer starting') try { - await server.jobQueueManager.startConsumer(jobHandlers) + await hub.jobQueueManager.startConsumer(jobHandlers) } catch (error) { try { - logOrThrowJobQueueError(server, error, `Cannot start job queue consumer!`) + logOrThrowJobQueueError(hub, error, `Cannot start job queue consumer!`) } catch { killProcess() } @@ -33,8 +54,8 @@ export async function startJobQueueConsumer(server: Hub, piscina: Piscina): Prom const stop = async () => { status.info('🔄', 'Stopping job queue consumer') - await server.jobQueueManager.stopConsumer() + await hub.jobQueueManager.stopConsumer() } - return { stop, resume: () => server.jobQueueManager.resumeConsumer() } + return { stop, resume: () => hub.jobQueueManager.resumeConsumer() } } diff --git a/plugin-server/src/main/job-queues/local/fs-queue.ts b/plugin-server/src/main/job-queues/local/fs-queue.ts index cd5826f264497a..45d561f56dc65c 100644 --- a/plugin-server/src/main/job-queues/local/fs-queue.ts +++ b/plugin-server/src/main/job-queues/local/fs-queue.ts @@ -7,8 +7,14 @@ import * as path from 'path' import { JobQueueBase } from '../job-queue-base' -interface FsJob extends EnqueuedJob { +interface FsJob { jobName: string + timestamp: number + type?: string + payload?: Record + eventPayload?: Record + pluginConfigId?: number + pluginConfigTeam?: number } export class FsQueue extends JobQueueBase { paused: boolean diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index b07a2fb9face44..250f2ed4a408e4 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -23,7 +23,6 @@ import { cancelAllScheduledJobs } from '../utils/node-schedule' import { PubSub } from '../utils/pubsub' import { status } from '../utils/status' import { delay, getPiscinaStats, stalenessCheck } from '../utils/utils' -import { clearBufferLocks, runBuffer } from './ingestion-queues/buffer' import { KafkaQueue } from './ingestion-queues/kafka-queue' import { startQueues } from './ingestion-queues/queue' import { startJobQueueConsumer } from './job-queues/job-queue-consumer' @@ -171,7 +170,7 @@ export async function startPluginsServer( if (hub.capabilities.pluginScheduledTasks) { pluginScheduleControl = await startPluginSchedules(hub, piscina) } - if (hub.capabilities.processJobs) { + if (hub.capabilities.ingestion || hub.capabilities.processPluginJobs) { jobQueueConsumer = await startJobQueueConsumer(hub, piscina) } @@ -235,20 +234,6 @@ export async function startPluginsServer( } }) - if (hub.capabilities.ingestion) { - // every 5 seconds process buffer events - schedule.scheduleJob('*/5 * * * * *', async () => { - if (piscina) { - await runBuffer(hub!, piscina) - } - }) - } - - // every 30 minutes clear any locks that may have lingered on the buffer table - schedule.scheduleJob('*/30 * * * *', async () => { - await clearBufferLocks(hub!) - }) - // every minute log information on kafka consumer if (queue) { schedule.scheduleJob('0 * * * * *', async () => { diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 20c0514ca76829..c39b4997fb7155 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -201,12 +201,13 @@ export interface Hub extends PluginsServerConfig { export interface PluginServerCapabilities { ingestion?: boolean pluginScheduledTasks?: boolean - processJobs?: boolean + processPluginJobs?: boolean processAsyncHandlers?: boolean http?: boolean } -export interface EnqueuedJob { +export type EnqueuedJob = EnqueuedPluginJob | EnqueuedBufferJob +export interface EnqueuedPluginJob { type: string payload: Record timestamp: number @@ -214,8 +215,14 @@ export interface EnqueuedJob { pluginConfigTeam: number } +export interface EnqueuedBufferJob { + eventPayload: PluginEvent + timestamp: number +} + export enum JobName { PLUGIN_JOB = 'pluginJob', + BUFFER_JOB = 'bufferJob', } export interface JobQueue { diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index ae8b57ec1458ac..300ce2a5d9c816 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -2041,13 +2041,4 @@ export class DB { ) return response.rowCount > 0 } - - public async addEventToBuffer(event: Record, processAt: DateTime): Promise { - await this.postgresQuery( - `INSERT INTO posthog_eventbuffer (event, process_at, locked) VALUES ($1, $2, $3)`, - [event, processAt.toISO(), false], - 'addEventToBuffer' - ) - this.statsd?.increment('events_sent_to_buffer') - } } diff --git a/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts index 94eb11c075dfe3..9c2f926701cb99 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts @@ -1,7 +1,6 @@ import { PluginEvent } from '@posthog/plugin-scaffold' -import { DateTime } from 'luxon' -import { Hub, IngestionPersonData, TeamId } from '../../../types' +import { Hub, IngestionPersonData, JobName, TeamId } from '../../../types' import { EventPipelineRunner, StepResult } from './runner' export async function emitToBufferStep( @@ -21,8 +20,12 @@ export async function emitToBufferStep( const person = await runner.hub.db.fetchPerson(event.team_id, event.distinct_id) if (shouldBuffer(runner.hub, event, person, event.team_id)) { - const processEventAt = DateTime.now().plus({ seconds: runner.hub.BUFFER_CONVERSION_SECONDS }) - await runner.hub.db.addEventToBuffer(event, processEventAt) + const processEventAt = Date.now() + runner.hub.BUFFER_CONVERSION_SECONDS * 1000 + await runner.hub.jobQueueManager.enqueue(JobName.BUFFER_JOB, { + eventPayload: event, + timestamp: processEventAt, + }) + runner.hub.statsd?.increment('events_sent_to_buffer') return null } else { return runner.nextStep('pluginsProcessEventStep', event, person) diff --git a/plugin-server/src/worker/tasks.ts b/plugin-server/src/worker/tasks.ts index 0e94a0971ac8dd..53c34a2bbb46cd 100644 --- a/plugin-server/src/worker/tasks.ts +++ b/plugin-server/src/worker/tasks.ts @@ -1,6 +1,6 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { Action, EnqueuedJob, Hub, IngestionEvent, JobName, PluginTaskType, Team } from '../types' +import { Action, EnqueuedPluginJob, Hub, IngestionEvent, PluginTaskType, Team } from '../types' import { convertToProcessedPluginEvent } from '../utils/event' import { EventPipelineRunner } from './ingestion/event-pipeline/runner' import { runPluginTask, runProcessEvent } from './plugins/run' @@ -10,7 +10,7 @@ import { teardownPlugins } from './plugins/teardown' type TaskRunner = (hub: Hub, args: any) => Promise | any export const workerTasks: Record = { - runJob: (hub, { job }: { job: EnqueuedJob }) => { + runPluginJob: (hub, { job }: { job: EnqueuedPluginJob }) => { return runPluginTask(hub, job.type, PluginTaskType.Job, job.pluginConfigId, job.payload) }, runEveryMinute: (hub, args: { pluginConfigId: number }) => { @@ -61,9 +61,6 @@ export const workerTasks: Record = { flushKafkaMessages: async (hub) => { await hub.kafkaProducer.flush() }, - enqueueJob: async (hub, { job }: { job: EnqueuedJob }) => { - await hub.jobQueueManager.enqueue(JobName.PLUGIN_JOB, job) - }, // Exported only for tests _testsRunProcessEvent: async (hub, args: { event: PluginEvent }) => { return runProcessEvent(hub, args.event) diff --git a/plugin-server/src/worker/vm/capabilities.ts b/plugin-server/src/worker/vm/capabilities.ts index 9b52a2d4296a5c..b24716e3ebbed3 100644 --- a/plugin-server/src/worker/vm/capabilities.ts +++ b/plugin-server/src/worker/vm/capabilities.ts @@ -41,7 +41,7 @@ function shouldSetupPlugin(serverCapability: keyof PluginServerCapabilities, plu if (serverCapability === 'pluginScheduledTasks') { return (pluginCapabilities.scheduled_tasks || []).length > 0 } - if (serverCapability === 'processJobs') { + if (serverCapability === 'processPluginJobs') { return (pluginCapabilities.jobs || []).length > 0 } if (serverCapability === 'processAsyncHandlers') { diff --git a/plugin-server/src/worker/worker.ts b/plugin-server/src/worker/worker.ts index b933b2dc64872e..2e24c9c96033a0 100644 --- a/plugin-server/src/worker/worker.ts +++ b/plugin-server/src/worker/worker.ts @@ -50,7 +50,7 @@ export const createTaskRunner = } hub.statsd?.timing(`piscina_task.${task}`, timer) - if (task === 'runJob') { + if (task === 'runPluginJob') { hub.statsd?.timing('plugin_job', timer, { type: String(args.job?.type), pluginConfigId: String(args.job?.pluginConfigId), diff --git a/plugin-server/tests/jobs.test.ts b/plugin-server/tests/jobs.test.ts index d089958f981752..55cb02dd5994e7 100644 --- a/plugin-server/tests/jobs.test.ts +++ b/plugin-server/tests/jobs.test.ts @@ -150,7 +150,6 @@ describe.skip('job queues', () => { const now = Date.now() const job: EnqueuedJob = { - type: 'pluginJob', payload: { key: 'value' }, timestamp: now + DELAY, pluginConfigId: 2, diff --git a/plugin-server/tests/main/capabilities.test.ts b/plugin-server/tests/main/capabilities.test.ts index 2548d30ff62dc1..52bff7abeb7696 100644 --- a/plugin-server/tests/main/capabilities.test.ts +++ b/plugin-server/tests/main/capabilities.test.ts @@ -2,28 +2,29 @@ import Piscina from '@posthog/piscina' import { KafkaQueue } from '../../src/main/ingestion-queues/kafka-queue' import { startQueues } from '../../src/main/ingestion-queues/queue' +import { startJobQueueConsumer } from '../../src/main/job-queues/job-queue-consumer' import { Hub, LogLevel } from '../../src/types' import { createHub } from '../../src/utils/db/hub' jest.mock('../../src/main/ingestion-queues/kafka-queue') -describe('queue', () => { - describe('capabilities', () => { - let hub: Hub - let piscina: Piscina - let closeHub: () => Promise +describe('capabilities', () => { + let hub: Hub + let piscina: Piscina + let closeHub: () => Promise - beforeEach(async () => { - ;[hub, closeHub] = await createHub({ - LOG_LEVEL: LogLevel.Warn, - }) - piscina = { run: jest.fn() } as any + beforeEach(async () => { + ;[hub, closeHub] = await createHub({ + LOG_LEVEL: LogLevel.Warn, }) + piscina = { run: jest.fn() } as any + }) - afterEach(async () => { - await closeHub() - }) + afterEach(async () => { + await closeHub() + }) + describe('queue', () => { it('starts ingestion queue by default', async () => { const queues = await startQueues(hub, piscina) @@ -43,4 +44,43 @@ describe('queue', () => { }) }) }) + + describe('startJobQueueConsumer()', () => { + it('sets up bufferJob handler if ingestion is on', async () => { + hub.jobQueueManager.startConsumer = jest.fn() + hub.capabilities.ingestion = true + hub.capabilities.processPluginJobs = false + + await startJobQueueConsumer(hub, piscina) + + expect(hub.jobQueueManager.startConsumer).toHaveBeenCalledWith({ + bufferJob: expect.anything(), + }) + }) + + it('sets up pluginJob handler if processPluginJobs is on', async () => { + hub.jobQueueManager.startConsumer = jest.fn() + hub.capabilities.ingestion = false + hub.capabilities.processPluginJobs = true + + await startJobQueueConsumer(hub, piscina) + + expect(hub.jobQueueManager.startConsumer).toHaveBeenCalledWith({ + pluginJob: expect.anything(), + }) + }) + + it('sets up bufferJob and pluginJob handlers if ingestion and processPluginJobs are on', async () => { + hub.jobQueueManager.startConsumer = jest.fn() + hub.capabilities.ingestion = true + hub.capabilities.processPluginJobs = true + + await startJobQueueConsumer(hub, piscina) + + expect(hub.jobQueueManager.startConsumer).toHaveBeenCalledWith({ + bufferJob: expect.anything(), + pluginJob: expect.anything(), + }) + }) + }) }) diff --git a/plugin-server/tests/main/db.test.ts b/plugin-server/tests/main/db.test.ts index a6a7e65523ce5c..6093aedf3c0428 100644 --- a/plugin-server/tests/main/db.test.ts +++ b/plugin-server/tests/main/db.test.ts @@ -1026,19 +1026,4 @@ describe('DB', () => { ) }) }) - - describe('addEventToBuffer', () => { - test('inserts event correctly', async () => { - const processAt = DateTime.now() - await db.addEventToBuffer({ foo: 'bar' }, processAt) - - const bufferResult = await db.postgresQuery( - 'SELECT event, process_at FROM posthog_eventbuffer', - [], - 'addEventToBufferTest' - ) - expect(bufferResult.rows[0].event).toEqual({ foo: 'bar' }) - expect(processAt).toEqual(DateTime.fromISO(bufferResult.rows[0].process_at)) - }) - }) }) diff --git a/plugin-server/tests/main/ingestion-queues/buffer.test.ts b/plugin-server/tests/main/ingestion-queues/buffer.test.ts deleted file mode 100644 index 9751bbc9a0318f..00000000000000 --- a/plugin-server/tests/main/ingestion-queues/buffer.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import Piscina from '@posthog/piscina' -import { DateTime } from 'luxon' - -import { runBuffer } from '../../../src/main/ingestion-queues/buffer' -import { runInstrumentedFunction } from '../../../src/main/utils' -import { Hub } from '../../../src/types' -import { DB } from '../../../src/utils/db/db' -import { createHub } from '../../../src/utils/db/hub' -import { resetTestDatabase } from '../../helpers/sql' - -// jest.mock('../../../src/utils') -jest.mock('../../../src/main/utils') - -describe('Event buffer', () => { - let hub: Hub - let closeServer: () => Promise - let db: DB - - beforeEach(async () => { - ;[hub, closeServer] = await createHub() - await resetTestDatabase(undefined, {}, {}, { withExtendedTestData: false }) - db = hub.db - }) - - afterEach(async () => { - await closeServer() - jest.clearAllMocks() - }) - - describe('runBuffer', () => { - test('processes events from buffer and deletes them', async () => { - const processAt = DateTime.now() - await db.addEventToBuffer({ foo: 'bar' }, processAt) - await db.addEventToBuffer({ foo: 'bar' }, processAt) - - await runBuffer(hub, {} as Piscina) - - expect(runInstrumentedFunction).toHaveBeenCalledTimes(2) - expect(runInstrumentedFunction).toHaveBeenLastCalledWith(expect.objectContaining({ event: { foo: 'bar' } })) - - const countResult = await db.postgresQuery( - 'SELECT count(*) FROM posthog_eventbuffer', - [], - 'eventBufferCountTest' - ) - - expect(Number(countResult.rows[0].count)).toEqual(0) - }) - }) -}) diff --git a/plugin-server/tests/server.test.ts b/plugin-server/tests/server.test.ts index 3fa9d81bd583a2..1766a360b3307d 100644 --- a/plugin-server/tests/server.test.ts +++ b/plugin-server/tests/server.test.ts @@ -111,20 +111,29 @@ describe('server', () => { test('disabling pluginScheduledTasks', async () => { pluginsServer = await createPluginServer( {}, - { ingestion: true, pluginScheduledTasks: false, processJobs: true } + { ingestion: true, pluginScheduledTasks: false, processPluginJobs: true } ) expect(startPluginSchedules).not.toHaveBeenCalled() expect(startJobQueueConsumer).toHaveBeenCalled() }) - test('disabling processJobs', async () => { + test('disabling processPluginJobs', async () => { pluginsServer = await createPluginServer( {}, - { ingestion: true, pluginScheduledTasks: true, processJobs: false } + { ingestion: true, pluginScheduledTasks: true, processPluginJobs: false } ) expect(startPluginSchedules).toHaveBeenCalled() + expect(startJobQueueConsumer).toHaveBeenCalled() + }) + + test('disabling processPluginJobs and ingestion', async () => { + pluginsServer = await createPluginServer( + {}, + { ingestion: false, pluginScheduledTasks: true, processPluginJobs: false } + ) + expect(startJobQueueConsumer).not.toHaveBeenCalled() }) }) diff --git a/plugin-server/tests/worker/capabilities.test.ts b/plugin-server/tests/worker/capabilities.test.ts index 517bbebe2b3dd7..caed30fabd3d67 100644 --- a/plugin-server/tests/worker/capabilities.test.ts +++ b/plugin-server/tests/worker/capabilities.test.ts @@ -72,7 +72,12 @@ describe('capabilities', () => { it('returns false if the plugin has no capabilities', () => { const shouldSetupPlugin = shouldSetupPluginInServer( - { ingestion: true, processAsyncHandlers: true, processJobs: true, pluginScheduledTasks: true }, + { + ingestion: true, + processAsyncHandlers: true, + processPluginJobs: true, + pluginScheduledTasks: true, + }, {} ) expect(shouldSetupPlugin).toEqual(false) @@ -117,13 +122,13 @@ describe('capabilities', () => { }) describe('jobs', () => { - it('returns true if plugin has any jobs and the server has processJobs capability', () => { - const shouldSetupPlugin = shouldSetupPluginInServer({ processJobs: true }, { jobs: ['someJob'] }) + it('returns true if plugin has any jobs and the server has processPluginJobs capability', () => { + const shouldSetupPlugin = shouldSetupPluginInServer({ processPluginJobs: true }, { jobs: ['someJob'] }) expect(shouldSetupPlugin).toEqual(true) }) - it('returns false if plugin has no jobs and the server has only processJobs capability', () => { - const shouldSetupPlugin = shouldSetupPluginInServer({ processJobs: true }, { jobs: [] }) + it('returns false if plugin has no jobs and the server has only processPluginJobs capability', () => { + const shouldSetupPlugin = shouldSetupPluginInServer({ processPluginJobs: true }, { jobs: [] }) expect(shouldSetupPlugin).toEqual(false) }) }) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts index a12164943aca4e..2d2af8bdaf860a 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts @@ -1,7 +1,7 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' -import { Person } from '../../../../src/types' +import { JobName, Person } from '../../../../src/types' import { UUIDT } from '../../../../src/utils/utils' import { emitToBufferStep, @@ -43,17 +43,26 @@ beforeEach(() => { hub: { CONVERSION_BUFFER_ENABLED: true, BUFFER_CONVERSION_SECONDS: 60, - db: { fetchPerson: jest.fn().mockResolvedValue(existingPerson), addEventToBuffer: jest.fn() }, + db: { fetchPerson: jest.fn().mockResolvedValue(existingPerson) }, eventsProcessor: {}, + jobQueueManager: { + enqueue: jest.fn(), + }, }, } }) describe('emitToBufferStep()', () => { - it('calls `addEventToBuffer` if event should be buffered, stops processing', async () => { + it('enqueues graphile job if event should be buffered, stops processing', async () => { + const unixNow = 1657710000000 + Date.now = jest.fn(() => unixNow) + const response = await emitToBufferStep(runner, pluginEvent, () => true) - expect(runner.hub.db.addEventToBuffer).toHaveBeenCalledWith(pluginEvent, expect.any(DateTime)) + expect(runner.hub.jobQueueManager.enqueue).toHaveBeenCalledWith(JobName.BUFFER_JOB, { + eventPayload: pluginEvent, + timestamp: unixNow + 60000, // runner.hub.BUFFER_CONVERSION_SECONDS * 1000 + }) expect(runner.hub.db.fetchPerson).toHaveBeenCalledWith(2, 'my_id') expect(response).toEqual(null) }) @@ -63,7 +72,7 @@ describe('emitToBufferStep()', () => { expect(response).toEqual(['pluginsProcessEventStep', pluginEvent, existingPerson]) expect(runner.hub.db.fetchPerson).toHaveBeenCalledWith(2, 'my_id') - expect(runner.hub.db.addEventToBuffer).not.toHaveBeenCalled() + expect(runner.hub.jobQueueManager.enqueue).not.toHaveBeenCalled() }) it('calls `processPersonsStep` for $snapshot events', async () => { @@ -73,7 +82,7 @@ describe('emitToBufferStep()', () => { expect(response).toEqual(['processPersonsStep', event, undefined]) expect(runner.hub.db.fetchPerson).not.toHaveBeenCalled() - expect(runner.hub.db.addEventToBuffer).not.toHaveBeenCalled() + expect(runner.hub.jobQueueManager.enqueue).not.toHaveBeenCalled() }) }) diff --git a/plugin-server/tests/worker/vm.lazy.test.ts b/plugin-server/tests/worker/vm.lazy.test.ts index 2e21205b8c2f8d..48ac90b5cbe5d3 100644 --- a/plugin-server/tests/worker/vm.lazy.test.ts +++ b/plugin-server/tests/worker/vm.lazy.test.ts @@ -27,7 +27,12 @@ describe('LazyPluginVM', () => { const mockServer: any = { db, - capabilities: { ingestion: true, pluginScheduledTasks: true, processJobs: true, processAsyncHandlers: true }, + capabilities: { + ingestion: true, + pluginScheduledTasks: true, + processPluginJobs: true, + processAsyncHandlers: true, + }, } const createVM = () => { From 3d8f8dfe26ff75b8eb9ad971b6a3f952ab0f24e2 Mon Sep 17 00:00:00 2001 From: timgl Date: Wed, 13 Jul 2022 15:37:53 +0200 Subject: [PATCH 046/213] perf(decide): Speed up decide endpoint by shortcircuiting middleware (#10744) * perf(decide): Speed up decide endpoint by shortcircuiting middleware * fix cypress tests --- cypress/e2e/toolbar.js | 16 +++++++- .../toolbar-launch/AuthorizedUrlsTable.tsx | 18 +++++++-- posthog/api/decide.py | 8 ---- posthog/api/test/test_decide.py | 38 ------------------- posthog/middleware.py | 22 +++++++++++ posthog/models/property/property.py | 3 +- posthog/settings/web.py | 3 +- 7 files changed, 54 insertions(+), 54 deletions(-) diff --git a/cypress/e2e/toolbar.js b/cypress/e2e/toolbar.js index ceed0694c2f9fe..b33fc3980aee80 100644 --- a/cypress/e2e/toolbar.js +++ b/cypress/e2e/toolbar.js @@ -1,7 +1,19 @@ describe('Toolbar', () => { it('Toolbar loads', () => { - cy.visit('/demo') - cy.get('#__POSTHOG_TOOLBAR__').should('exist') + cy.get('[data-attr="menu-item-toolbar-launch"]').click() + cy.get('[data-attr="sidebar-launch-toolbar"]').contains('Add toolbar URL').click() + cy.location().then((loc) => { + cy.get('[data-attr="url-input"]').clear().type(`http://${loc.host}/demo`) + cy.get('[data-attr="url-save"]').click() + cy.get('[data-attr="toolbar-open"]') + .first() + .parent() + .invoke('attr', 'href') + .then((href) => { + cy.visit(href) + }) + cy.get('#__POSTHOG_TOOLBAR__').shadow().find('div').should('exist') + }) }) it('toolbar item in sidebar has launch options', () => { diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx b/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx index 4b7212eb34475f..d210b7788febad 100644 --- a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx +++ b/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx @@ -76,8 +76,9 @@ export function AuthorizedUrlsTable({ pageKey, actionId }: AuthorizedUrlsTableIn onPressEnter={save} autoFocus placeholder="Enter a URL or wildcard subdomain (e.g. https://*.posthog.com)" + data-attr="url-input" /> -
@@ -93,12 +94,21 @@ export function AuthorizedUrlsTable({ pageKey, actionId }: AuthorizedUrlsTableIn return (
{record.type === 'suggestion' ? ( - addUrl(record.url)}> + addUrl(record.url)} + data-attr="toolbar-apply-suggestion" + > Apply suggestion ) : ( <> - + Open with Toolbar
- + Add{pageKey === 'toolbar-launch' && ' authorized URL'} diff --git a/posthog/api/decide.py b/posthog/api/decide.py index 6ab93fef29186a..3a46680f2b56eb 100644 --- a/posthog/api/decide.py +++ b/posthog/api/decide.py @@ -1,5 +1,4 @@ import re -import secrets from typing import Any, Dict, Optional, Tuple from urllib.parse import urlparse @@ -84,13 +83,6 @@ def get_decide(request: HttpRequest): "supportedCompression": ["gzip", "gzip-js", "lz64"], } - if request.user.is_authenticated: - r, update_user_token = decide_editor_params(request) - response.update(r) - if update_user_token: - request.user.temporary_token = secrets.token_urlsafe(32) - request.user.save() - response["featureFlags"] = [] response["sessionRecording"] = False diff --git a/posthog/api/test/test_decide.py b/posthog/api/test/test_decide.py index 1c7d9a2e51d179..7fb8f7dba3eeaf 100644 --- a/posthog/api/test/test_decide.py +++ b/posthog/api/test/test_decide.py @@ -55,31 +55,6 @@ def test_defaults_to_v2_if_conflicting_parameters(self): ) self.assertEqual(response.status_code, status.HTTP_200_OK) - def test_user_on_own_site_enabled(self): - user = self.organization.members.first() - user.toolbar_mode = "toolbar" - user.save() - - self.team.app_urls = ["https://example.com/maybesubdomain"] - self.team.save() - response = self.client.get("/decide/", HTTP_ORIGIN="https://example.com").json() - self.assertEqual(response["isAuthenticated"], True) - self.assertEqual(response["supportedCompression"], ["gzip", "gzip-js", "lz64"]) - self.assertEqual(response["editorParams"]["toolbarVersion"], "toolbar") - - def test_user_on_own_site_disabled(self): - user = self.organization.members.first() - user.toolbar_mode = "disabled" - user.save() - - self.team.app_urls = ["https://example.com/maybesubdomain"] - self.team.save() - - # Make sure the endpoint works with and without the trailing slash - response = self.client.get("/decide", HTTP_ORIGIN="https://example.com").json() - self.assertEqual(response["isAuthenticated"], True) - self.assertIsNone(response["editorParams"].get("toolbarVersion")) - def test_user_on_evil_site(self): user = self.organization.members.first() user.toolbar_mode = "toolbar" @@ -91,19 +66,6 @@ def test_user_on_evil_site(self): self.assertEqual(response["isAuthenticated"], False) self.assertIsNone(response["editorParams"].get("toolbarVersion", None)) - def test_user_on_local_host(self): - user = self.organization.members.first() - user.toolbar_mode = "toolbar" - user.save() - - self.team.app_urls = ["https://example.com"] - self.team.save() - response = self.client.get("/decide", HTTP_ORIGIN="http://127.0.0.1:8000").json() - self.assertEqual(response["isAuthenticated"], True) - self.assertEqual(response["sessionRecording"], False) - self.assertEqual(response["editorParams"]["toolbarVersion"], "toolbar") - self.assertEqual(response["supportedCompression"], ["gzip", "gzip-js", "lz64"]) - def test_user_session_recording_opt_in(self): # :TRICKY: Test for regression around caching response = self._post_decide().json() diff --git a/posthog/middleware.py b/posthog/middleware.py index 9a2d1d9034915d..2db74877585a10 100644 --- a/posthog/middleware.py +++ b/posthog/middleware.py @@ -10,6 +10,7 @@ from django.utils.cache import add_never_cache_headers from loginas.utils import is_impersonated_session +from posthog.api.decide import get_decide from posthog.internal_metrics import incr from posthog.models import Action, Cohort, Dashboard, FeatureFlag, Insight, Team, User @@ -187,3 +188,24 @@ def __call__(self, request: HttpRequest): client._request_information = None return response + + +def shortcircuitmiddleware(f): + """ view decorator, the sole purpose to is 'rename' the function + '_shortcircuitmiddleware' """ + + def _shortcircuitmiddleware(*args, **kwargs): + return f(*args, **kwargs) + + return _shortcircuitmiddleware + + +class ShortCircuitMiddleware(object): + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request: HttpRequest): + if request.path == "/decide/" or request.path == "/decide": + return get_decide(request) + response: HttpResponse = self.get_response(request) + return response diff --git a/posthog/models/property/property.py b/posthog/models/property/property.py index 7df7b102153c5c..e089d3572db574 100644 --- a/posthog/models/property/property.py +++ b/posthog/models/property/property.py @@ -262,7 +262,8 @@ def property_to_Q(self) -> Q: from posthog.models.cohort import Cohort cohort_id = int(cast(Union[str, int], value)) - cohort = Cohort.objects.get(pk=cohort_id) + + cohort = Cohort.objects.only("version").get(pk=cohort_id) return Q( Exists( CohortPeople.objects.filter( diff --git a/posthog/settings/web.py b/posthog/settings/web.py index 58710587636805..2a8cae6f2f2813 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -47,10 +47,11 @@ "django_structlog.middlewares.RequestMiddleware", "django_structlog.middlewares.CeleryMiddleware", "django.middleware.security.SecurityMiddleware", - "posthog.middleware.AllowIPMiddleware", + "posthog.middleware.ShortCircuitMiddleware", # NOTE: we need healthcheck high up to avoid hitting middlewares that may be # using dependencies that the healthcheck should be checking. It should be # ok below the above middlewares however. + "posthog.middleware.AllowIPMiddleware", "posthog.health.healthcheck_middleware", "google.cloud.sqlcommenter.django.middleware.SqlCommenter", "django.contrib.sessions.middleware.SessionMiddleware", From b420da0a253e0b0ff693cd095c59c3e6cb8b7c90 Mon Sep 17 00:00:00 2001 From: Ben White Date: Wed, 13 Jul 2022 17:00:38 +0200 Subject: [PATCH 047/213] fix: Add Slack to preflight (#10764) * Added slack service info to preflight * Correct message for non-staff users Co-authored-by: Neil Kakkar --- cypress/fixtures/_preflight.json | 3 + .../SubscriptionsModal.stories.tsx | 1 + .../scenes/PreflightCheck/preflightLogic.tsx | 1 + .../insights/EmptyStates/timeout-state.json | 3 + .../project/Settings/SlackIntegration.tsx | 69 ++--- .../project/Settings/integrationsLogic.ts | 11 +- frontend/src/types.ts | 4 + posthog/api/test/test_preflight.py | 239 ++++++------------ posthog/models/test/test_integration_model.py | 22 ++ posthog/test/test_middleware.py | 2 +- posthog/views.py | 4 + 11 files changed, 158 insertions(+), 201 deletions(-) create mode 100644 posthog/models/test/test_integration_model.py diff --git a/cypress/fixtures/_preflight.json b/cypress/fixtures/_preflight.json index 3e9eb2e745e54c..d8bb36cffbd31b 100644 --- a/cypress/fixtures/_preflight.json +++ b/cypress/fixtures/_preflight.json @@ -458,6 +458,9 @@ "opt_out_capture": false, "posthog_version": "1.23.1", "email_service_available": false, + "slack_service": { + "available": false + }, "is_debug": 1, "is_event_property_usage_enabled": false } diff --git a/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx b/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx index f7900fe7a12773..cd2a9bdc8500d1 100644 --- a/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx +++ b/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx @@ -32,6 +32,7 @@ const Template = ( ...preflightJson, realm: Realm.Cloud, email_service_available: noIntegrations ? false : true, + slack_service: noIntegrations ? { available: false } : { available: true, client_id: 'test-client-id' }, site_url: noIntegrations ? 'bad-value' : window.location.origin, }, '/api/projects/:id/subscriptions': { diff --git a/frontend/src/scenes/PreflightCheck/preflightLogic.tsx b/frontend/src/scenes/PreflightCheck/preflightLogic.tsx index 7bf6f4ec449e17..82f8bbe0e72860 100644 --- a/frontend/src/scenes/PreflightCheck/preflightLogic.tsx +++ b/frontend/src/scenes/PreflightCheck/preflightLogic.tsx @@ -250,6 +250,7 @@ export const preflightLogic = kea([ posthog_version: values.preflight.posthog_version, realm: values.realm, email_service_available: values.preflight.email_service_available, + slack_service_available: values.preflight.slack_service?.available, }) if (values.preflight.site_url) { diff --git a/frontend/src/scenes/insights/EmptyStates/timeout-state.json b/frontend/src/scenes/insights/EmptyStates/timeout-state.json index 705a59b0676d8b..a445f919d969ce 100644 --- a/frontend/src/scenes/insights/EmptyStates/timeout-state.json +++ b/frontend/src/scenes/insights/EmptyStates/timeout-state.json @@ -189,6 +189,9 @@ }, "can_create_org": true, "email_service_available": false, + "slack_service": { + "available": false + }, "ee_available": true, "db_backend": "clickhouse", "available_timezones": { diff --git a/frontend/src/scenes/project/Settings/SlackIntegration.tsx b/frontend/src/scenes/project/Settings/SlackIntegration.tsx index de55db1e9b85bb..ade8ddec64ab02 100644 --- a/frontend/src/scenes/project/Settings/SlackIntegration.tsx +++ b/frontend/src/scenes/project/Settings/SlackIntegration.tsx @@ -7,11 +7,13 @@ import { IconDelete, IconSlack } from 'lib/components/icons' import { Modal } from 'antd' import { UserActivityIndicator } from 'lib/components/UserActivityIndicator/UserActivityIndicator' import { urls } from 'scenes/urls' +import { userLogic } from 'scenes/userLogic' export function SlackIntegration(): JSX.Element { const { slackIntegration, addToSlackButtonUrl } = useValues(integrationsLogic) const { deleteIntegration } = useActions(integrationsLogic) const [showSlackInstructions, setShowSlackInstructions] = useState(false) + const { user } = useValues(userLogic) const onDeleteClick = (): void => { Modal.confirm({ @@ -72,37 +74,44 @@ export function SlackIntegration(): JSX.Element { srcSet="https://platform.slack-edge.com/img/add_to_slack.png 1x, https://platform.slack-edge.com/img/add_to_slack@2x.png 2x" /> - ) : !showSlackInstructions ? ( - <> - setShowSlackInstructions(true)}> - Show Instructions - - - ) : ( - <> -
To get started
-

-

    -
  1. Copy the below Slack App Template
    2. -
  2. - Go to{' '} - - Slack Apps - -
    3. -
  3. Create an App using the provided template
    4. -
  4. - Go to Instance Settings and update the{' '} - "SLACK_" properties using the values from the App Credentials{' '} - section of your Slack Apps -
    5. -
+ ) : user?.is_staff ? ( + !showSlackInstructions ? ( + <> + setShowSlackInstructions(true)}> + Show Instructions + + + ) : ( + <> +
To get started
+

+

    +
  1. Copy the below Slack App Template
    2. +
  2. + Go to{' '} + + Slack Apps + +
    3. +
  3. Create an App using the provided template
    4. +
  4. + Go to Instance Settings and update the{' '} + "SLACK_" properties using the values from the{' '} + App Credentials section of your Slack Apps +
    5. +
- - {JSON.stringify(getSlackAppManifest(), null, 2)} - -

- + + {JSON.stringify(getSlackAppManifest(), null, 2)} + +

+ + ) + ) : ( +

+ This PostHog instance is not configured for Slack. Please contact the instance owner to + configure it. +

)}

diff --git a/frontend/src/scenes/project/Settings/integrationsLogic.ts b/frontend/src/scenes/project/Settings/integrationsLogic.ts index be6bca794f43be..e585b28c6d2a3b 100644 --- a/frontend/src/scenes/project/Settings/integrationsLogic.ts +++ b/frontend/src/scenes/project/Settings/integrationsLogic.ts @@ -3,7 +3,6 @@ import { kea, path, listeners, selectors, connect, afterMount, actions } from 'k import { loaders } from 'kea-loaders' import { router, urlToAction } from 'kea-router' import api from 'lib/api' -import { systemStatusLogic } from 'scenes/instance/SystemStatus/systemStatusLogic' import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' import { urls } from 'scenes/urls' import { IntegrationType, SlackChannelType } from '~/types' @@ -59,8 +58,7 @@ export const getSlackAppManifest = (): any => ({ export const integrationsLogic = kea([ path(['scenes', 'project', 'Settings', 'integrationsLogic']), connect({ - values: [preflightLogic, ['siteUrlMisconfigured', 'preflight'], systemStatusLogic, ['instanceSettings']], - actions: [systemStatusLogic, ['loadInstanceSettings']], + values: [preflightLogic, ['siteUrlMisconfigured', 'preflight']], }), actions({ @@ -134,7 +132,6 @@ export const integrationsLogic = kea([ })), afterMount(({ actions }) => { actions.loadIntegrations() - actions.loadInstanceSettings() }), urlToAction(({ actions }) => ({ @@ -165,10 +162,10 @@ export const integrationsLogic = kea([ }, ], addToSlackButtonUrl: [ - (s) => [s.instanceSettings], - (instanceSettings) => { + (s) => [s.preflight], + (preflight) => { return (next: string = '') => { - const clientId = instanceSettings.find((item) => item.key === 'SLACK_APP_CLIENT_ID')?.value + const clientId = preflight?.slack_service?.client_id return clientId ? `https://slack.com/oauth/v2/authorize?client_id=${clientId}&scope=channels:read,groups:read,chat:write&redirect_uri=${encodeURIComponent( diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 7c331f1af1ae92..53869af81730cb 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1427,6 +1427,10 @@ export interface PreflightStatus { opt_out_capture?: boolean posthog_version?: string email_service_available: boolean + slack_service: { + available: boolean + client_id?: string + } /** Whether PostHog is running in DEBUG mode. */ is_debug?: boolean is_event_property_usage_enabled?: boolean diff --git a/posthog/api/test/test_preflight.py b/posthog/api/test/test_preflight.py index e58e17bb675369..b8096bfb8c0652 100644 --- a/posthog/api/test/test_preflight.py +++ b/posthog/api/test/test_preflight.py @@ -21,6 +21,48 @@ def instance_preferences(self, **kwargs): **kwargs, } + def preflight_dict(self, options={}): + preflight = { + "django": True, + "redis": True, + "plugins": True, + "celery": True, + "db": True, + "initiated": True, + "cloud": False, + "demo": False, + "clickhouse": True, + "kafka": True, + "realm": "hosted-clickhouse", + "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, + "can_create_org": False, + "email_service_available": False, + "slack_service": {"available": False, "client_id": None}, + "object_storage": False, + } + + preflight.update(options) + + return preflight + + def preflight_authenticated_dict(self, options={}): + preflight = { + "opt_out_capture": False, + "posthog_version": VERSION, + "is_debug": False, + "is_event_property_usage_enabled": True, + "licensed_users_available": None, + "site_url": "http://localhost:8000", + "can_create_org": False, + "instance_preferences": {"debug_queries": True, "disable_paid_fs": False,}, + "object_storage": False, + "buffer_conversion_seconds": 60, + } + + preflight.update(options) + + return self.preflight_dict(preflight) + def test_preflight_request_unauthenticated(self): """ For security purposes, the information contained in an unauthenticated preflight request is minimal. @@ -30,26 +72,7 @@ def test_preflight_request_unauthenticated(self): response = self.client.get("/_preflight/") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual( - response.json(), - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": False, - "demo": False, - "clickhouse": True, - "kafka": True, - "realm": "hosted-clickhouse", - "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, - "can_create_org": False, - "email_service_available": False, - "object_storage": False, - }, - ) + self.assertEqual(response.json(), self.preflight_dict()) def test_preflight_request(self): with self.settings( @@ -62,34 +85,7 @@ def test_preflight_request(self): response = response.json() available_timezones = cast(dict, response).pop("available_timezones") - self.assertEqual( - response, - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": False, - "demo": False, - "clickhouse": True, - "kafka": True, - "realm": "hosted-clickhouse", - "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, - "opt_out_capture": False, - "posthog_version": VERSION, - "email_service_available": False, - "is_debug": False, - "is_event_property_usage_enabled": True, - "licensed_users_available": None, - "site_url": "http://localhost:8000", - "can_create_org": False, - "instance_preferences": {"debug_queries": True, "disable_paid_fs": False,}, - "object_storage": False, - "buffer_conversion_seconds": 60, - }, - ) + self.assertEqual(response, self.preflight_authenticated_dict()) self.assertDictContainsSubset({"Europe/Moscow": 3, "UTC": 0}, available_timezones) @patch("posthog.storage.object_storage._client") @@ -106,39 +102,13 @@ def test_preflight_request_with_object_storage_available(self, patched_s3_client response = response.json() available_timezones = cast(dict, response).pop("available_timezones") - self.assertEqual( - response, - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": False, - "demo": False, - "clickhouse": True, - "kafka": True, - "realm": "hosted-clickhouse", - "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, - "opt_out_capture": False, - "posthog_version": VERSION, - "email_service_available": False, - "is_debug": False, - "is_event_property_usage_enabled": True, - "licensed_users_available": None, - "site_url": "http://localhost:8000", - "can_create_org": False, - "instance_preferences": {"debug_queries": True, "disable_paid_fs": False,}, - "object_storage": True, - "buffer_conversion_seconds": 60, - }, - ) + self.assertEqual(response, self.preflight_authenticated_dict({"object_storage": True})) self.assertDictContainsSubset({"Europe/Moscow": 3, "UTC": 0}, available_timezones) @pytest.mark.ee def test_cloud_preflight_request_unauthenticated(self): set_instance_setting("EMAIL_HOST", "localhost") + set_instance_setting("SLACK_APP_CLIENT_ID", "slack-client-id") self.client.logout() # make sure it works anonymously @@ -148,23 +118,15 @@ def test_cloud_preflight_request_unauthenticated(self): self.assertEqual( response.json(), - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": True, - "demo": False, - "clickhouse": True, - "kafka": True, - "realm": "cloud", - "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, - "can_create_org": True, - "email_service_available": True, - "object_storage": False, - }, + self.preflight_dict( + { + "email_service_available": True, + "slack_service": {"available": True, "client_id": "slack-client-id",}, + "can_create_org": True, + "cloud": True, + "realm": "cloud", + } + ), ) @pytest.mark.ee @@ -177,31 +139,15 @@ def test_cloud_preflight_request(self): self.assertEqual( response, - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": True, - "demo": False, - "clickhouse": True, - "kafka": True, - "realm": "cloud", - "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, - "opt_out_capture": False, - "posthog_version": VERSION, - "email_service_available": False, - "is_debug": False, - "is_event_property_usage_enabled": True, - "licensed_users_available": None, - "site_url": "https://app.posthog.com", - "can_create_org": True, - "instance_preferences": {"debug_queries": False, "disable_paid_fs": False,}, - "object_storage": False, - "buffer_conversion_seconds": 60, - }, + self.preflight_authenticated_dict( + { + "can_create_org": True, + "cloud": True, + "realm": "cloud", + "instance_preferences": {"debug_queries": False, "disable_paid_fs": False,}, + "site_url": "https://app.posthog.com", + } + ), ) self.assertDictContainsSubset({"Europe/Moscow": 3, "UTC": 0}, available_timezones) @@ -223,31 +169,17 @@ def test_cloud_preflight_request_with_social_auth_providers(self): self.assertEqual( response, - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": True, - "demo": False, - "clickhouse": True, - "kafka": True, - "realm": "cloud", - "available_social_auth_providers": {"google-oauth2": True, "github": False, "gitlab": False,}, - "opt_out_capture": False, - "posthog_version": VERSION, - "email_service_available": True, - "is_debug": False, - "is_event_property_usage_enabled": True, - "licensed_users_available": None, - "site_url": "http://localhost:8000", - "can_create_org": True, - "instance_preferences": {"debug_queries": False, "disable_paid_fs": True,}, - "object_storage": False, - "buffer_conversion_seconds": 60, - }, + self.preflight_authenticated_dict( + { + "can_create_org": True, + "cloud": True, + "realm": "cloud", + "instance_preferences": {"debug_queries": False, "disable_paid_fs": True,}, + "site_url": "http://localhost:8000", + "available_social_auth_providers": {"google-oauth2": True, "github": False, "gitlab": False,}, + "email_service_available": True, + } + ), ) self.assertDictContainsSubset({"Europe/Moscow": 3, "UTC": 0}, available_timezones) @@ -259,26 +191,7 @@ def test_demo(self): response = self.client.get("/_preflight/") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual( - response.json(), - { - "django": True, - "redis": True, - "plugins": True, - "celery": True, - "db": True, - "initiated": True, - "cloud": False, - "demo": True, - "clickhouse": True, - "kafka": True, - "realm": "demo", - "available_social_auth_providers": {"google-oauth2": False, "github": False, "gitlab": False,}, - "can_create_org": True, - "email_service_available": False, - "object_storage": False, - }, - ) + self.assertEqual(response.json(), self.preflight_dict({"demo": True, "can_create_org": True, "realm": "demo"})) @pytest.mark.ee @pytest.mark.skip_on_multitenancy diff --git a/posthog/models/test/test_integration_model.py b/posthog/models/test/test_integration_model.py new file mode 100644 index 00000000000000..9459798caad621 --- /dev/null +++ b/posthog/models/test/test_integration_model.py @@ -0,0 +1,22 @@ +from posthog.models.instance_setting import set_instance_setting +from posthog.models.integration import SlackIntegration +from posthog.test.base import BaseTest + + +class TestIntgerationModel(BaseTest): + def test_slack_integration_config(self): + set_instance_setting("SLACK_APP_CLIENT_ID", None) + set_instance_setting("SLACK_APP_CLIENT_SECRET", None) + set_instance_setting("SLACK_APP_SIGNING_SECRET", None) + + assert not SlackIntegration.slack_config() == {} + + set_instance_setting("SLACK_APP_CLIENT_ID", "client-id") + set_instance_setting("SLACK_APP_CLIENT_SECRET", "client-secret") + set_instance_setting("SLACK_APP_SIGNING_SECRET", "not-so-secret") + + assert SlackIntegration.slack_config() == { + "SLACK_APP_CLIENT_ID": "client-id", + "SLACK_APP_CLIENT_SECRET": "client-secret", + "SLACK_APP_SIGNING_SECRET": "not-so-secret", + } diff --git a/posthog/test/test_middleware.py b/posthog/test/test_middleware.py index 8a11902e4d2255..b902ab0eff095d 100644 --- a/posthog/test/test_middleware.py +++ b/posthog/test/test_middleware.py @@ -92,7 +92,7 @@ def test_trust_all_proxies(self): class TestAutoProjectMiddleware(APIBaseTest): # How many queries are made in the base app # On Cloud there's an additional multi_tenancy_organizationbilling query - BASE_APP_NUM_QUERIES = 38 if not settings.MULTI_TENANCY else 39 + BASE_APP_NUM_QUERIES = 39 if not settings.MULTI_TENANCY else 40 second_team: Team diff --git a/posthog/views.py b/posthog/views.py index 4815fc6612fbe3..84e65c78ef352b 100644 --- a/posthog/views.py +++ b/posthog/views.py @@ -15,6 +15,7 @@ from posthog.email import is_email_available from posthog.health import is_clickhouse_connected, is_kafka_connected from posthog.models import Organization, User +from posthog.models.integration import SlackIntegration from posthog.utils import ( get_available_timezones_with_offsets, get_can_create_org, @@ -91,6 +92,8 @@ def security_txt(request): @never_cache def preflight_check(request: HttpRequest) -> JsonResponse: + slack_client_id = SlackIntegration.slack_config().get("SLACK_APP_CLIENT_ID") + response = { "django": True, "redis": is_redis_alive() or settings.TEST, @@ -106,6 +109,7 @@ def preflight_check(request: HttpRequest) -> JsonResponse: "available_social_auth_providers": get_instance_available_sso_providers(), "can_create_org": get_can_create_org(request.user), "email_service_available": is_email_available(with_absolute_urls=True), + "slack_service": {"available": bool(slack_client_id), "client_id": slack_client_id or None}, "object_storage": is_object_storage_available(), } From 3948ac50594508011e354f536633fa0204ac182a Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 13 Jul 2022 16:14:47 +0100 Subject: [PATCH 048/213] chore: update chart js (#10768) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7331606d79e210..a65e7eaa20e174 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "antd": "^4.17.1", "antd-dayjs-webpack-plugin": "^1.0.6", "babel-preset-nano-react-app": "^0.1.0", - "chart.js": "^3.6.2", + "chart.js": "^3.8.0", "chartjs-adapter-dayjs": "^1.0.0", "chartjs-plugin-crosshair": "^1.2.0", "clsx": "^1.1.1", diff --git a/yarn.lock b/yarn.lock index bb3c3aca1463d8..0b7cc54365f394 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6659,10 +6659,10 @@ charenc@0.0.2: resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" integrity sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc= -chart.js@^3.6.2: - version "3.6.2" - resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-3.6.2.tgz#47342c551f688ffdda2cd53b534cb7e461ecec33" - integrity sha512-Xz7f/fgtVltfQYWq0zL1Xbv7N2inpG+B54p3D5FSvpCdy3sM+oZhbqa42eNuYXltaVvajgX5UpKCU2GeeJIgxg== +chart.js@^3.8.0: + version "3.8.0" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-3.8.0.tgz#c6c14c457b9dc3ce7f1514a59e9b262afd6f1a94" + integrity sha512-cr8xhrXjLIXVLOBZPkBZVF6NDeiVIrPLHcMhnON7UufudL+CNeRrD+wpYanswlm8NpudMdrt3CHoLMQMxJhHRg== chartjs-adapter-dayjs@^1.0.0: version "1.0.0" From 0d526789fae7f844e3478a303b38161165631e64 Mon Sep 17 00:00:00 2001 From: Alex Gyujin Kim Date: Wed, 13 Jul 2022 11:31:48 -0400 Subject: [PATCH 049/213] fix(insight-tooltips): height clipping row contents (#10771) --- frontend/src/scenes/insights/InsightTooltip/InsightTooltip.scss | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.scss b/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.scss index 0b1e0411aa603c..c5323cc6924c98 100644 --- a/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.scss +++ b/frontend/src/scenes/insights/InsightTooltip/InsightTooltip.scss @@ -61,7 +61,6 @@ font-weight: 600; display: flex; align-items: center; - height: inherit; .ant-typography { text-align: left; From 01bfd4d7936d9eefbd7fd138f9cbb78e47fcaf1b Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 13 Jul 2022 19:06:57 +0100 Subject: [PATCH 050/213] feat: new toolbar settings design (#10500) * convert to kea 3 with broken form * first step layout change * loading state * fix editing values * fixup adding new urls * disable save on errors * remove outdated TODO * add toolbar enable switch * better button separation * rename refactor * fix width to handle long domains * oops, checking 0 as truthy * handle empty states * remove unused css * listen for submit success to reset form * set starting value of form input from logic * tests suggest validation is correct but no errors show up in the actual UI * merge from master the right way around * let someone submit form to get errors triggered * right align the buttons * put padding back on suggestion button * simplify logic around updating edit url index * let lemon row be tall without being chunky * add a confirmation to removing authorised URL * show empty state if not adding a new entry * punctuation * bonfire of the css * even more CSS destruction * no breakpoint * even more custom style removal * maybe even delete the empty file * correct toolbar footer helper class changes * delete another empty file * remove unnecessary clsx * space-x * url can be the empty string when unset * flex-grow * removes row and col * highlight on hover * doesn't need space-x with or without dot * switch to standard hover highlight --- .../AppEditorLink/AppEditorLink.tsx | 4 +- .../src/lib/components/LemonRow/LemonRow.scss | 8 +- .../components/LemonRow/LemonRow.stories.tsx | 9 + .../src/lib/components/LemonRow/LemonRow.tsx | 9 +- .../src/lib/components/LemonTag/LemonTag.scss | 4 + .../src/lib/components/LemonTag/LemonTag.tsx | 2 +- .../src/scenes/actions/NewActionButton.tsx | 4 +- .../project/Settings/ToolbarSettings.tsx | 1 - .../src/scenes/project/Settings/index.tsx | 4 +- .../scenes/toolbar-launch/AuthorizedUrls.scss | 11 + .../scenes/toolbar-launch/AuthorizedUrls.tsx | 206 ++++++++++++++++++ .../toolbar-launch/AuthorizedUrlsTable.scss | 20 -- .../toolbar-launch/AuthorizedUrlsTable.tsx | 176 --------------- .../scenes/toolbar-launch/ToolbarLaunch.scss | 20 +- .../scenes/toolbar-launch/ToolbarLaunch.tsx | 48 +++- ...ic.test.ts => authorizedUrlsLogic.test.ts} | 13 ++ .../toolbar-launch/authorizedUrlsLogic.ts | 162 ++++++++++---- frontend/src/styles/global.scss | 25 +++ 18 files changed, 457 insertions(+), 269 deletions(-) create mode 100644 frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss create mode 100644 frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx delete mode 100644 frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.scss delete mode 100644 frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx rename frontend/src/scenes/toolbar-launch/{authorizedUlrsLogic.test.ts => authorizedUrlsLogic.test.ts} (74%) diff --git a/frontend/src/lib/components/AppEditorLink/AppEditorLink.tsx b/frontend/src/lib/components/AppEditorLink/AppEditorLink.tsx index 28a7fcca2a25e9..31963b6ca97aab 100644 --- a/frontend/src/lib/components/AppEditorLink/AppEditorLink.tsx +++ b/frontend/src/lib/components/AppEditorLink/AppEditorLink.tsx @@ -2,7 +2,7 @@ import React, { useState } from 'react' import { useValues } from 'kea' import { teamLogic } from 'scenes/teamLogic' import { Modal, Button } from 'antd' -import { AuthorizedUrlsTable } from 'scenes/toolbar-launch/AuthorizedUrlsTable' +import { AuthorizedUrls } from 'scenes/toolbar-launch/AuthorizedUrls' import { appEditorUrl } from 'scenes/toolbar-launch/authorizedUrlsLogic' export function AppEditorLink({ @@ -40,7 +40,7 @@ export function AppEditorLink({ footer={} onCancel={() => setModalOpen(false)} > - + ) diff --git a/frontend/src/lib/components/LemonRow/LemonRow.scss b/frontend/src/lib/components/LemonRow/LemonRow.scss index e01742199a1709..ba019b636553e5 100644 --- a/frontend/src/lib/components/LemonRow/LemonRow.scss +++ b/frontend/src/lib/components/LemonRow/LemonRow.scss @@ -103,10 +103,9 @@ padding: 0; } -.LemonRow--large { +.LemonRow--tall { min-height: 3.5rem; padding: 0.5rem 1rem; - font-size: 1rem; &.LemonRow--symbolic { min-height: 0; @@ -114,6 +113,11 @@ width: 1.75rem; padding: 0; } +} + +.LemonRow--large { + @extend .LemonRow--tall; + font-size: 1rem; .LemonRow__icon { font-size: 1.75rem; diff --git a/frontend/src/lib/components/LemonRow/LemonRow.stories.tsx b/frontend/src/lib/components/LemonRow/LemonRow.stories.tsx index 8c104a8ec1884b..8e9a6e2fa04c63 100644 --- a/frontend/src/lib/components/LemonRow/LemonRow.stories.tsx +++ b/frontend/src/lib/components/LemonRow/LemonRow.stories.tsx @@ -65,16 +65,25 @@ Loading.args = { export const Small = Template.bind({}) Small.args = { + outlined: true, size: 'small', } +export const Tall = Template.bind({}) +Tall.args = { + outlined: true, + size: 'tall', +} + export const Large = Template.bind({}) Large.args = { + outlined: true, size: 'large', } export const FullWidth = Template.bind({}) FullWidth.args = { + outlined: true, fullWidth: true, } diff --git a/frontend/src/lib/components/LemonRow/LemonRow.tsx b/frontend/src/lib/components/LemonRow/LemonRow.tsx index 641b4c19889e82..df2c348296e235 100644 --- a/frontend/src/lib/components/LemonRow/LemonRow.tsx +++ b/frontend/src/lib/components/LemonRow/LemonRow.tsx @@ -31,8 +31,12 @@ export interface LemonRowPropsBase center?: boolean /** Whether the element should be outlined with a standard border. */ outlined?: any - /** Variation on sizes - default is medium. Small looks better inline with text. Large is a chunkier row. */ - size?: 'small' | 'medium' | 'large' + /** Variation on sizes - default is medium. + * Small looks better inline with text. + * Large is a chunkier row. + * Tall is a chunkier row without changing font size. + * */ + size?: 'small' | 'medium' | 'tall' | 'large' 'data-attr'?: string } @@ -82,6 +86,7 @@ export const LemonRow = React.forwardRef(function LemonRowInternal { type?: LemonTagPropsType children: JSX.Element | string diff --git a/frontend/src/scenes/actions/NewActionButton.tsx b/frontend/src/scenes/actions/NewActionButton.tsx index 388dec5ecb5307..de6af9821f499a 100644 --- a/frontend/src/scenes/actions/NewActionButton.tsx +++ b/frontend/src/scenes/actions/NewActionButton.tsx @@ -3,7 +3,7 @@ import { Modal, Button, Card, Row, Col } from 'antd' import { SearchOutlined } from '@ant-design/icons' import { router } from 'kea-router' import { urls } from 'scenes/urls' -import { AuthorizedUrlsTable } from 'scenes/toolbar-launch/AuthorizedUrlsTable' +import { AuthorizedUrls } from 'scenes/toolbar-launch/AuthorizedUrls' import { IconEdit } from 'lib/components/icons' import { LemonButton } from 'lib/components/LemonButton' import { useValues } from 'kea' @@ -73,7 +73,7 @@ export function NewActionButton(): JSX.Element { )} - {appUrlsVisible && } + {appUrlsVisible && } ) diff --git a/frontend/src/scenes/project/Settings/ToolbarSettings.tsx b/frontend/src/scenes/project/Settings/ToolbarSettings.tsx index 6b6ce8a7c4dc41..babbb6e2f26a03 100644 --- a/frontend/src/scenes/project/Settings/ToolbarSettings.tsx +++ b/frontend/src/scenes/project/Settings/ToolbarSettings.tsx @@ -3,7 +3,6 @@ import { useValues, useActions } from 'kea' import { userLogic } from 'scenes/userLogic' import { Col, Row, Switch } from 'antd' -/* TODO: This should be moved to user's settings (good first issue) */ export function ToolbarSettings(): JSX.Element { const { user, userLoading } = useValues(userLogic) const { updateUser } = useActions(userLogic) diff --git a/frontend/src/scenes/project/Settings/index.tsx b/frontend/src/scenes/project/Settings/index.tsx index bed2edcc74108f..86ac0bad360be2 100644 --- a/frontend/src/scenes/project/Settings/index.tsx +++ b/frontend/src/scenes/project/Settings/index.tsx @@ -29,7 +29,7 @@ import { SceneExport } from 'scenes/sceneTypes' import { CorrelationConfig } from './CorrelationConfig' import { urls } from 'scenes/urls' import { LemonTag } from 'lib/components/LemonTag/LemonTag' -import { AuthorizedUrlsTable } from 'scenes/toolbar-launch/AuthorizedUrlsTable' +import { AuthorizedUrls } from 'scenes/toolbar-launch/AuthorizedUrls' import { GroupAnalytics } from 'scenes/project/Settings/GroupAnalytics' import { IconInfo, IconRefresh } from 'lib/components/icons' import { PersonDisplayNameProperties } from './PersonDisplayNameProperties' @@ -266,7 +266,7 @@ export function ProjectSettings(): JSX.Element { Domains and wilcard subdomains are allowed (example: https://*.example.com). However, wildcarded top-level domains cannot be used (for security reasons).

- +

Data attributes diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss new file mode 100644 index 00000000000000..9c8a55a56e856c --- /dev/null +++ b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss @@ -0,0 +1,11 @@ +.AuthorizedUrlRow { + &:hover { + background: var(--primary-bg-hover); + } + + .Url { + span { + max-width: 80%; + } + } +} diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx new file mode 100644 index 00000000000000..2de4676fa8ed17 --- /dev/null +++ b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx @@ -0,0 +1,206 @@ +import React from 'react' +import './AuthorizedUrls.scss' +import clsx from 'clsx' +import { useActions, useValues } from 'kea' +import { LemonTag } from 'lib/components/LemonTag/LemonTag' +import { LemonButton } from 'lib/components/LemonButton' +import { Input, Popconfirm } from 'antd' +import { authorizedUrlsLogic } from './authorizedUrlsLogic' +import { isMobile } from 'lib/utils' +import { LemonRow } from 'lib/components/LemonRow' +import { IconDelete, IconEdit, IconOpenInApp, IconPlus } from 'lib/components/icons' +import { Spinner } from 'lib/components/Spinner/Spinner' +import { Form } from 'kea-forms' +import { LemonInput } from 'lib/components/LemonInput/LemonInput' +import { Field } from 'lib/forms/Field' +import Typography from 'antd/lib/typography' + +interface AuthorizedUrlsTableInterface { + pageKey?: string + actionId?: number +} + +function EmptyState({ + numberOfResults, + isSearching, + isAddingEntry, +}: { + numberOfResults: number + isSearching: boolean + isAddingEntry: boolean +}): JSX.Element | null { + if (numberOfResults > 0) { + return null + } + + return isSearching ? ( + + There are no authorized URLs that match your search. + + ) : isAddingEntry ? null : ( + + There are no authorized URLs or domains. Add one to get started. + + ) +} + +function AuthorizedUrlForm({ actionId }: { actionId?: number }): JSX.Element { + const logic = authorizedUrlsLogic({ actionId }) + const { isProposedUrlSubmitting } = useValues(logic) + const { cancelProposingUrl } = useActions(logic) + return ( +
+ + + +
+ + Cancel + + + Save + +
+
+ ) +} + +export function AuthorizedUrls({ pageKey, actionId }: AuthorizedUrlsTableInterface): JSX.Element { + const logic = authorizedUrlsLogic({ actionId }) + const { appUrlsKeyed, suggestionsLoading, searchTerm, launchUrl, editUrlIndex, isAddUrlFormVisible } = + useValues(logic) + const { addUrl, removeUrl, setSearchTerm, newUrl, setEditUrlIndex } = useActions(logic) + + return ( +
+
+
+ { + setSearchTerm(e.target.value) + }} + autoFocus={pageKey === 'toolbar-launch' && !isMobile()} + /> +
+ } data-attr="toolbar-add-url"> + Add{pageKey === 'toolbar-launch' && ' authorized URL'} + +
+ {suggestionsLoading ? ( + + + + ) : ( + <> + {isAddUrlFormVisible && ( + + + + )} + 0} + isAddingEntry={isAddUrlFormVisible} + /> + {appUrlsKeyed.map((keyedAppURL, index) => { + return ( + + {editUrlIndex === index ? ( + + ) : ( + <> +
+ {keyedAppURL.type === 'suggestion' && ( + + Suggestion + + )} + + {keyedAppURL.url} + +
+
+ {keyedAppURL.type === 'suggestion' ? ( + addUrl(keyedAppURL.url)} + icon={} + outlined={false} + data-attr="toolbar-apply-suggestion" + > + Apply suggestion + + ) : ( + <> + } + href={launchUrl(keyedAppURL.url)} + tooltip={'Launch toolbar'} + center + className="ActionButton" + data-attr="toolbar-open" + /> + + } + onClick={() => setEditUrlIndex(keyedAppURL.originalIndex)} + tooltip={'Edit'} + center + className="ActionButton" + /> + Are you sure you want to remove this authorized url? + } + onConfirm={() => removeUrl(index)} + > + } + tooltip={'Remove URL'} + center + className="ActionButton" + /> + + + )} +
+ + )} + + ) + })} + + )} +
+ ) +} diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.scss b/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.scss deleted file mode 100644 index de12a29297a17e..00000000000000 --- a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.scss +++ /dev/null @@ -1,20 +0,0 @@ -.authorized-urls-table { - .authorized-url-col { - display: flex; - align-items: center; - .LemonTag { - margin-left: 6px; - } - &.suggestion { - color: var(--text-muted-alt); - } - &.authorized { - color: var(--success); - } - } - .actions-col { - display: flex; - align-items: center; - justify-content: flex-end; - } -} diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx b/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx deleted file mode 100644 index d210b7788febad..00000000000000 --- a/frontend/src/scenes/toolbar-launch/AuthorizedUrlsTable.tsx +++ /dev/null @@ -1,176 +0,0 @@ -import React, { useEffect, useState } from 'react' -import './AuthorizedUrlsTable.scss' -import clsx from 'clsx' -import { useActions, useValues } from 'kea' -import { LemonTable, LemonTableColumns } from 'lib/components/LemonTable' -import { LemonTag } from 'lib/components/LemonTag/LemonTag' -import { CheckCircleFilled } from '@ant-design/icons' -import { LemonButton } from 'lib/components/LemonButton' -import { Button, Input } from 'antd' -import { authorizedUrlsLogic, KeyedAppUrl, NEW_URL } from './authorizedUrlsLogic' -import { isMobile, isURL } from 'lib/utils' -import { More } from 'lib/components/LemonButton/More' - -interface AuthorizedUrlsTableInterface { - pageKey?: string - actionId?: number -} - -export function AuthorizedUrlsTable({ pageKey, actionId }: AuthorizedUrlsTableInterface): JSX.Element { - const logic = authorizedUrlsLogic({ actionId }) - const { appUrlsKeyed, suggestionsLoading, searchTerm, launchUrl, appUrls, editUrlIndex } = useValues(logic) - const { addUrl, removeUrl, setSearchTerm, updateUrl, newUrl, setEditUrlIndex } = useActions(logic) - - const columns: LemonTableColumns = [ - { - title: 'URLs', - dataIndex: 'url', - key: 'url', - render: function Render(url, record) { - const [urlUpdatingState, setUrlUpdatingState] = useState(record.url) - const [errorState, setErrorState] = useState('') - useEffect(() => setUrlUpdatingState(record.url), [record]) - const save = (): void => { - setErrorState('') - if (urlUpdatingState === NEW_URL) { - removeUrl(record.originalIndex) - } - // See https://regex101.com/r/UMBc9g/1 for tests - if ( - urlUpdatingState.indexOf('*') > -1 && - !urlUpdatingState.match(/^(.*)\*[^\*]*\.[^\*]+\.[^\*]+$/) - ) { - setErrorState( - 'You can only wildcard subdomains. If you wildcard the domain or TLD, people might be able to gain access to your PostHog data.' - ) - return - } - if (!isURL(urlUpdatingState)) { - setErrorState('Please type a valid URL or domain.') - return - } - - if ( - appUrls.indexOf(urlUpdatingState) > -1 && - appUrls.indexOf(urlUpdatingState, record.originalIndex) !== record.originalIndex && - appUrls.indexOf(urlUpdatingState, record.originalIndex + 1) !== record.originalIndex - ) { - setErrorState('This URL is already registered.') - return - } - - updateUrl(record.originalIndex, urlUpdatingState) - } - return record.type === 'suggestion' || (url !== NEW_URL && editUrlIndex !== record.originalIndex) ? ( -
- {record.type === 'authorized' && } - {url} - {record.type === 'suggestion' && Suggestion} -
- ) : ( -
-
- setUrlUpdatingState(e.target.value)} - onPressEnter={save} - autoFocus - placeholder="Enter a URL or wildcard subdomain (e.g. https://*.posthog.com)" - data-attr="url-input" - /> - -
- {errorState && {errorState}} -
- ) - }, - }, - { - title: '', - key: 'actions', - render: function Render(_, record, index) { - return ( -
- {record.type === 'suggestion' ? ( - addUrl(record.url)} - data-attr="toolbar-apply-suggestion" - > - Apply suggestion - - ) : ( - <> - - Open with Toolbar - - - setEditUrlIndex(record.originalIndex)} - > - Edit authorized URL - - removeUrl(index)} - > - Remove authorized URL - - - } - /> - - )} -
- ) - }, - }, - ] - - return ( -
-
-
- { - setSearchTerm(e.target.value) - }} - autoFocus={pageKey === 'toolbar-launch' && !isMobile()} - /> -
- - Add{pageKey === 'toolbar-launch' && ' authorized URL'} - -
- -
- ) -} diff --git a/frontend/src/scenes/toolbar-launch/ToolbarLaunch.scss b/frontend/src/scenes/toolbar-launch/ToolbarLaunch.scss index 20b1c986bf2e8f..c3ae218436eb28 100644 --- a/frontend/src/scenes/toolbar-launch/ToolbarLaunch.scss +++ b/frontend/src/scenes/toolbar-launch/ToolbarLaunch.scss @@ -1,31 +1,25 @@ .toolbar-launch-page { margin-bottom: 48px; - .footer-caption { - text-align: center; - color: var(--muted); - margin-top: 1rem; + .EnableToolbarSwitch { + max-width: 400px; } .feature-highlight-list { max-width: 800px; - margin: 0 auto; - margin-top: 2rem; .fh-item { - display: flex; - align-items: center; - margin-top: 1rem; + width: calc(100% * (1 / 2) - 8px); + + @media (max-width: 599px) { + width: calc(100% - 8px); + } .fh-icon { - color: var(--text-muted-alt); - margin-right: 8px; font-size: 1.6em; } h4 { - color: var(--text-muted-alt); - margin-bottom: 0; font-size: 1.2em; } } diff --git a/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx b/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx index ace5d4f23d06eb..f1f1d423eb8bdf 100644 --- a/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx +++ b/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx @@ -6,14 +6,20 @@ import { SearchOutlined } from '@ant-design/icons' import { Link } from 'lib/components/Link' import { urls } from 'scenes/urls' import { IconFlag, IconGroupedEvents, IconHeatmap } from 'lib/components/icons' -import { Col, Row } from 'antd' -import { AuthorizedUrlsTable } from './AuthorizedUrlsTable' +import { AuthorizedUrls } from './AuthorizedUrls' +import { LemonDivider } from 'lib/components/LemonDivider' +import { LemonSwitch } from 'lib/components/LemonSwitch/LemonSwitch' +import { useActions, useValues } from 'kea' +import { userLogic } from 'scenes/userLogic' export const scene: SceneExport = { component: ToolbarLaunch, } function ToolbarLaunch(): JSX.Element { + const { user, userLoading } = useValues(userLogic) + const { updateUser } = useActions(userLogic) + const features: FeatureHighlightProps[] = [ { title: 'Heatmaps', @@ -40,19 +46,41 @@ function ToolbarLaunch(): JSX.Element { return (
+ + + + updateUser({ + toolbar_mode: user?.toolbar_mode === 'disabled' ? 'toolbar' : 'disabled', + }) + } + checked={user?.toolbar_mode !== 'disabled'} + disabled={userLoading} + loading={userLoading} + type="primary" + className="EnableToolbarSwitch mt mb pt pb full-width" + /> - +

+ Authorized URLs for Toolbar +

+

+ These are the domains and URLs where the Toolbar will + automatically launch if you're signed in to your PostHog account. +

+ -
+
Make sure you're using the HTML snippet or the latest posthog-js version.
- +
{features.map((feature) => ( ))} - +
) } @@ -65,12 +93,12 @@ interface FeatureHighlightProps { function FeatureHighlight({ title, caption, icon }: FeatureHighlightProps): JSX.Element { return ( - -
{icon}
+
+
{icon}
-

{title}

+

{title}

{caption}
- +
) } diff --git a/frontend/src/scenes/toolbar-launch/authorizedUlrsLogic.test.ts b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts similarity index 74% rename from frontend/src/scenes/toolbar-launch/authorizedUlrsLogic.test.ts rename to frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts index f71834ded30290..df796c24d8927a 100644 --- a/frontend/src/scenes/toolbar-launch/authorizedUlrsLogic.test.ts +++ b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts @@ -39,4 +39,17 @@ describe('the authorized urls logic', () => { router.actions.push(urls.toolbarLaunch()) await expectLogic(logic).toNotHaveDispatchedActions(['newUrl']) }) + + describe('the proposed URL form', () => { + it('shows errors when the value is invalid', async () => { + await expectLogic(logic, () => { + logic.actions.setProposedUrlValue('url', 'not a domain or url') + }).toMatchValues({ + proposedUrl: { url: 'not a domain or url' }, + proposedUrlChanged: true, + proposedUrlHasErrors: true, + proposedUrlValidationErrors: { url: 'Please type a valid URL or domain.' }, + }) + }) + }) }) diff --git a/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts index 7686c0637639e6..9652e32f687f4b 100644 --- a/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts +++ b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts @@ -1,13 +1,50 @@ -import { kea } from 'kea' +import { + actions, + afterMount, + connect, + kea, + key, + listeners, + path, + props, + reducers, + selectors, + sharedListeners, +} from 'kea' import api from 'lib/api' -import { toParams } from 'lib/utils' +import { isURL, toParams } from 'lib/utils' import { EditorProps, TrendResult } from '~/types' import { teamLogic } from 'scenes/teamLogic' import { dayjs } from 'lib/dayjs' import Fuse from 'fuse.js' import type { authorizedUrlsLogicType } from './authorizedUrlsLogicType' -import { encodeParams } from 'kea-router' +import { encodeParams, urlToAction } from 'kea-router' import { urls } from 'scenes/urls' +import { loaders } from 'kea-loaders' +import { forms } from 'kea-forms' + +export interface ProposeNewUrlFormType { + url: string +} + +const validateProposedURL = (proposedUrl: string, currentUrls: string[]): string | undefined => { + if (proposedUrl === '') { + return 'Please type a valid URL or domain.' + } + // See https://regex101.com/r/UMBc9g/1 for tests + if (proposedUrl.indexOf('*') > -1 && !proposedUrl.match(/^(.*)\*[^*]*\.[^*]+\.[^*]+$/)) { + return 'You can only wildcard subdomains. If you wildcard the domain or TLD, people might be able to gain access to your PostHog data.' + } + if (!isURL(proposedUrl)) { + return 'Please type a valid URL or domain.' + } + + if (currentUrls.indexOf(proposedUrl) > -1) { + return 'This URL is already registered.' + } + + return +} /** defaultIntent: whether to launch with empty intent (i.e. toolbar mode is default) */ export function appEditorUrl(appUrl?: string, actionId?: number, defaultIntent?: boolean): string { @@ -27,18 +64,20 @@ export interface KeyedAppUrl { originalIndex: number } -export const authorizedUrlsLogic = kea({ - path: (key) => ['lib', 'components', 'AppEditorLink', 'appUrlsLogic', key], - key: (props) => `${props.pageKey}${props.actionId}` || 'global', - props: {} as { - actionId?: number - pageKey?: string - }, - connect: { +export const authorizedUrlsLogic = kea([ + path((key) => ['lib', 'components', 'AppEditorLink', 'appUrlsLogic', key]), + key((props) => `${props.pageKey}${props.actionId}` || 'global'), + props( + {} as { + actionId?: number + pageKey?: string + } + ), + connect({ values: [teamLogic, ['currentTeam', 'currentTeamId']], actions: [teamLogic, ['updateCurrentTeam']], - }, - actions: () => ({ + }), + actions(() => ({ setAppUrls: (appUrls: string[]) => ({ appUrls }), addUrl: (url: string, launch?: boolean) => ({ url, launch }), newUrl: true, @@ -47,9 +86,9 @@ export const authorizedUrlsLogic = kea({ launchAtUrl: (url: string) => ({ url }), setSearchTerm: (term: string) => ({ term }), setEditUrlIndex: (originalIndex: number | null) => ({ originalIndex }), - }), - - loaders: ({ values }) => ({ + cancelProposingUrl: true, + })), + loaders(({ values }) => ({ suggestions: { __default: [] as string[], loadSuggestions: async () => { @@ -88,22 +127,42 @@ export const authorizedUrlsLogic = kea({ .slice(0, 20) }, }, + })), + afterMount(({ actions, values }) => { + actions.loadSuggestions() + if (values.currentTeam) { + actions.setAppUrls(values.currentTeam.app_urls) + } }), - events: ({ actions, values }) => ({ - afterMount: () => { - actions.loadSuggestions() - if (values.currentTeam) { - actions.setAppUrls(values.currentTeam.app_urls) - } + forms(({ values, actions }) => ({ + proposedUrl: { + defaults: { url: '' } as ProposeNewUrlFormType, + errors: ({ url }) => ({ + url: validateProposedURL(url, values.appUrls), + }), + submit: async ({ url }) => { + if (values.editUrlIndex !== null && values.editUrlIndex >= 0) { + actions.updateUrl(values.editUrlIndex, url) + } else { + actions.addUrl(url) + } + }, }, - }), - reducers: () => ({ + })), + reducers(() => ({ + showProposedURLForm: [ + false as boolean, + { + newUrl: () => true, + submitProposedUrlSuccess: () => false, + cancelProposingUrl: () => false, + }, + ], appUrls: [ [] as string[], { setAppUrls: (_, { appUrls }) => appUrls, addUrl: (state, { url }) => state.concat([url]), - newUrl: (state) => (state.includes(NEW_URL) ? state : [NEW_URL].concat(state)), updateUrl: (state, { index, url }) => Object.assign([...state], { [index]: url }), removeUrl: (state, { index }) => { const newAppUrls = [...state] @@ -134,10 +193,25 @@ export const authorizedUrlsLogic = kea({ : index === editUrlIndex ? null : editUrlIndex, + newUrl: () => -1, + updateUrl: () => null, + addUrl: () => null, + cancelProposingUrl: () => null, }, ], - }), - listeners: ({ sharedListeners, values, actions }) => ({ + })), + sharedListeners(({ values }) => ({ + saveAppUrls: () => { + teamLogic.actions.updateCurrentTeam({ app_urls: values.appUrls }) + }, + })), + listeners(({ sharedListeners, values, actions }) => ({ + setEditUrlIndex: () => { + actions.setProposedUrlValue('url', values.urlToEdit) + }, + newUrl: () => { + actions.setProposedUrlValue('url', NEW_URL) + }, addUrl: [ sharedListeners.saveAppUrls, async ({ url, launch }) => { @@ -147,7 +221,7 @@ export const authorizedUrlsLogic = kea({ }, ], removeUrl: sharedListeners.saveAppUrls, - updateUrl: [sharedListeners.saveAppUrls, () => actions.setEditUrlIndex(null)], + updateUrl: sharedListeners.saveAppUrls, [teamLogic.actionTypes.loadCurrentTeamSuccess]: async ({ currentTeam }) => { if (currentTeam) { actions.setAppUrls(currentTeam.app_urls) @@ -156,13 +230,24 @@ export const authorizedUrlsLogic = kea({ launchAtUrl: ({ url }) => { window.location.href = values.launchUrl(url) }, - }), - sharedListeners: ({ values }) => ({ - saveAppUrls: () => { - teamLogic.actions.updateCurrentTeam({ app_urls: values.appUrls }) + cancelProposingUrl: () => { + actions.resetProposedUrl() }, - }), - selectors: ({ props }) => ({ + submitProposedUrlSuccess: () => { + actions.setEditUrlIndex(null) + actions.resetProposedUrl() + }, + })), + selectors(({ props }) => ({ + urlToEdit: [ + (s) => [s.appUrls, s.editUrlIndex], + (appUrls, editUrlIndex) => { + if (editUrlIndex === null || editUrlIndex === -1) { + return NEW_URL + } + return appUrls[editUrlIndex] + }, + ], appUrlsKeyed: [ (s) => [s.appUrls, s.suggestions, s.searchTerm], (appUrls, suggestions, searchTerm): KeyedAppUrl[] => { @@ -193,12 +278,13 @@ export const authorizedUrlsLogic = kea({ }, ], launchUrl: [() => [], () => (url: string) => appEditorUrl(url, props.actionId, !props.actionId)], - }), - urlToAction: ({ actions }) => ({ + isAddUrlFormVisible: [(s) => [s.editUrlIndex], (editUrlIndex) => editUrlIndex === -1], + })), + urlToAction(({ actions }) => ({ [urls.toolbarLaunch()]: (_, searchParams) => { if (searchParams.addNew) { actions.newUrl() } }, - }), -}) + })), +]) diff --git a/frontend/src/styles/global.scss b/frontend/src/styles/global.scss index 935ed5b0b54a4b..9e21a5281cda83 100644 --- a/frontend/src/styles/global.scss +++ b/frontend/src/styles/global.scss @@ -476,6 +476,11 @@ code.code { margin-left: 0.25rem; } +.mx-auto { + margin-left: auto; + margin-right: auto; +} + .mx-0 { margin-left: 0 !important; margin-right: 0 !important; @@ -609,6 +614,10 @@ code.code { justify-content: center; } +.justify-end { + justify-content: end; +} + .flex { display: flex; } @@ -618,6 +627,10 @@ code.code { flex-direction: column; } +.flex-row { + flex-direction: row; +} + .flex-wrap { flex-wrap: wrap; } @@ -626,6 +639,10 @@ code.code { flex: auto; } +.flex-grow { + flex-grow: 1; +} + .flex-center { display: flex; align-items: center; @@ -671,6 +688,14 @@ input::-ms-clear { color: var(--warning) !important; } +.text-muted { + color: var(--text-muted) !important; +} + +.text-muted-alt { + color: var(--text-muted-alt) !important; +} + // Random general styles .cursor-pointer { From 2d7f929aa6cdd7fec1165fd6d0ab2202e33efe19 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 13 Jul 2022 20:12:34 +0100 Subject: [PATCH 051/213] fix: test toolbar domain checks (#10777) * fix: URLs with wild cards are URLs too --- frontend/src/lib/utils.test.ts | 2 + frontend/src/lib/utils.tsx | 2 +- .../authorizedUrlsLogic.test.ts | 42 ++++++++++++++++++- .../toolbar-launch/authorizedUrlsLogic.ts | 5 ++- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/utils.test.ts b/frontend/src/lib/utils.test.ts index 1c8d170859ce93..47f814d9a92453 100644 --- a/frontend/src/lib/utils.test.ts +++ b/frontend/src/lib/utils.test.ts @@ -155,6 +155,8 @@ describe('isURL()', () => { expect(isURL('https://sevenapp.events/')).toEqual(true) expect(isURL('https://seven-stagingenv.web.app/')).toEqual(true) expect(isURL('https://salesforce.co.uk/')).toEqual(true) + expect(isURL('https://valid.*.example.com')).toEqual(true) + expect(isURL('https://*.valid.com')).toEqual(true) }) it('recognizes non-URLs properly', () => { diff --git a/frontend/src/lib/utils.tsx b/frontend/src/lib/utils.tsx index 4a68273d7403d1..4679fee5905016 100644 --- a/frontend/src/lib/utils.tsx +++ b/frontend/src/lib/utils.tsx @@ -604,7 +604,7 @@ export function isURL(input: any): boolean { return false } // Regex by regextester.com/115236 - const regexp = /^(?:http(s)?:\/\/)([\w.-])+(?:[\w\.-]+)+([\w\-\._~:/?#[\]@%!\$&'\(\)\*\+,;=.])+$/ + const regexp = /^(?:http(s)?:\/\/)([\w*.-])+(?:[\w*\.-]+)+([\w\-\._~:/?#[\]@%!\$&'\(\)\*\+,;=.])+$/ return !!input.trim().match(regexp) } diff --git a/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts index df796c24d8927a..cda14a1f929db6 100644 --- a/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts +++ b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.test.ts @@ -1,4 +1,4 @@ -import { appEditorUrl, authorizedUrlsLogic } from 'scenes/toolbar-launch/authorizedUrlsLogic' +import { appEditorUrl, authorizedUrlsLogic, validateProposedURL } from 'scenes/toolbar-launch/authorizedUrlsLogic' import { initKeaTests } from '~/test/init' import { router } from 'kea-router' import { expectLogic } from 'kea-test-utils' @@ -52,4 +52,44 @@ describe('the authorized urls logic', () => { }) }) }) + + describe('validating proposed URLs', () => { + const testCases = [ + { proposedUrl: 'https://valid.*.example.com', validityMessage: undefined }, + { + proposedUrl: 'https://notsovalid.*.*', + validityMessage: + 'You can only wildcard subdomains. If you wildcard the domain or TLD, people might be able to gain access to your PostHog data.', + }, + { + proposedUrl: 'https://*.*.*', + validityMessage: + 'You can only wildcard subdomains. If you wildcard the domain or TLD, people might be able to gain access to your PostHog data.', + }, + { proposedUrl: 'https://valid*.example.com', validityMessage: undefined }, + { proposedUrl: 'https://*.valid.com', validityMessage: undefined }, + { + proposedUrl: 'https://not.*.valid.*', + validityMessage: + 'You can only wildcard subdomains. If you wildcard the domain or TLD, people might be able to gain access to your PostHog data.', + }, + ] + + testCases.forEach((testCase) => { + it(`a proposal of "${testCase.proposedUrl}" has validity message "${testCase.validityMessage}"`, () => { + expect(validateProposedURL(testCase.proposedUrl, [])).toEqual(testCase.validityMessage) + }) + }) + + it('fails if the proposed URL is already authorized', () => { + expect(validateProposedURL('https://valid.*.example.com', ['https://valid.*.example.com'])).toBe( + 'This URL is already registered.' + ) + expect( + validateProposedURL('https://valid.and-not-already-authorized.example.com', [ + 'https://valid.*.example.com', + ]) + ).toBe(undefined) + }) + }) }) diff --git a/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts index 9652e32f687f4b..1b2f6007942a89 100644 --- a/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts +++ b/frontend/src/scenes/toolbar-launch/authorizedUrlsLogic.ts @@ -27,14 +27,15 @@ export interface ProposeNewUrlFormType { url: string } -const validateProposedURL = (proposedUrl: string, currentUrls: string[]): string | undefined => { +export const validateProposedURL = (proposedUrl: string, currentUrls: string[]): string | undefined => { if (proposedUrl === '') { return 'Please type a valid URL or domain.' } - // See https://regex101.com/r/UMBc9g/1 for tests + if (proposedUrl.indexOf('*') > -1 && !proposedUrl.match(/^(.*)\*[^*]*\.[^*]+\.[^*]+$/)) { return 'You can only wildcard subdomains. If you wildcard the domain or TLD, people might be able to gain access to your PostHog data.' } + if (!isURL(proposedUrl)) { return 'Please type a valid URL or domain.' } From 29d3fd9be70a3c440e4d9cbca894a92bebf30a41 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 13 Jul 2022 22:14:02 +0100 Subject: [PATCH 052/213] fix: toolbar launch styles (#10778) --- .../src/scenes/toolbar-launch/AuthorizedUrls.scss | 2 +- .../src/scenes/toolbar-launch/AuthorizedUrls.tsx | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss index 9c8a55a56e856c..d5ea38960cb62e 100644 --- a/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss +++ b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.scss @@ -1,5 +1,5 @@ .AuthorizedUrlRow { - &:hover { + &.highlight-on-hover:hover { background: var(--primary-bg-hover); } diff --git a/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx index 2de4676fa8ed17..e177bc0adad4f4 100644 --- a/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx +++ b/frontend/src/scenes/toolbar-launch/AuthorizedUrls.tsx @@ -112,9 +112,9 @@ export function AuthorizedUrls({ pageKey, actionId }: AuthorizedUrlsTableInterfa ) : ( - <> +
{isAddUrlFormVisible && ( - + )} @@ -130,7 +130,13 @@ export function AuthorizedUrls({ pageKey, actionId }: AuthorizedUrlsTableInterfa fullWidth size="tall" key={index} - className={clsx('AuthorizedUrlRow', keyedAppURL.type, 'flex-center', 'mb-05', 'mr')} + className={clsx( + 'AuthorizedUrlRow', + keyedAppURL.type, + 'flex-center', + 'mr', + editUrlIndex !== index && 'highlight-on-hover' + )} > {editUrlIndex === index ? ( @@ -199,7 +205,7 @@ export function AuthorizedUrls({ pageKey, actionId }: AuthorizedUrlsTableInterfa ) })} - +
)}
) From 5bf4a147cda8fdecc9dd4e1c187185001f08f67e Mon Sep 17 00:00:00 2001 From: Guido Iaquinti <4038041+guidoiaquinti@users.noreply.github.com> Date: Thu, 14 Jul 2022 11:47:44 +0200 Subject: [PATCH 053/213] chore(sentry-sdk): reduce traces sample rate (part #2) (#10786) --- posthog/settings/sentry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/settings/sentry.py b/posthog/settings/sentry.py index b965dad8a21e64..bb46375f7a81d8 100644 --- a/posthog/settings/sentry.py +++ b/posthog/settings/sentry.py @@ -21,5 +21,5 @@ request_bodies="always", sample_rate=1.0, # Configures the sample rate for error events, in the range of 0.0 to 1.0. The default is 1.0 which means that 100% of error events are sent. If set to 0.1 only 10% of error events will be sent. Events are picked randomly. send_default_pii=True, - traces_sample_rate=0.00001, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. + traces_sample_rate=0.0000001, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. ) From 675abe75b61c1c24630efe17d5058cfbb238a1cf Mon Sep 17 00:00:00 2001 From: Guido Iaquinti <4038041+guidoiaquinti@users.noreply.github.com> Date: Thu, 14 Jul 2022 12:12:56 +0200 Subject: [PATCH 054/213] chore: minor cleanup of bin folder (#10757) --- bin/pull_production_db | 8 -------- bin/start | 9 +-------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100755 bin/pull_production_db diff --git a/bin/pull_production_db b/bin/pull_production_db deleted file mode 100755 index 9b3e2ab0d6d062..00000000000000 --- a/bin/pull_production_db +++ /dev/null @@ -1,8 +0,0 @@ -read -r -p "Are you sure you want to pull production db? It'll destroy your local db [y/N] " response -if [[ "$response" =~ ^([yY][eE][sS]|[yY])+$ ]] -then - dropdb posthog; - heroku pg:pull DATABASE_URL posthog --app posthog -else - echo 'doing nothing'; -fi \ No newline at end of file diff --git a/bin/start b/bin/start index 1282a30e86e545..3679cc1347285d 100755 --- a/bin/start +++ b/bin/start @@ -7,15 +7,8 @@ trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT export DEBUG=${DEBUG:-1} export SKIP_SERVICE_VERSION_REQUIREMENTS=1 -ARCH=$(uname -m) -if [ "$ARCH" == "arm64" ]; then - DOCKER_COMPOSE_VARIANT='arm64' -else - DOCKER_COMPOSE_VARIANT='dev' -fi - service_warning() { - echo -e "\033[0;31m$1 isn't ready. You can run the stack with:\ndocker compose -f docker-compose.${DOCKER_COMPOSE_VARIANT}.yml up kafka clickhouse db redis\nIf you have already ran that, just make sure that services are starting properly, and sit back.\nWaiting for $1 to start...\033[0m" + echo -e "\033[0;31m$1 isn't ready. You can run the stack with:\ndocker compose -f docker-compose.dev.yml up\nIf you have already ran that, just make sure that services are starting properly, and sit back.\nWaiting for $1 to start...\033[0m" } nc -z localhost 9092 || ( service_warning 'Kafka'; bin/check_kafka_clickhouse_up ) From 648ac3820b5b11c096887668c8333460b39ad030 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 14 Jul 2022 11:14:13 +0100 Subject: [PATCH 055/213] feat: add sports hog to the export toast after 30 seconds (#10762) * feat: add sports hog to the export toast after 30 seconds * remove console log * small is small * hedgehog _is_ spinner * remove unnecesary custom styles * dancing hedgehog should be bigger in toast --- .../lib/components/Animation/Animation.scss | 14 +++++- .../Animation/Animation.stories.tsx | 48 ++++++++----------- .../lib/components/Animation/Animation.tsx | 3 ++ .../{exporter.ts => exporter.tsx} | 41 ++++++++++++++-- frontend/src/lib/components/lemonToast.tsx | 7 +-- 5 files changed, 75 insertions(+), 38 deletions(-) rename frontend/src/lib/components/ExportButton/{exporter.ts => exporter.tsx} (69%) diff --git a/frontend/src/lib/components/Animation/Animation.scss b/frontend/src/lib/components/Animation/Animation.scss index c195f1b6f85657..3799b7dbf77e25 100644 --- a/frontend/src/lib/components/Animation/Animation.scss +++ b/frontend/src/lib/components/Animation/Animation.scss @@ -1,5 +1,4 @@ .Animation { - width: 100%; max-width: 300px; // A correct aspect-ratio is be passed via a style prop. This is as a fallback. aspect-ratio: 1 / 1; @@ -23,4 +22,17 @@ display: block; } } + + &.Animation--large { + width: 100%; + } + + &.Animation--small { + overflow: visible; + + svg { + width: 45px !important; + height: 45px !important; + } + } } diff --git a/frontend/src/lib/components/Animation/Animation.stories.tsx b/frontend/src/lib/components/Animation/Animation.stories.tsx index 7fb32314d1b2db..fc6754ebf7bde1 100644 --- a/frontend/src/lib/components/Animation/Animation.stories.tsx +++ b/frontend/src/lib/components/Animation/Animation.stories.tsx @@ -1,13 +1,11 @@ import * as React from 'react' -import { animations, AnimationType } from '../../animations/animations' -import { Meta } from '@storybook/react' -import { LemonTable } from '../LemonTable' +import { AnimationType } from 'lib/animations/animations' +import { ComponentStory, Meta } from '@storybook/react' import { Animation } from 'lib/components/Animation/Animation' export default { title: 'Layout/Animations', parameters: { - options: { showPanel: false }, docs: { description: { component: @@ -15,30 +13,22 @@ export default { }, }, }, -} as Meta + argTypes: { + size: { + options: ['small', 'large'], + control: { type: 'radio' }, + }, + type: { + options: Object.values(AnimationType), + mapping: AnimationType, + control: { type: 'radio' }, + }, + }, +} as Meta -export function Animations(): JSX.Element { - return ( - ({ key }))} - columns={[ - { - title: 'Code', - key: 'code', - dataIndex: 'key', - render: function RenderCode(name) { - return {``} - }, - }, - { - title: 'Animation', - key: 'animation', - dataIndex: 'key', - render: function RenderAnimation(key) { - return - }, - }, - ]} - /> - ) +const Template: ComponentStory = ({ size, type }): JSX.Element => { + return } + +export const Animations = Template.bind({}) +Animations.args = { size: 'large' } diff --git a/frontend/src/lib/components/Animation/Animation.tsx b/frontend/src/lib/components/Animation/Animation.tsx index edc0d1c65dce31..fe643025d2fbcd 100644 --- a/frontend/src/lib/components/Animation/Animation.tsx +++ b/frontend/src/lib/components/Animation/Animation.tsx @@ -12,6 +12,7 @@ export interface AnimationProps { delay?: number className?: string style?: React.CSSProperties + size?: 'small' | 'large' } export function Animation({ @@ -19,6 +20,7 @@ export function Animation({ style, delay = 300, type = AnimationType.LaptopHog, + size = 'large', }: AnimationProps): JSX.Element { const [visible, setVisible] = useState(delay === 0) const [source, setSource] = useState>(null) @@ -57,6 +59,7 @@ export function Animation({ className={clsx( 'Animation', { 'Animation--hidden': !(visible && (source || showFallbackSpinner)) }, + `Animation--${size}`, className )} style={{ aspectRatio: `${width} / ${height}`, ...style }} diff --git a/frontend/src/lib/components/ExportButton/exporter.ts b/frontend/src/lib/components/ExportButton/exporter.tsx similarity index 69% rename from frontend/src/lib/components/ExportButton/exporter.ts rename to frontend/src/lib/components/ExportButton/exporter.tsx index 4f63d31b83e9f4..a18fbcb4c88972 100644 --- a/frontend/src/lib/components/ExportButton/exporter.ts +++ b/frontend/src/lib/components/ExportButton/exporter.tsx @@ -3,6 +3,11 @@ import { delay } from 'lib/utils' import posthog from 'posthog-js' import { ExportedAssetType, ExporterFormat } from '~/types' import { lemonToast } from '../lemonToast' +import { useEffect, useState } from 'react' +import React from 'react' +import { AnimationType } from 'lib/animations/animations' +import { Animation } from 'lib/components/Animation/Animation' +import { Spinner } from 'lib/components/Spinner/Spinner' const POLL_DELAY_MS = 1000 const MAX_PNG_POLL = 10 @@ -76,9 +81,35 @@ export async function triggerExport(asset: TriggerExportProps): Promise { reject(`Export failed: ${JSON.stringify(e)}`) } }) - await lemonToast.promise(poller, { - pending: 'Export started...', - success: 'Export complete!', - error: 'Export failed!', - }) + await lemonToast.promise( + poller, + { + pending: , + success: 'Export complete!', + error: 'Export failed!', + }, + { + pending: ( + } + afterDelay={} + /> + ), + } + ) +} + +interface DelayedContentProps { + atStart: JSX.Element | string + afterDelay: JSX.Element | string +} + +function DelayedContent({ atStart, afterDelay }: DelayedContentProps): JSX.Element { + const [content, setContent] = useState(atStart) + useEffect(() => { + setTimeout(() => { + setContent(afterDelay) + }, 30000) + }, []) + return <>{content} } diff --git a/frontend/src/lib/components/lemonToast.tsx b/frontend/src/lib/components/lemonToast.tsx index addf21ce5d0dc0..337f644b8b5259 100644 --- a/frontend/src/lib/components/lemonToast.tsx +++ b/frontend/src/lib/components/lemonToast.tsx @@ -97,6 +97,7 @@ export const lemonToast = { promise( promise: Promise, messages: { pending: string | JSX.Element; success: string | JSX.Element; error: string | JSX.Element }, + icons: { pending?: JSX.Element; success?: JSX.Element; error?: JSX.Element } = {}, { button, ...toastOptions }: ToastOptionsWithButton = {} ): Promise { toastOptions = ensureToastId(toastOptions) @@ -106,19 +107,19 @@ export const lemonToast = { { pending: { render: , - icon: , + icon: icons.pending ?? , }, success: { render({ data }: ToastifyRenderProps) { return }, - icon: , + icon: icons.success ?? , }, error: { render({ data }: ToastifyRenderProps) { return }, - icon: , + icon: icons.error ?? , }, }, toastOptions From 9d77ab6a1647d5196751976b0b95d997d83d5382 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 14 Jul 2022 11:18:46 +0100 Subject: [PATCH 056/213] fix: filters hash needs to be set on tiles and insights (#10785) * fix: filters hash needs to be set on tiles and insights * deal with some unused variables * tighten up assertions --- posthog/tasks/test/test_update_cache.py | 138 ++++++++++++++++-------- posthog/tasks/update_cache.py | 56 +++++++++- 2 files changed, 144 insertions(+), 50 deletions(-) diff --git a/posthog/tasks/test/test_update_cache.py b/posthog/tasks/test/test_update_cache.py index 27b2bfe3007927..cfadab53dc301a 100644 --- a/posthog/tasks/test/test_update_cache.py +++ b/posthog/tasks/test/test_update_cache.py @@ -1,7 +1,6 @@ from copy import copy from datetime import datetime, timedelta -from typing import Any, Dict -from unittest import skip +from typing import Any, Dict, Optional, Tuple from unittest.mock import MagicMock, patch import pytz @@ -206,7 +205,7 @@ def test_refresh_dashboard_cache_types( @freeze_time("2012-01-15") def test_update_cache_item_calls_right_class(self) -> None: filter = Filter(data={"insight": "TRENDS", "events": [{"id": "$pageview"}]}) - dashboard_item = self._create_dashboard(filter) + insight, _ = self._create_dashboard(filter) update_cache_item( generate_cache_key("{}_{}".format(filter.toJSON(), self.team.pk)), @@ -214,7 +213,7 @@ def test_update_cache_item_calls_right_class(self) -> None: {"filter": filter.toJSON(), "team_id": self.team.pk,}, ) - updated_dashboard_item = Insight.objects.get(pk=dashboard_item.pk) + updated_dashboard_item = Insight.objects.get(pk=insight.pk) self.assertEqual(updated_dashboard_item.refreshing, False) self.assertEqual(updated_dashboard_item.last_refresh, now()) @@ -306,14 +305,19 @@ def test_update_cache_item_calls_right_funnel_class_clickhouse( def _test_refresh_dashboard_cache_types( self, filter: FilterType, cache_type: CacheType, patch_update_cache_item: MagicMock, ) -> None: - self._create_dashboard(filter) + insight, dashboard = self._create_dashboard(filter) update_cached_items() expected_args = [ generate_cache_key("{}_{}".format(filter.toJSON(), self.team.pk)), cache_type, - {"filter": filter.toJSON(), "team_id": self.team.pk,}, + { + "filter": filter.toJSON(), + "team_id": self.team.pk, + "insight_id": insight.id, + "dashboard_id": dashboard.id, + }, ] patch_update_cache_item.assert_any_call(*expected_args) @@ -323,14 +327,14 @@ def _test_refresh_dashboard_cache_types( item_key = generate_cache_key("{}_{}".format(filter.toJSON(), self.team.pk)) self.assertIsNotNone(get_safe_cache(item_key)) - def _create_dashboard(self, filter: FilterType, item_refreshing: bool = False) -> Insight: + def _create_dashboard(self, filter: FilterType) -> Tuple[Insight, Dashboard]: dashboard_to_cache = create_shared_dashboard(team=self.team, is_shared=True, last_accessed_at=now()) insight = Insight.objects.create( filters=filter.to_dict(), team=self.team, last_refresh=now() - timedelta(days=30), ) DashboardTile.objects.create(insight=insight, dashboard=dashboard_to_cache) - return insight + return insight, dashboard_to_cache @patch("posthog.tasks.update_cache.group.apply_async") @patch("posthog.celery.update_cache_item_task.s") @@ -498,9 +502,13 @@ def _update_cached_items() -> None: @freeze_time("2021-08-25T22:09:14.252Z") def test_filters_multiple_dashboard(self) -> None: # Regression test. Previously if we had insights with the same filter, but different dashboard filters, we would only update one of those - dashboard1: Dashboard = create_shared_dashboard(filters={"date_from": "-14d"}, team=self.team, is_shared=True) - dashboard2: Dashboard = create_shared_dashboard(filters={"date_from": "-30d"}, team=self.team, is_shared=True) - dashboard3: Dashboard = create_shared_dashboard(team=self.team, is_shared=True) + dashboard_14_days: Dashboard = create_shared_dashboard( + filters={"date_from": "-14d"}, team=self.team, is_shared=True + ) + dashboard_30_days: Dashboard = create_shared_dashboard( + filters={"date_from": "-30d"}, team=self.team, is_shared=True + ) + dashboard_no_filter: Dashboard = create_shared_dashboard(team=self.team, is_shared=True) filter = {"events": [{"id": "$pageview"}]} filters_hash_with_no_dashboard = generate_cache_key( @@ -510,29 +518,29 @@ def test_filters_multiple_dashboard(self) -> None: item1 = Insight.objects.create(filters=filter, team=self.team) self.assertEqual(item1.filters_hash, filters_hash_with_no_dashboard) - DashboardTile.objects.create(insight=item1, dashboard=dashboard1) + DashboardTile.objects.create(insight=item1, dashboard=dashboard_14_days) # link another insight to a dashboard with a filter item2 = Insight.objects.create(filters=filter, team=self.team) - DashboardTile.objects.create(insight=item2, dashboard=dashboard2) - dashboard2.save() + DashboardTile.objects.create(insight=item2, dashboard=dashboard_30_days) + dashboard_30_days.save() # link an insight to a dashboard with no filters item3 = Insight.objects.create(filters=filter, team=self.team) - DashboardTile.objects.create(insight=item3, dashboard=dashboard3) - dashboard3.save() + DashboardTile.objects.create(insight=item3, dashboard=dashboard_no_filter) + dashboard_no_filter.save() update_cached_items() self._assert_number_of_days_in_results( - DashboardTile.objects.get(insight=item1, dashboard=dashboard1), number_of_days_in_results=15 + DashboardTile.objects.get(insight=item1, dashboard=dashboard_14_days), number_of_days_in_results=15 ) self._assert_number_of_days_in_results( - DashboardTile.objects.get(insight=item2, dashboard=dashboard2), number_of_days_in_results=31 + DashboardTile.objects.get(insight=item2, dashboard=dashboard_30_days), number_of_days_in_results=31 ) self._assert_number_of_days_in_results( - DashboardTile.objects.get(insight=item3, dashboard=dashboard3), number_of_days_in_results=8 + DashboardTile.objects.get(insight=item3, dashboard=dashboard_no_filter), number_of_days_in_results=8 ) self.assertEqual( @@ -550,31 +558,6 @@ def _assert_number_of_days_in_results(self, dashboard_tile: DashboardTile, numbe number_of_results = len(cache_result["result"][0]["data"]) self.assertEqual(number_of_results, number_of_days_in_results) - @freeze_time("2021-08-25T22:09:14.252Z") - @skip( - """ - This makes an assumption that the insight's filters_hash can be set without knowledge of the dashboard context - but that is no longer true, - will need a different solution to this - """ - ) - def test_insights_old_filter(self) -> None: - # Some filters hashes are wrong (likely due to changes in our filters models) and previously we would not save changes to those insights and constantly retry them. - dashboard = create_shared_dashboard(team=self.team, is_shared=True) - filter = {"events": [{"id": "$pageview"}]} - item = Insight.objects.create(filters=filter, filters_hash="cache_thisiswrong", team=self.team) - DashboardTile.objects.create(insight=item, dashboard=dashboard) - Insight.objects.all().update(filters_hash="cache_thisiswrong") - self.assertEquals(Insight.objects.get().filters_hash, "cache_thisiswrong") - - update_cached_items() - - self.assertEquals( - Insight.objects.get().filters_hash, - generate_cache_key("{}_{}".format(Filter(data=filter).toJSON(), self.team.pk)), - ) - self.assertEquals(Insight.objects.get().last_refresh.isoformat(), "2021-08-25T22:09:14.252000+00:00") - @freeze_time("2021-08-25T22:09:14.252Z") @patch("posthog.tasks.update_cache.insight_update_task_params") def test_broken_insights(self, dashboard_item_update_task_params: MagicMock) -> None: @@ -653,16 +636,77 @@ def test_refresh_insight_cache(self, patch_update_cache_item: MagicMock, patch_a for insight in other_insights_out_of_range: assert not Insight.objects.get(pk=insight.pk).last_refresh == datetime(2022, 1, 2).replace(tzinfo=pytz.utc) + @freeze_time("2021-08-25T22:09:14.252Z") def test_update_insight_filters_hash(self) -> None: test_hash = "rongi rattad ragisevad" + insight = self._create_insight_with_known_cache_key(test_hash) + + update_insight_cache(insight, None) + + insight.refresh_from_db() + assert insight.filters_hash != test_hash + assert insight.last_refresh.isoformat(), "2021-08-25T22:09:14.252000+00:00" + + @freeze_time("2021-08-25T22:09:14.252Z") + def test_update_dashboard_tile_updates_tile_and_insight_filters_hash_when_dashboard_has_no_filters(self) -> None: + test_hash = "rongi rattad ragisevad" + insight = self._create_insight_with_known_cache_key(test_hash) + dashboard, tile = self._create_dashboard_tile_with_known_cache_key(insight, test_hash) + + update_insight_cache(insight, dashboard) + + insight.refresh_from_db() + tile.refresh_from_db() + assert insight.filters_hash != test_hash + assert insight.last_refresh.isoformat(), "2021-08-25T22:09:14.252000+00:00" + assert tile.filters_hash != test_hash + assert tile.last_refresh.isoformat(), "2021-08-25T22:09:14.252000+00:00" + + @freeze_time("2021-08-25T22:09:14.252Z") + def test_update_dashboard_tile_updates_only_tile_when_different_filters(self) -> None: + test_hash = "rongi rattad ragisevad" + insight = self._create_insight_with_known_cache_key(test_hash) + dashboard, tile = self._create_dashboard_tile_with_known_cache_key( + insight, test_hash, dashboard_filters={"date_from": "-30d"} + ) + + update_insight_cache(insight, dashboard) + + tile.refresh_from_db() + insight.refresh_from_db() + + assert insight.filters_hash == test_hash + assert insight.last_refresh is None + assert tile.filters_hash != test_hash + assert tile.last_refresh.isoformat(), "2021-08-25T22:09:14.252000+00:00" + + def _create_insight_with_known_cache_key(self, test_hash: str) -> Insight: filter_dict: Dict[str, Any] = { "events": [{"id": "$pageview"}], "properties": [{"key": "$browser", "value": "Mac OS X"}], } - insight = Insight.objects.create(team=self.team, filters=filter_dict) + insight: Insight = Insight.objects.create(team=self.team, filters=filter_dict) insight.filters_hash = test_hash insight.save(update_fields=["filters_hash"]) + insight.refresh_from_db() assert insight.filters_hash == test_hash - update_insight_cache(insight, None) - assert insight.filters_hash != test_hash + + return insight + + def _create_dashboard_tile_with_known_cache_key( + self, insight: Insight, test_hash: str, dashboard_filters: Optional[Dict] = None + ) -> Tuple[Dashboard, DashboardTile]: + dashboard: Dashboard = Dashboard.objects.create( + team=self.team, filters=dashboard_filters if dashboard_filters else {} + ) + tile: DashboardTile = DashboardTile.objects.create(insight=insight, dashboard=dashboard) + tile.filters_hash = test_hash + tile.save(update_fields=["filters_hash"]) + + tile.refresh_from_db() + insight.refresh_from_db() + assert tile.filters_hash == test_hash + assert insight.filters_hash == test_hash + + return dashboard, tile diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index 1ca376dc92bf18..4b685dd7306874 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -70,7 +70,15 @@ def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Di elif insight_result is not None: result = insight_result else: - statsd.incr("update_cache_item_no_results", tags={"team": team_id, "cache_key": key}) + statsd.incr( + "update_cache_item_no_results", + tags={ + "team": team_id, + "cache_key": key, + "insight_id": payload.get("insight_id", "unknown"), + "dashboard_id": payload.get("dashboard_id", None), + }, + ) return [] return result @@ -104,10 +112,47 @@ def _update_cache_for_queryset( def update_insight_cache(insight: Insight, dashboard: Optional[Dashboard]) -> List[Dict[str, Any]]: cache_key, cache_type, payload = insight_update_task_params(insight, dashboard) - # cache key changed, usually because of a new default filter + # check if the cache key has changed, usually because of a new default filter + # there are three possibilities + # 1) the insight is not being updated in a dashboard context + # --> so set its cache key if it doesn't match + # 2) the insight is being updated in a dashboard context and the dashboard has different filters to the insight + # --> so set only the dashboard tile's filters_hash + # 3) the insight is being updated in a dashboard context and the dashboard has matching or no filters + # --> so set the dashboard tile and the insight's filters hash + + should_update_insight_filters_hash = False + should_update_dashboard_tile_filters_hash = False + if not dashboard and insight.filters_hash and insight.filters_hash != cache_key: + should_update_insight_filters_hash = True + + if dashboard: + should_update_dashboard_tile_filters_hash = True + if not dashboard.filters or dashboard.filters == insight.filters: + should_update_insight_filters_hash = True + + if should_update_insight_filters_hash: insight.filters_hash = cache_key insight.save() + + if should_update_dashboard_tile_filters_hash: + dashboard_tiles = DashboardTile.objects.filter(insight=insight, dashboard=dashboard,).exclude( + filters_hash=cache_key + ) + dashboard_tiles.update(filters_hash=cache_key) + + if should_update_insight_filters_hash or should_update_dashboard_tile_filters_hash: + statsd.incr( + "update_cache_item_set_new_cache_key", + tags={ + "team": insight.team.id, + "cache_key": cache_key, + "insight_id": insight.id, + "dashboard_id": None if not dashboard else dashboard.id, + }, + ) + result = update_cache_item(cache_key, cache_type, payload) insight.refresh_from_db() return result @@ -196,7 +241,12 @@ def insight_update_task_params(insight: Insight, dashboard: Optional[Dashboard] cache_key = generate_cache_key("{}_{}".format(filter.toJSON(), insight.team_id)) cache_type = get_cache_type(filter) - payload = {"filter": filter.toJSON(), "team_id": insight.team_id} + payload = { + "filter": filter.toJSON(), + "team_id": insight.team_id, + "insight_id": insight.id, + "dashboard_id": None if not dashboard else dashboard.id, + } return cache_key, cache_type, payload From 96f4b9beedb03110540ff2f42bab61f5f3b11945 Mon Sep 17 00:00:00 2001 From: Li Yi Yu Date: Thu, 14 Jul 2022 06:50:47 -0400 Subject: [PATCH 057/213] fix(feature-flags): undo reloads feature flag (#10779) --- frontend/src/scenes/feature-flags/featureFlagLogic.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/scenes/feature-flags/featureFlagLogic.ts b/frontend/src/scenes/feature-flags/featureFlagLogic.ts index 17a77745ee4fd3..0748a02013303a 100644 --- a/frontend/src/scenes/feature-flags/featureFlagLogic.ts +++ b/frontend/src/scenes/feature-flags/featureFlagLogic.ts @@ -298,6 +298,7 @@ export const featureFlagLogic = kea([ object: { name: featureFlag.name, id: featureFlag.id }, callback: () => { featureFlag.id && featureFlagsLogic.findMounted()?.actions.deleteFlag(featureFlag.id) + featureFlagsLogic.findMounted()?.actions.loadFeatureFlags() router.actions.push(urls.featureFlags()) }, }) From addf3fdadbec949735a01d0171af88c1f201c2e4 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 14 Jul 2022 14:40:17 +0100 Subject: [PATCH 058/213] fix: add to dashboard permissions (#10791) * fix: add to dashboard permissions * add a test that someone with permissions can still add to a dashboard * add snapshot back --- posthog/api/insight.py | 13 +++++++ posthog/api/test/test_insight.py | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/posthog/api/insight.py b/posthog/api/insight.py index aa8493b71916e7..5c896366f81b0a 100644 --- a/posthog/api/insight.py +++ b/posthog/api/insight.py @@ -12,6 +12,7 @@ from drf_spectacular.utils import OpenApiResponse from rest_framework import exceptions, request, serializers, status, viewsets from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.settings import api_settings @@ -274,6 +275,18 @@ def update(self, instance: Insight, validated_data: Dict, **kwargs) -> Insight: ids_to_add = [id for id in new_dashboard_ids if id not in old_dashboard_ids] ids_to_remove = [id for id in old_dashboard_ids if id not in new_dashboard_ids] + # does this user have permission on dashboards to add... if they are restricted + # it will mean this dashboard becomes restricted because of the patch + dashboard: Dashboard + for dashboard in Dashboard.objects.filter(id__in=ids_to_add): + if ( + dashboard.get_effective_privilege_level(self.context["request"].user) + == Dashboard.PrivilegeLevel.CAN_VIEW + ): + raise PermissionDenied( + f"You don't have permission to add insights to dashboard: {dashboard.id}" + ) + for dashboard in Dashboard.objects.filter(id__in=ids_to_add): if dashboard.team != instance.team: raise serializers.ValidationError("Dashboard not found") diff --git a/posthog/api/test/test_insight.py b/posthog/api/test/test_insight.py index 731a158b7814c5..7346032d699743 100644 --- a/posthog/api/test/test_insight.py +++ b/posthog/api/test/test_insight.py @@ -1481,6 +1481,72 @@ def test_cannot_update_an_insight_if_on_restricted_dashboard(self): ) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + def test_non_admin_user_cannot_add_an_insight_to_a_restricted_dashboard(self): + # create insight and dashboard separately with default user + dashboard_restricted: Dashboard = Dashboard.objects.create( + team=self.team, restriction_level=Dashboard.RestrictionLevel.ONLY_COLLABORATORS_CAN_EDIT + ) + + insight_id, response_data = self._create_insight(data={"name": "starts un-restricted dashboard"}) + + # user with no permissions on the dashboard cannot add insight to it + user_without_permissions = User.objects.create_and_join( + organization=self.organization, email="no_access_user@posthog.com", password=None, + ) + self.client.force_login(user_without_permissions) + + response = self.client.patch( + f"/api/projects/{self.team.id}/insights/{insight_id}", {"dashboards": [dashboard_restricted.id]}, + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_non_admin_user_with_privilege_can_add_an_insight_to_a_restricted_dashboard(self): + # create insight and dashboard separately with default user + dashboard_restricted: Dashboard = Dashboard.objects.create( + team=self.team, restriction_level=Dashboard.RestrictionLevel.ONLY_COLLABORATORS_CAN_EDIT + ) + + insight_id, response_data = self._create_insight(data={"name": "starts un-restricted dashboard"}) + + user_with_permissions = User.objects.create_and_join( + organization=self.organization, email="with_access_user@posthog.com", password=None, + ) + + DashboardPrivilege.objects.create( + dashboard=dashboard_restricted, + user=user_with_permissions, + level=Dashboard.RestrictionLevel.ONLY_COLLABORATORS_CAN_EDIT, + ) + + self.client.force_login(user_with_permissions) + + response = self.client.patch( + f"/api/projects/{self.team.id}/insights/{insight_id}", {"dashboards": [dashboard_restricted.id]}, + ) + assert response.status_code == status.HTTP_200_OK + + def test_admin_user_can_add_an_insight_to_a_restricted_dashboard(self): + # create insight and dashboard separately with default user + dashboard_restricted: Dashboard = Dashboard.objects.create( + team=self.team, restriction_level=Dashboard.RestrictionLevel.ONLY_COLLABORATORS_CAN_EDIT + ) + + insight_id, response_data = self._create_insight(data={"name": "starts un-restricted dashboard"}) + + # an admin user has implicit permissions on the dashboard and can add the insight to it + admin = User.objects.create_and_join( + organization=self.organization, + email="team2@posthog.com", + password=None, + level=OrganizationMembership.Level.ADMIN, + ) + self.client.force_login(admin) + + response = self.client.patch( + f"/api/projects/{self.team.id}/insights/{insight_id}", {"dashboards": [dashboard_restricted.id]}, + ) + assert response.status_code == status.HTTP_200_OK + def test_saving_an_insight_with_new_filters_updates_the_dashboard_tile(self): dashboard_id, _ = self._create_dashboard({}) insight_id, _ = self._create_insight( From be7e634ae84c4c9a19384c6d55927298c2d8f9cd Mon Sep 17 00:00:00 2001 From: Ben White Date: Thu, 14 Jul 2022 15:44:02 +0200 Subject: [PATCH 059/213] feat: Extended MultiSelect to always have a searchable label (#10787) --- .../LemonSelectMultiple/LemonSelectMultiple.stories.tsx | 7 ++++--- .../components/LemonSelectMultiple/LemonSelectMultiple.tsx | 7 +++++-- frontend/src/lib/components/Subscriptions/utils.tsx | 5 +++-- .../components/Subscriptions/views/EditSubscription.tsx | 2 -- frontend/src/lib/components/UserSelectItem.tsx | 3 ++- frontend/src/scenes/persons/MergeSplitPerson.tsx | 2 +- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.stories.tsx b/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.stories.tsx index 51b0ed3132d672..4502210dbe4d85 100644 --- a/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.stories.tsx +++ b/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.stories.tsx @@ -10,10 +10,10 @@ export default { argTypes: { options: { defaultValue: ['ben', 'marius', 'paul', 'tiina', 'li'].reduce( - (acc, x) => ({ + (acc, x, i) => ({ ...acc, - [`${x}@posthog.com`]: { - label: ( + [`user-${i}`]: { + labelComponent: ( @@ -21,6 +21,7 @@ export default { ), + label: `${x} ${x}@posthog.com>`, }, }), {} diff --git a/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.tsx b/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.tsx index 6c85e7f8d27c60..3f8846bc28cbc4 100644 --- a/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.tsx +++ b/frontend/src/lib/components/LemonSelectMultiple/LemonSelectMultiple.tsx @@ -5,9 +5,10 @@ import { LemonSnack } from '../LemonSnack/LemonSnack' import './LemonSelectMultiple.scss' export interface LemonSelectMultipleOption { - label: string | React.ReactNode + label: string disabled?: boolean 'data-attr'?: string + labelComponent?: React.ReactNode } export interface LemonSelectMultipleOptionItem extends LemonSelectMultipleOption { @@ -51,7 +52,8 @@ export function LemonSelectMultiple({ const antOptions = optionsAsList.map((option) => ({ key: option.key, value: option.key, - label: option.label, + label: option.labelComponent || option.label, + labelString: option.label, })) return ( @@ -66,6 +68,7 @@ export function LemonSelectMultiple({ tokenSeparators={[',']} value={value ? value : []} dropdownRender={(menu) =>
{menu}
} + optionFilterProp="labelString" options={antOptions} placeholder={placeholder} notFoundContent={ diff --git a/frontend/src/lib/components/Subscriptions/utils.tsx b/frontend/src/lib/components/Subscriptions/utils.tsx index cb705c8effdd22..2c87b29f9e79e2 100644 --- a/frontend/src/lib/components/Subscriptions/utils.tsx +++ b/frontend/src/lib/components/Subscriptions/utils.tsx @@ -96,18 +96,19 @@ export const getSlackChannelOptions = ( return slackChannels ? slackChannels.map((x) => ({ key: `${x.id}|#${x.name}`, - label: ( + labelComponent: ( {x.is_private ? `🔒${x.name}` : `#${x.name}`} {x.is_ext_shared ? : null} ), + label: `${x.id} #${x.name}`, })) : value ? [ { key: value, - label: value?.split('|')?.pop(), + label: value?.split('|')?.pop() || value, }, ] : [] diff --git a/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx b/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx index 1bd4f058552a58..50b9a3c4de2f90 100644 --- a/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx +++ b/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx @@ -199,7 +199,6 @@ export function EditSubscription({ onChange(val.join(','))} value={value?.split(',').filter(Boolean)} - filterOption={false} disabled={emailDisabled} mode="multiple-custom" data-attr="subscribed-emails" @@ -270,7 +269,6 @@ export function EditSubscription({ onChange(val)} value={value} - filterOption={true} disabled={slackDisabled} mode="single" data-attr="select-slack-channel" diff --git a/frontend/src/lib/components/UserSelectItem.tsx b/frontend/src/lib/components/UserSelectItem.tsx index f6b5f222368209..b7260b7bc6f7d3 100644 --- a/frontend/src/lib/components/UserSelectItem.tsx +++ b/frontend/src/lib/components/UserSelectItem.tsx @@ -24,6 +24,7 @@ export function usersLemonSelectOptions( ): LemonSelectMultipleOptionItem[] { return users.map((user) => ({ key: user[key], - label: , + label: `${user.first_name} ${user.email}`, + labelComponent: , })) } diff --git a/frontend/src/scenes/persons/MergeSplitPerson.tsx b/frontend/src/scenes/persons/MergeSplitPerson.tsx index 0673985eab3b3a..a48e5ab748db2d 100644 --- a/frontend/src/scenes/persons/MergeSplitPerson.tsx +++ b/frontend/src/scenes/persons/MergeSplitPerson.tsx @@ -85,7 +85,7 @@ function MergePerson(): JSX.Element { .filter((p: PersonType) => p.id && p.uuid !== person.uuid) .map((p) => ({ key: `${p.id}`, - label: p.name, + label: `${p.name || p.id}`, }))} disabled={executedLoading} /> From d95b4f13232c25e15edd1690398c834176b19d8f Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 14 Jul 2022 15:31:34 +0100 Subject: [PATCH 060/213] chore: add timers to slow cache update method (#10794) --- posthog/tasks/update_cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index 4b685dd7306874..e1403792bb1c9c 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -251,6 +251,7 @@ def insight_update_task_params(insight: Insight, dashboard: Optional[Dashboard] return cache_key, cache_type, payload +@timed("update_cache_item_timer.calculate_by_filter") def _calculate_by_filter(filter: FilterType, key: str, team: Team, cache_type: CacheType) -> List[Dict[str, Any]]: insight_class = CACHE_TYPE_TO_INSIGHT_CLASS[cache_type] @@ -261,6 +262,7 @@ def _calculate_by_filter(filter: FilterType, key: str, team: Team, cache_type: C return result +@timed("update_cache_item_timer.calculate_funnel") def _calculate_funnel(filter: Filter, key: str, team: Team) -> List[Dict[str, Any]]: if filter.funnel_viz_type == FunnelVizType.TRENDS: result = ClickhouseFunnelTrends(team=team, filter=filter).run() From 12e33dab5eeec103bc3c1f7e86d2ce2b174bfbeb Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 14 Jul 2022 15:33:35 +0100 Subject: [PATCH 061/213] chore: gauge entire insight cache queue as well as dashboards and insights separately (#10795) --- posthog/tasks/update_cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index e1403792bb1c9c..c1ae9208642139 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -231,6 +231,8 @@ def update_cached_items() -> Tuple[int, int]: taskset = group(tasks) taskset.apply_async() queue_depth = dashboard_tiles.count() + shared_insights.count() + statsd.gauge("update_cache_queue_depth.shared_insights", shared_insights.count()) + statsd.gauge("update_cache_queue_depth.dashboards", dashboard_tiles.count()) statsd.gauge("update_cache_queue_depth", queue_depth) return len(tasks), queue_depth From aa109da1e64a574d9c56e8e4c98553ca8321cce8 Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Thu, 14 Jul 2022 17:44:26 +0000 Subject: [PATCH 062/213] feat: make export events batches larger (#10800) * feat: make export events batches larger * lighter numbers --- plugin-server/src/worker/vm/upgrades/export-events.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin-server/src/worker/vm/upgrades/export-events.ts b/plugin-server/src/worker/vm/upgrades/export-events.ts index 59231004cad075..416c28598fb5ae 100644 --- a/plugin-server/src/worker/vm/upgrades/export-events.ts +++ b/plugin-server/src/worker/vm/upgrades/export-events.ts @@ -8,11 +8,11 @@ import { ExportEventsBuffer } from './utils/export-events-buffer' export const MAXIMUM_RETRIES = 3 const EXPORT_BUFFER_BYTES_MINIMUM = 1 -const EXPORT_BUFFER_BYTES_DEFAULT = 1024 * 1024 +const EXPORT_BUFFER_BYTES_DEFAULT = 10 * 1024 * 1024 const EXPORT_BUFFER_BYTES_MAXIMUM = 100 * 1024 * 1024 const EXPORT_BUFFER_SECONDS_MINIMUM = 1 const EXPORT_BUFFER_SECONDS_MAXIMUM = 600 -const EXPORT_BUFFER_SECONDS_DEFAULT = isTestEnv() ? EXPORT_BUFFER_SECONDS_MAXIMUM : 10 +const EXPORT_BUFFER_SECONDS_DEFAULT = isTestEnv() ? EXPORT_BUFFER_SECONDS_MAXIMUM : 30 type ExportEventsUpgrade = Plugin<{ global: { From ca8c4d0271cbc9ee294fca24d58305f12e3c635b Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Thu, 14 Jul 2022 18:24:58 +0000 Subject: [PATCH 063/213] Revert "feat: buffer 3.0 (graphile) (#10735)" (#10802) This reverts commit 9a2a9046cba852ecca17dd254c96cbfd5da2fa43. --- plugin-server/src/capabilities.ts | 9 +-- .../src/main/ingestion-queues/buffer.ts | 75 +++++++++++++++++++ plugin-server/src/main/job-queues/buffer.ts | 9 --- .../src/main/job-queues/job-queue-consumer.ts | 43 +++-------- .../src/main/job-queues/local/fs-queue.ts | 8 +- plugin-server/src/main/pluginsServer.ts | 17 ++++- plugin-server/src/types.ts | 11 +-- plugin-server/src/utils/db/db.ts | 9 +++ .../event-pipeline/1-emitToBufferStep.ts | 11 +-- plugin-server/src/worker/tasks.ts | 7 +- plugin-server/src/worker/vm/capabilities.ts | 2 +- plugin-server/src/worker/worker.ts | 2 +- plugin-server/tests/jobs.test.ts | 1 + plugin-server/tests/main/capabilities.test.ts | 66 ++++------------ plugin-server/tests/main/db.test.ts | 15 ++++ .../main/ingestion-queues/buffer.test.ts | 50 +++++++++++++ plugin-server/tests/server.test.ts | 15 +--- .../tests/worker/capabilities.test.ts | 15 ++-- .../event-pipeline/emitToBufferStep.test.ts | 21 ++---- plugin-server/tests/worker/vm.lazy.test.ts | 7 +- 20 files changed, 221 insertions(+), 172 deletions(-) create mode 100644 plugin-server/src/main/ingestion-queues/buffer.ts delete mode 100644 plugin-server/src/main/job-queues/buffer.ts create mode 100644 plugin-server/tests/main/ingestion-queues/buffer.test.ts diff --git a/plugin-server/src/capabilities.ts b/plugin-server/src/capabilities.ts index 91ac6aa87027bb..d88b5d14d7e26d 100644 --- a/plugin-server/src/capabilities.ts +++ b/plugin-server/src/capabilities.ts @@ -10,18 +10,13 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin return { ingestion: true, pluginScheduledTasks: true, - processPluginJobs: true, + processJobs: true, processAsyncHandlers: true, ...sharedCapabilities, } case 'ingestion': return { ingestion: true, ...sharedCapabilities } case 'async': - return { - pluginScheduledTasks: true, - processPluginJobs: true, - processAsyncHandlers: true, - ...sharedCapabilities, - } + return { pluginScheduledTasks: true, processJobs: true, processAsyncHandlers: true, ...sharedCapabilities } } } diff --git a/plugin-server/src/main/ingestion-queues/buffer.ts b/plugin-server/src/main/ingestion-queues/buffer.ts new file mode 100644 index 00000000000000..3f0e76f3a26b5b --- /dev/null +++ b/plugin-server/src/main/ingestion-queues/buffer.ts @@ -0,0 +1,75 @@ +import Piscina from '@posthog/piscina' +import { PluginEvent } from '@posthog/plugin-scaffold' + +import { Hub } from '../../types' +import { runInstrumentedFunction } from '../utils' + +export function runBufferEventPipeline(hub: Hub, piscina: Piscina, event: PluginEvent) { + hub.lastActivity = new Date().valueOf() + hub.lastActivityType = 'runBufferEventPipeline' + return piscina.run({ task: 'runBufferEventPipeline', args: { event } }) +} + +export async function runBuffer(hub: Hub, piscina: Piscina): Promise { + let eventRows: { id: number; event: PluginEvent }[] = [] + await hub.db.postgresTransaction(async (client) => { + const eventsResult = await client.query(` + UPDATE posthog_eventbuffer SET locked=true WHERE id IN ( + SELECT id FROM posthog_eventbuffer + WHERE process_at <= now() AND process_at > (now() - INTERVAL '30 minute') AND locked=false + ORDER BY id + LIMIT 10 + FOR UPDATE SKIP LOCKED + ) + RETURNING id, event + `) + eventRows = eventsResult.rows + }) + + const idsToDelete: number[] = [] + + // We don't indiscriminately delete all IDs to prevent the case when we don't trigger `runInstrumentedFunction` + // Once that runs, events will either go to the events table or the dead letter queue + const processBufferEvent = async (event: PluginEvent, id: number) => { + await runInstrumentedFunction({ + server: hub, + event: event, + func: () => runBufferEventPipeline(hub, piscina, event), + statsKey: `kafka_queue.ingest_buffer_event`, + timeoutMessage: 'After 30 seconds still running runBufferEventPipeline', + }) + idsToDelete.push(id) + } + + await Promise.all(eventRows.map((eventRow) => processBufferEvent(eventRow.event, eventRow.id))) + + if (idsToDelete.length > 0) { + await hub.db.postgresQuery( + `DELETE FROM posthog_eventbuffer WHERE id IN (${idsToDelete.join(',')})`, + [], + 'completeBufferEvent' + ) + hub.statsd?.increment('events_deleted_from_buffer', idsToDelete.length) + } +} + +export async function clearBufferLocks(hub: Hub): Promise { + /* + * If we crash during runBuffer we may end up with 2 scenarios: + * 1. "locked" rows with events that were never processed (crashed after fetching and before running the pipeline) + * 2. "locked" rows with events that were processed (crashed after the pipeline and before deletion) + * This clears any old locks such that the events are processed again. If there are any duplicates ClickHouse should collapse them. + */ + const recordsUpdated = await hub.db.postgresQuery( + `UPDATE posthog_eventbuffer + SET locked=false, process_at=now() + WHERE locked=true AND process_at < (now() - INTERVAL '30 minute') + RETURNING 1`, + [], + 'clearBufferLocks' + ) + + if (recordsUpdated.rowCount > 0 && hub.statsd) { + hub.statsd.increment('buffer_locks_cleared', recordsUpdated.rowCount) + } +} diff --git a/plugin-server/src/main/job-queues/buffer.ts b/plugin-server/src/main/job-queues/buffer.ts deleted file mode 100644 index 1c52b5eefe6e88..00000000000000 --- a/plugin-server/src/main/job-queues/buffer.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Piscina from '@posthog/piscina' -import { PluginEvent } from '@posthog/plugin-scaffold' -import { Hub } from 'types' - -export function runBufferEventPipeline(hub: Hub, piscina: Piscina, event: PluginEvent): Promise { - hub.lastActivity = new Date().valueOf() - hub.lastActivityType = 'runBufferEventPipeline' - return piscina.run({ task: 'runBufferEventPipeline', args: { event } }) -} diff --git a/plugin-server/src/main/job-queues/job-queue-consumer.ts b/plugin-server/src/main/job-queues/job-queue-consumer.ts index f18fc1ea02fc90..a41911766480b0 100644 --- a/plugin-server/src/main/job-queues/job-queue-consumer.ts +++ b/plugin-server/src/main/job-queues/job-queue-consumer.ts @@ -1,52 +1,31 @@ import Piscina from '@posthog/piscina' import { TaskList } from 'graphile-worker' -import { EnqueuedBufferJob, EnqueuedPluginJob, Hub, JobQueueConsumerControl } from '../../types' +import { EnqueuedJob, Hub, JobQueueConsumerControl } from '../../types' import { killProcess } from '../../utils/kill' import { status } from '../../utils/status' import { logOrThrowJobQueueError } from '../../utils/utils' import { pauseQueueIfWorkerFull } from '../ingestion-queues/queue' -import { runInstrumentedFunction } from '../utils' -import { runBufferEventPipeline } from './buffer' -export async function startJobQueueConsumer(hub: Hub, piscina: Piscina): Promise { +export async function startJobQueueConsumer(server: Hub, piscina: Piscina): Promise { status.info('🔄', 'Starting job queue consumer, trying to get lock...') - const ingestionJobHandlers: TaskList = { - bufferJob: async (job) => { - const eventPayload = (job as EnqueuedBufferJob).eventPayload - await runInstrumentedFunction({ - server: hub, - event: eventPayload, - func: () => runBufferEventPipeline(hub, piscina, eventPayload), - statsKey: `kafka_queue.ingest_buffer_event`, - timeoutMessage: 'After 30 seconds still running runBufferEventPipeline', - }) - hub.statsd?.increment('events_deleted_from_buffer') - }, - } - - const pluginJobHandlers: TaskList = { + const jobHandlers: TaskList = { pluginJob: async (job) => { - pauseQueueIfWorkerFull(() => hub.jobQueueManager.pauseConsumer(), hub, piscina) - hub.statsd?.increment('triggered_job', { - instanceId: hub.instanceId.toString(), + pauseQueueIfWorkerFull(() => server.jobQueueManager.pauseConsumer(), server, piscina) + server.statsd?.increment('triggered_job', { + instanceId: server.instanceId.toString(), }) - await piscina.run({ task: 'runPluginJob', args: { job: job as EnqueuedPluginJob } }) + await piscina.run({ task: 'runJob', args: { job: job as EnqueuedJob } }) }, } - const jobHandlers: TaskList = { - ...(hub.capabilities.ingestion ? ingestionJobHandlers : {}), - ...(hub.capabilities.processPluginJobs ? pluginJobHandlers : {}), - } - status.info('🔄', 'Job queue consumer starting') try { - await hub.jobQueueManager.startConsumer(jobHandlers) + await server.jobQueueManager.startConsumer(jobHandlers) } catch (error) { try { - logOrThrowJobQueueError(hub, error, `Cannot start job queue consumer!`) + logOrThrowJobQueueError(server, error, `Cannot start job queue consumer!`) } catch { killProcess() } @@ -54,8 +33,8 @@ export async function startJobQueueConsumer(hub: Hub, piscina: Piscina): Promise const stop = async () => { status.info('🔄', 'Stopping job queue consumer') - await hub.jobQueueManager.stopConsumer() + await server.jobQueueManager.stopConsumer() } - return { stop, resume: () => hub.jobQueueManager.resumeConsumer() } + return { stop, resume: () => server.jobQueueManager.resumeConsumer() } } diff --git a/plugin-server/src/main/job-queues/local/fs-queue.ts b/plugin-server/src/main/job-queues/local/fs-queue.ts index 45d561f56dc65c..cd5826f264497a 100644 --- a/plugin-server/src/main/job-queues/local/fs-queue.ts +++ b/plugin-server/src/main/job-queues/local/fs-queue.ts @@ -7,14 +7,8 @@ import * as path from 'path' import { JobQueueBase } from '../job-queue-base' -interface FsJob { +interface FsJob extends EnqueuedJob { jobName: string - timestamp: number - type?: string - payload?: Record - eventPayload?: Record - pluginConfigId?: number - pluginConfigTeam?: number } export class FsQueue extends JobQueueBase { paused: boolean diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index 250f2ed4a408e4..b07a2fb9face44 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -23,6 +23,7 @@ import { cancelAllScheduledJobs } from '../utils/node-schedule' import { PubSub } from '../utils/pubsub' import { status } from '../utils/status' import { delay, getPiscinaStats, stalenessCheck } from '../utils/utils' +import { clearBufferLocks, runBuffer } from './ingestion-queues/buffer' import { KafkaQueue } from './ingestion-queues/kafka-queue' import { startQueues } from './ingestion-queues/queue' import { startJobQueueConsumer } from './job-queues/job-queue-consumer' @@ -170,7 +171,7 @@ export async function startPluginsServer( if (hub.capabilities.pluginScheduledTasks) { pluginScheduleControl = await startPluginSchedules(hub, piscina) } - if (hub.capabilities.ingestion || hub.capabilities.processPluginJobs) { + if (hub.capabilities.processJobs) { jobQueueConsumer = await startJobQueueConsumer(hub, piscina) } @@ -234,6 +235,20 @@ export async function startPluginsServer( } }) + if (hub.capabilities.ingestion) { + // every 5 seconds process buffer events + schedule.scheduleJob('*/5 * * * * *', async () => { + if (piscina) { + await runBuffer(hub!, piscina) + } + }) + } + + // every 30 minutes clear any locks that may have lingered on the buffer table + schedule.scheduleJob('*/30 * * * *', async () => { + await clearBufferLocks(hub!) + }) + // every minute log information on kafka consumer if (queue) { schedule.scheduleJob('0 * * * * *', async () => { diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index c39b4997fb7155..20c0514ca76829 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -201,13 +201,12 @@ export interface Hub extends PluginsServerConfig { export interface PluginServerCapabilities { ingestion?: boolean pluginScheduledTasks?: boolean - processPluginJobs?: boolean + processJobs?: boolean processAsyncHandlers?: boolean http?: boolean } -export type EnqueuedJob = EnqueuedPluginJob | EnqueuedBufferJob -export interface EnqueuedPluginJob { +export interface EnqueuedJob { type: string payload: Record timestamp: number @@ -215,14 +214,8 @@ export interface EnqueuedPluginJob { pluginConfigTeam: number } -export interface EnqueuedBufferJob { - eventPayload: PluginEvent - timestamp: number -} - export enum JobName { PLUGIN_JOB = 'pluginJob', - BUFFER_JOB = 'bufferJob', } export interface JobQueue { diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index 300ce2a5d9c816..ae8b57ec1458ac 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -2041,4 +2041,13 @@ export class DB { ) return response.rowCount > 0 } + + public async addEventToBuffer(event: Record, processAt: DateTime): Promise { + await this.postgresQuery( + `INSERT INTO posthog_eventbuffer (event, process_at, locked) VALUES ($1, $2, $3)`, + [event, processAt.toISO(), false], + 'addEventToBuffer' + ) + this.statsd?.increment('events_sent_to_buffer') + } } diff --git a/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts b/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts index 9c2f926701cb99..94eb11c075dfe3 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/1-emitToBufferStep.ts @@ -1,6 +1,7 @@ import { PluginEvent } from '@posthog/plugin-scaffold' +import { DateTime } from 'luxon' -import { Hub, IngestionPersonData, JobName, TeamId } from '../../../types' +import { Hub, IngestionPersonData, TeamId } from '../../../types' import { EventPipelineRunner, StepResult } from './runner' export async function emitToBufferStep( @@ -20,12 +21,8 @@ export async function emitToBufferStep( const person = await runner.hub.db.fetchPerson(event.team_id, event.distinct_id) if (shouldBuffer(runner.hub, event, person, event.team_id)) { - const processEventAt = Date.now() + runner.hub.BUFFER_CONVERSION_SECONDS * 1000 - await runner.hub.jobQueueManager.enqueue(JobName.BUFFER_JOB, { - eventPayload: event, - timestamp: processEventAt, - }) - runner.hub.statsd?.increment('events_sent_to_buffer') + const processEventAt = DateTime.now().plus({ seconds: runner.hub.BUFFER_CONVERSION_SECONDS }) + await runner.hub.db.addEventToBuffer(event, processEventAt) return null } else { return runner.nextStep('pluginsProcessEventStep', event, person) diff --git a/plugin-server/src/worker/tasks.ts b/plugin-server/src/worker/tasks.ts index 53c34a2bbb46cd..0e94a0971ac8dd 100644 --- a/plugin-server/src/worker/tasks.ts +++ b/plugin-server/src/worker/tasks.ts @@ -1,6 +1,6 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' -import { Action, EnqueuedPluginJob, Hub, IngestionEvent, PluginTaskType, Team } from '../types' +import { Action, EnqueuedJob, Hub, IngestionEvent, JobName, PluginTaskType, Team } from '../types' import { convertToProcessedPluginEvent } from '../utils/event' import { EventPipelineRunner } from './ingestion/event-pipeline/runner' import { runPluginTask, runProcessEvent } from './plugins/run' @@ -10,7 +10,7 @@ import { teardownPlugins } from './plugins/teardown' type TaskRunner = (hub: Hub, args: any) => Promise | any export const workerTasks: Record = { - runPluginJob: (hub, { job }: { job: EnqueuedPluginJob }) => { + runJob: (hub, { job }: { job: EnqueuedJob }) => { return runPluginTask(hub, job.type, PluginTaskType.Job, job.pluginConfigId, job.payload) }, runEveryMinute: (hub, args: { pluginConfigId: number }) => { @@ -61,6 +61,9 @@ export const workerTasks: Record = { flushKafkaMessages: async (hub) => { await hub.kafkaProducer.flush() }, + enqueueJob: async (hub, { job }: { job: EnqueuedJob }) => { + await hub.jobQueueManager.enqueue(JobName.PLUGIN_JOB, job) + }, // Exported only for tests _testsRunProcessEvent: async (hub, args: { event: PluginEvent }) => { return runProcessEvent(hub, args.event) diff --git a/plugin-server/src/worker/vm/capabilities.ts b/plugin-server/src/worker/vm/capabilities.ts index b24716e3ebbed3..9b52a2d4296a5c 100644 --- a/plugin-server/src/worker/vm/capabilities.ts +++ b/plugin-server/src/worker/vm/capabilities.ts @@ -41,7 +41,7 @@ function shouldSetupPlugin(serverCapability: keyof PluginServerCapabilities, plu if (serverCapability === 'pluginScheduledTasks') { return (pluginCapabilities.scheduled_tasks || []).length > 0 } - if (serverCapability === 'processPluginJobs') { + if (serverCapability === 'processJobs') { return (pluginCapabilities.jobs || []).length > 0 } if (serverCapability === 'processAsyncHandlers') { diff --git a/plugin-server/src/worker/worker.ts b/plugin-server/src/worker/worker.ts index 2e24c9c96033a0..b933b2dc64872e 100644 --- a/plugin-server/src/worker/worker.ts +++ b/plugin-server/src/worker/worker.ts @@ -50,7 +50,7 @@ export const createTaskRunner = } hub.statsd?.timing(`piscina_task.${task}`, timer) - if (task === 'runPluginJob') { + if (task === 'runJob') { hub.statsd?.timing('plugin_job', timer, { type: String(args.job?.type), pluginConfigId: String(args.job?.pluginConfigId), diff --git a/plugin-server/tests/jobs.test.ts b/plugin-server/tests/jobs.test.ts index 55cb02dd5994e7..d089958f981752 100644 --- a/plugin-server/tests/jobs.test.ts +++ b/plugin-server/tests/jobs.test.ts @@ -150,6 +150,7 @@ describe.skip('job queues', () => { const now = Date.now() const job: EnqueuedJob = { + type: 'pluginJob', payload: { key: 'value' }, timestamp: now + DELAY, pluginConfigId: 2, diff --git a/plugin-server/tests/main/capabilities.test.ts b/plugin-server/tests/main/capabilities.test.ts index 52bff7abeb7696..2548d30ff62dc1 100644 --- a/plugin-server/tests/main/capabilities.test.ts +++ b/plugin-server/tests/main/capabilities.test.ts @@ -2,29 +2,28 @@ import Piscina from '@posthog/piscina' import { KafkaQueue } from '../../src/main/ingestion-queues/kafka-queue' import { startQueues } from '../../src/main/ingestion-queues/queue' -import { startJobQueueConsumer } from '../../src/main/job-queues/job-queue-consumer' import { Hub, LogLevel } from '../../src/types' import { createHub } from '../../src/utils/db/hub' jest.mock('../../src/main/ingestion-queues/kafka-queue') -describe('capabilities', () => { - let hub: Hub - let piscina: Piscina - let closeHub: () => Promise +describe('queue', () => { + describe('capabilities', () => { + let hub: Hub + let piscina: Piscina + let closeHub: () => Promise - beforeEach(async () => { - ;[hub, closeHub] = await createHub({ - LOG_LEVEL: LogLevel.Warn, + beforeEach(async () => { + ;[hub, closeHub] = await createHub({ + LOG_LEVEL: LogLevel.Warn, + }) + piscina = { run: jest.fn() } as any }) - piscina = { run: jest.fn() } as any - }) - afterEach(async () => { - await closeHub() - }) + afterEach(async () => { + await closeHub() + }) - describe('queue', () => { it('starts ingestion queue by default', async () => { const queues = await startQueues(hub, piscina) @@ -44,43 +43,4 @@ describe('capabilities', () => { }) }) }) - - describe('startJobQueueConsumer()', () => { - it('sets up bufferJob handler if ingestion is on', async () => { - hub.jobQueueManager.startConsumer = jest.fn() - hub.capabilities.ingestion = true - hub.capabilities.processPluginJobs = false - - await startJobQueueConsumer(hub, piscina) - - expect(hub.jobQueueManager.startConsumer).toHaveBeenCalledWith({ - bufferJob: expect.anything(), - }) - }) - - it('sets up pluginJob handler if processPluginJobs is on', async () => { - hub.jobQueueManager.startConsumer = jest.fn() - hub.capabilities.ingestion = false - hub.capabilities.processPluginJobs = true - - await startJobQueueConsumer(hub, piscina) - - expect(hub.jobQueueManager.startConsumer).toHaveBeenCalledWith({ - pluginJob: expect.anything(), - }) - }) - - it('sets up bufferJob and pluginJob handlers if ingestion and processPluginJobs are on', async () => { - hub.jobQueueManager.startConsumer = jest.fn() - hub.capabilities.ingestion = true - hub.capabilities.processPluginJobs = true - - await startJobQueueConsumer(hub, piscina) - - expect(hub.jobQueueManager.startConsumer).toHaveBeenCalledWith({ - bufferJob: expect.anything(), - pluginJob: expect.anything(), - }) - }) - }) }) diff --git a/plugin-server/tests/main/db.test.ts b/plugin-server/tests/main/db.test.ts index 6093aedf3c0428..a6a7e65523ce5c 100644 --- a/plugin-server/tests/main/db.test.ts +++ b/plugin-server/tests/main/db.test.ts @@ -1026,4 +1026,19 @@ describe('DB', () => { ) }) }) + + describe('addEventToBuffer', () => { + test('inserts event correctly', async () => { + const processAt = DateTime.now() + await db.addEventToBuffer({ foo: 'bar' }, processAt) + + const bufferResult = await db.postgresQuery( + 'SELECT event, process_at FROM posthog_eventbuffer', + [], + 'addEventToBufferTest' + ) + expect(bufferResult.rows[0].event).toEqual({ foo: 'bar' }) + expect(processAt).toEqual(DateTime.fromISO(bufferResult.rows[0].process_at)) + }) + }) }) diff --git a/plugin-server/tests/main/ingestion-queues/buffer.test.ts b/plugin-server/tests/main/ingestion-queues/buffer.test.ts new file mode 100644 index 00000000000000..9751bbc9a0318f --- /dev/null +++ b/plugin-server/tests/main/ingestion-queues/buffer.test.ts @@ -0,0 +1,50 @@ +import Piscina from '@posthog/piscina' +import { DateTime } from 'luxon' + +import { runBuffer } from '../../../src/main/ingestion-queues/buffer' +import { runInstrumentedFunction } from '../../../src/main/utils' +import { Hub } from '../../../src/types' +import { DB } from '../../../src/utils/db/db' +import { createHub } from '../../../src/utils/db/hub' +import { resetTestDatabase } from '../../helpers/sql' + +// jest.mock('../../../src/utils') +jest.mock('../../../src/main/utils') + +describe('Event buffer', () => { + let hub: Hub + let closeServer: () => Promise + let db: DB + + beforeEach(async () => { + ;[hub, closeServer] = await createHub() + await resetTestDatabase(undefined, {}, {}, { withExtendedTestData: false }) + db = hub.db + }) + + afterEach(async () => { + await closeServer() + jest.clearAllMocks() + }) + + describe('runBuffer', () => { + test('processes events from buffer and deletes them', async () => { + const processAt = DateTime.now() + await db.addEventToBuffer({ foo: 'bar' }, processAt) + await db.addEventToBuffer({ foo: 'bar' }, processAt) + + await runBuffer(hub, {} as Piscina) + + expect(runInstrumentedFunction).toHaveBeenCalledTimes(2) + expect(runInstrumentedFunction).toHaveBeenLastCalledWith(expect.objectContaining({ event: { foo: 'bar' } })) + + const countResult = await db.postgresQuery( + 'SELECT count(*) FROM posthog_eventbuffer', + [], + 'eventBufferCountTest' + ) + + expect(Number(countResult.rows[0].count)).toEqual(0) + }) + }) +}) diff --git a/plugin-server/tests/server.test.ts b/plugin-server/tests/server.test.ts index 1766a360b3307d..3fa9d81bd583a2 100644 --- a/plugin-server/tests/server.test.ts +++ b/plugin-server/tests/server.test.ts @@ -111,29 +111,20 @@ describe('server', () => { test('disabling pluginScheduledTasks', async () => { pluginsServer = await createPluginServer( {}, - { ingestion: true, pluginScheduledTasks: false, processPluginJobs: true } + { ingestion: true, pluginScheduledTasks: false, processJobs: true } ) expect(startPluginSchedules).not.toHaveBeenCalled() expect(startJobQueueConsumer).toHaveBeenCalled() }) - test('disabling processPluginJobs', async () => { + test('disabling processJobs', async () => { pluginsServer = await createPluginServer( {}, - { ingestion: true, pluginScheduledTasks: true, processPluginJobs: false } + { ingestion: true, pluginScheduledTasks: true, processJobs: false } ) expect(startPluginSchedules).toHaveBeenCalled() - expect(startJobQueueConsumer).toHaveBeenCalled() - }) - - test('disabling processPluginJobs and ingestion', async () => { - pluginsServer = await createPluginServer( - {}, - { ingestion: false, pluginScheduledTasks: true, processPluginJobs: false } - ) - expect(startJobQueueConsumer).not.toHaveBeenCalled() }) }) diff --git a/plugin-server/tests/worker/capabilities.test.ts b/plugin-server/tests/worker/capabilities.test.ts index caed30fabd3d67..517bbebe2b3dd7 100644 --- a/plugin-server/tests/worker/capabilities.test.ts +++ b/plugin-server/tests/worker/capabilities.test.ts @@ -72,12 +72,7 @@ describe('capabilities', () => { it('returns false if the plugin has no capabilities', () => { const shouldSetupPlugin = shouldSetupPluginInServer( - { - ingestion: true, - processAsyncHandlers: true, - processPluginJobs: true, - pluginScheduledTasks: true, - }, + { ingestion: true, processAsyncHandlers: true, processJobs: true, pluginScheduledTasks: true }, {} ) expect(shouldSetupPlugin).toEqual(false) @@ -122,13 +117,13 @@ describe('capabilities', () => { }) describe('jobs', () => { - it('returns true if plugin has any jobs and the server has processPluginJobs capability', () => { - const shouldSetupPlugin = shouldSetupPluginInServer({ processPluginJobs: true }, { jobs: ['someJob'] }) + it('returns true if plugin has any jobs and the server has processJobs capability', () => { + const shouldSetupPlugin = shouldSetupPluginInServer({ processJobs: true }, { jobs: ['someJob'] }) expect(shouldSetupPlugin).toEqual(true) }) - it('returns false if plugin has no jobs and the server has only processPluginJobs capability', () => { - const shouldSetupPlugin = shouldSetupPluginInServer({ processPluginJobs: true }, { jobs: [] }) + it('returns false if plugin has no jobs and the server has only processJobs capability', () => { + const shouldSetupPlugin = shouldSetupPluginInServer({ processJobs: true }, { jobs: [] }) expect(shouldSetupPlugin).toEqual(false) }) }) diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts index 2d2af8bdaf860a..a12164943aca4e 100644 --- a/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts +++ b/plugin-server/tests/worker/ingestion/event-pipeline/emitToBufferStep.test.ts @@ -1,7 +1,7 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { DateTime } from 'luxon' -import { JobName, Person } from '../../../../src/types' +import { Person } from '../../../../src/types' import { UUIDT } from '../../../../src/utils/utils' import { emitToBufferStep, @@ -43,26 +43,17 @@ beforeEach(() => { hub: { CONVERSION_BUFFER_ENABLED: true, BUFFER_CONVERSION_SECONDS: 60, - db: { fetchPerson: jest.fn().mockResolvedValue(existingPerson) }, + db: { fetchPerson: jest.fn().mockResolvedValue(existingPerson), addEventToBuffer: jest.fn() }, eventsProcessor: {}, - jobQueueManager: { - enqueue: jest.fn(), - }, }, } }) describe('emitToBufferStep()', () => { - it('enqueues graphile job if event should be buffered, stops processing', async () => { - const unixNow = 1657710000000 - Date.now = jest.fn(() => unixNow) - + it('calls `addEventToBuffer` if event should be buffered, stops processing', async () => { const response = await emitToBufferStep(runner, pluginEvent, () => true) - expect(runner.hub.jobQueueManager.enqueue).toHaveBeenCalledWith(JobName.BUFFER_JOB, { - eventPayload: pluginEvent, - timestamp: unixNow + 60000, // runner.hub.BUFFER_CONVERSION_SECONDS * 1000 - }) + expect(runner.hub.db.addEventToBuffer).toHaveBeenCalledWith(pluginEvent, expect.any(DateTime)) expect(runner.hub.db.fetchPerson).toHaveBeenCalledWith(2, 'my_id') expect(response).toEqual(null) }) @@ -72,7 +63,7 @@ describe('emitToBufferStep()', () => { expect(response).toEqual(['pluginsProcessEventStep', pluginEvent, existingPerson]) expect(runner.hub.db.fetchPerson).toHaveBeenCalledWith(2, 'my_id') - expect(runner.hub.jobQueueManager.enqueue).not.toHaveBeenCalled() + expect(runner.hub.db.addEventToBuffer).not.toHaveBeenCalled() }) it('calls `processPersonsStep` for $snapshot events', async () => { @@ -82,7 +73,7 @@ describe('emitToBufferStep()', () => { expect(response).toEqual(['processPersonsStep', event, undefined]) expect(runner.hub.db.fetchPerson).not.toHaveBeenCalled() - expect(runner.hub.jobQueueManager.enqueue).not.toHaveBeenCalled() + expect(runner.hub.db.addEventToBuffer).not.toHaveBeenCalled() }) }) diff --git a/plugin-server/tests/worker/vm.lazy.test.ts b/plugin-server/tests/worker/vm.lazy.test.ts index 48ac90b5cbe5d3..2e21205b8c2f8d 100644 --- a/plugin-server/tests/worker/vm.lazy.test.ts +++ b/plugin-server/tests/worker/vm.lazy.test.ts @@ -27,12 +27,7 @@ describe('LazyPluginVM', () => { const mockServer: any = { db, - capabilities: { - ingestion: true, - pluginScheduledTasks: true, - processPluginJobs: true, - processAsyncHandlers: true, - }, + capabilities: { ingestion: true, pluginScheduledTasks: true, processJobs: true, processAsyncHandlers: true }, } const createVM = () => { From ba730d15295a5f16cc5cd0be7f64120fe2de2bdf Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 14 Jul 2022 19:40:42 +0100 Subject: [PATCH 064/213] chore: enable sentry in celery workers (#10801) --- posthog/celery.py | 9 ++++++++- posthog/settings/sentry.py | 14 ++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/posthog/celery.py b/posthog/celery.py index 9008a48a2cc012..b2be2b68fc5516 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -4,7 +4,7 @@ from celery import Celery from celery.schedules import crontab -from celery.signals import setup_logging, task_postrun, task_prerun +from celery.signals import setup_logging, task_postrun, task_prerun, worker_process_init from django.conf import settings from django.db import connection from django.utils import timezone @@ -54,6 +54,13 @@ def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs) -> Non logging.config.dictConfig(logs.LOGGING) # type: ignore +@worker_process_init.connect +def on_worker_start(**kwargs) -> None: + from posthog.settings import sentry_init + + sentry_init() + + @app.on_after_configure.connect def setup_periodic_tasks(sender: Celery, **kwargs): # Monitoring tasks diff --git a/posthog/settings/sentry.py b/posthog/settings/sentry.py index bb46375f7a81d8..cd172e56a8c3a1 100644 --- a/posthog/settings/sentry.py +++ b/posthog/settings/sentry.py @@ -9,8 +9,9 @@ from posthog.settings.base_variables import TEST -if not TEST: - if os.getenv("SENTRY_DSN"): + +def sentry_init() -> None: + if not TEST and os.getenv("SENTRY_DSN"): sentry_sdk.utils.MAX_STRING_LENGTH = 10_000_000 # https://docs.sentry.io/platforms/python/ sentry_logging = sentry_logging = LoggingIntegration(level=logging.INFO, event_level=None) @@ -19,7 +20,12 @@ environment=os.getenv("SENTRY_ENVIRONMENT", "production"), integrations=[DjangoIntegration(), CeleryIntegration(), RedisIntegration(), sentry_logging], request_bodies="always", - sample_rate=1.0, # Configures the sample rate for error events, in the range of 0.0 to 1.0. The default is 1.0 which means that 100% of error events are sent. If set to 0.1 only 10% of error events will be sent. Events are picked randomly. + sample_rate=1.0, + # Configures the sample rate for error events, in the range of 0.0 to 1.0. The default is 1.0 which means that 100% of error events are sent. If set to 0.1 only 10% of error events will be sent. Events are picked randomly. send_default_pii=True, - traces_sample_rate=0.0000001, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. + traces_sample_rate=0.0000001, + # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. ) + + +sentry_init() From 08831f4fda76889701bf060300920c2b2cb40f1f Mon Sep 17 00:00:00 2001 From: timgl Date: Thu, 14 Jul 2022 22:54:43 +0200 Subject: [PATCH 065/213] test: MASSIVELY speed up backend tests with this one weird trick (#10803) --- posthog/models/cohort/cohort.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/posthog/models/cohort/cohort.py b/posthog/models/cohort/cohort.py index c745067df141e8..7002baec8f7b2f 100644 --- a/posthog/models/cohort/cohort.py +++ b/posthog/models/cohort/cohort.py @@ -215,7 +215,8 @@ def calculate_people(self, new_version: int, batch_size=10000, pg_batch_size=100 cursor += batch_size persons = self._clickhouse_persons_query(batch_size=batch_size, offset=cursor) - time.sleep(5) + if len(persons) > 0 and not TEST: + time.sleep(5) except Exception as err: # Clear the pending version people if there's an error From d51416afd310260e6802cbf37c71618cc9d9334d Mon Sep 17 00:00:00 2001 From: timgl Date: Fri, 15 Jul 2022 10:29:14 +0200 Subject: [PATCH 066/213] feat(licensing): Automatically renew licenses (#10737) --- ee/tasks/send_license_usage.py | 35 ++++++++++++------- ee/tasks/test/test_send_license_usage.py | 44 +++++++++++++++++++++++- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/ee/tasks/send_license_usage.py b/ee/tasks/send_license_usage.py index b0d69cd00caa44..45410ddf5d1573 100644 --- a/ee/tasks/send_license_usage.py +++ b/ee/tasks/send_license_usage.py @@ -2,6 +2,7 @@ import requests from dateutil.relativedelta import relativedelta from django.utils import timezone +from django.utils.timezone import now from ee.models.license import License from posthog.client import sync_execute @@ -18,6 +19,7 @@ def send_license_usage(): try: date_from = (timezone.now() - relativedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) date_to = timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) + events_count = sync_execute( "select count(1) from events where timestamp >= %(date_from)s and timestamp < %(date_to)s and not startsWith(event, '$$')", {"date_from": date_from, "date_to": date_to}, @@ -27,7 +29,14 @@ def send_license_usage(): data={"date": date_from.strftime("%Y-%m-%d"), "key": license.key, "events_count": events_count,}, ) - response.raise_for_status() + if response.status_code == 404 and response.json().get("code") == "not_found": + license.valid_until = now() - relativedelta(hours=1) + license.save() + + if response.json().get("valid_until"): + license.valid_until = response.json()["valid_until"] + license.save() + if not response.ok: posthoganalytics.capture( user.distinct_id, # type: ignore @@ -42,18 +51,18 @@ def send_license_usage(): groups={"organization": str(user.current_organization.id), "instance": SITE_URL,}, # type: ignore ) return - - posthoganalytics.capture( - user.distinct_id, # type: ignore - "send license usage data", - { - "date": date_from.strftime("%Y-%m-%d"), - "events_count": events_count, - "license_keys": get_instance_licenses(), - "organization_name": user.current_organization.name, # type: ignore - }, - groups={"organization": str(user.current_organization.id), "instance": SITE_URL,}, # type: ignore - ) + else: + posthoganalytics.capture( + user.distinct_id, # type: ignore + "send license usage data", + { + "date": date_from.strftime("%Y-%m-%d"), + "events_count": events_count, + "license_keys": get_instance_licenses(), + "organization_name": user.current_organization.name, # type: ignore + }, + groups={"organization": str(user.current_organization.id), "instance": SITE_URL,}, # type: ignore + ) except Exception as err: posthoganalytics.capture( user.distinct_id, # type: ignore diff --git a/ee/tasks/test/test_send_license_usage.py b/ee/tasks/test/test_send_license_usage.py index 7d085d3fe67ae9..62d20ff4e65ad4 100644 --- a/ee/tasks/test/test_send_license_usage.py +++ b/ee/tasks/test/test_send_license_usage.py @@ -1,8 +1,9 @@ -from unittest.mock import ANY, patch +from unittest.mock import ANY, Mock, patch from freezegun import freeze_time from ee.api.test.base import LicensedTestMixin +from ee.models.license import License from ee.tasks.send_license_usage import send_license_usage from posthog.models.team import Team from posthog.test.base import APIBaseTest, ClickhouseDestroyTablesMixin, _create_event, flush_persons_and_events @@ -27,6 +28,10 @@ def test_send_license_usage(self, mock_post, mock_capture): _create_event(event="$pageview", team=self.team, distinct_id=1, timestamp="2021-10-10T14:01:01Z") flush_persons_and_events() + mockresponse = Mock() + mock_post.return_value = mockresponse + mockresponse.json = lambda: {"ok": True, "valid_until": "2021-11-10T23:01:00Z"} + send_license_usage() mock_post.assert_called_once_with( "https://license.posthog.com/licenses/usage", @@ -38,6 +43,7 @@ def test_send_license_usage(self, mock_post, mock_capture): {"date": "2021-10-09", "events_count": 3, "license_keys": ["enterprise"], "organization_name": "Test"}, groups={"instance": ANY, "organization": str(self.organization.id)}, ) + self.assertEqual(License.objects.get().valid_until.isoformat(), "2021-11-10T23:01:00+00:00") @freeze_time("2021-10-10T23:01:00Z") @patch("posthoganalytics.capture") @@ -64,6 +70,42 @@ def test_send_license_error(self, mock_post, mock_capture): groups={"instance": ANY, "organization": str(self.organization.id)}, ) + @freeze_time("2021-10-10T23:01:00Z") + @patch("posthoganalytics.capture") + @patch("requests.post") + def test_send_license_not_found(self, mock_post, mock_capture): + team2 = Team.objects.create(organization=self.organization) + _create_event(event="$pageview", team=self.team, distinct_id=1, timestamp="2021-10-08T14:01:01Z") + _create_event(event="$pageview", team=self.team, distinct_id=1, timestamp="2021-10-09T12:01:01Z") + _create_event(event="$pageview", team=self.team, distinct_id=1, timestamp="2021-10-09T13:01:01Z") + _create_event( + event="$$internal_metrics_shouldnt_be_billed", + team=self.team, + distinct_id=1, + timestamp="2021-10-09T13:01:01Z", + ) + _create_event(event="$pageview", team=team2, distinct_id=1, timestamp="2021-10-09T14:01:01Z") + _create_event(event="$pageview", team=self.team, distinct_id=1, timestamp="2021-10-10T14:01:01Z") + flush_persons_and_events() + flush_persons_and_events() + + mockresponse = Mock() + mock_post.return_value = mockresponse + mockresponse.status_code = 404 + mockresponse.ok = False + mockresponse.json = lambda: {"code": "not_found"} + mockresponse.content = "" + + send_license_usage() + + mock_capture.assert_called_once_with( + self.user.distinct_id, + "send license usage data error", + {"error": "", "date": "2021-10-09", "organization_name": "Test", "status_code": 404, "events_count": 3}, + groups={"instance": ANY, "organization": str(self.organization.id)}, + ) + self.assertEqual(License.objects.get().valid_until.isoformat(), "2021-10-10T22:01:00+00:00") + class SendLicenseUsageNoLicenseTest(APIBaseTest): @freeze_time("2021-10-10T23:01:00Z") From 8a2feca641dfe84efaf2b1f4c9d3dcb6a551bf9b Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 10:09:16 +0100 Subject: [PATCH 067/213] chore: set worker trace rate higher than web (#10812) --- posthog/celery.py | 2 +- posthog/settings/sentry.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/posthog/celery.py b/posthog/celery.py index b2be2b68fc5516..9bc6b8777344a4 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -58,7 +58,7 @@ def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs) -> Non def on_worker_start(**kwargs) -> None: from posthog.settings import sentry_init - sentry_init() + sentry_init(traces_sample_rate=0.5) @app.on_after_configure.connect diff --git a/posthog/settings/sentry.py b/posthog/settings/sentry.py index cd172e56a8c3a1..3d5db24055cd4e 100644 --- a/posthog/settings/sentry.py +++ b/posthog/settings/sentry.py @@ -10,7 +10,7 @@ from posthog.settings.base_variables import TEST -def sentry_init() -> None: +def sentry_init(traces_sample_rate: float = 0.0000001) -> None: if not TEST and os.getenv("SENTRY_DSN"): sentry_sdk.utils.MAX_STRING_LENGTH = 10_000_000 # https://docs.sentry.io/platforms/python/ @@ -23,7 +23,7 @@ def sentry_init() -> None: sample_rate=1.0, # Configures the sample rate for error events, in the range of 0.0 to 1.0. The default is 1.0 which means that 100% of error events are sent. If set to 0.1 only 10% of error events will be sent. Events are picked randomly. send_default_pii=True, - traces_sample_rate=0.0000001, + traces_sample_rate=traces_sample_rate, # A number between 0 and 1, controlling the percentage chance a given transaction will be sent to Sentry. (0 represents 0% while 1 represents 100%.) Applies equally to all transactions created in the app. ) From 25a5b5de50fb68dd038ffe080eeb8ab47f67ab51 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 10:33:02 +0100 Subject: [PATCH 068/213] fix: counts a refresh attempt for cache_keys with no results (#10797) * fix: counts a refresh attempt for cache_keys with no results * don't reset the refresh attempt counter on empty results --- posthog/tasks/test/test_update_cache.py | 62 ++++++++++++++++++++++++- posthog/tasks/update_cache.py | 35 ++++++++++---- 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/posthog/tasks/test/test_update_cache.py b/posthog/tasks/test/test_update_cache.py index cfadab53dc301a..ec4231b3e07682 100644 --- a/posthog/tasks/test/test_update_cache.py +++ b/posthog/tasks/test/test_update_cache.py @@ -418,7 +418,7 @@ def _update_cached_items() -> None: # Magically succeeds, reset counter patch_calculate_by_filter.side_effect = None - patch_calculate_by_filter.return_value = {} + patch_calculate_by_filter.return_value = {"some": "exciting results"} _update_cached_items() self.assertEqual(Insight.objects.get().refresh_attempt, 0) self.assertEqual(DashboardTile.objects.get().refresh_attempt, 0) @@ -475,7 +475,7 @@ def _update_cached_items() -> None: # Magically succeeds, reset counter patch_calculate_by_filter.side_effect = None - patch_calculate_by_filter.return_value = {} + patch_calculate_by_filter.return_value = {"some": "exciting results"} _update_cached_items() self.assertEqual(Insight.objects.get().refresh_attempt, None) self.assertEqual(DashboardTile.objects.get().refresh_attempt, 0) @@ -680,6 +680,64 @@ def test_update_dashboard_tile_updates_only_tile_when_different_filters(self) -> assert tile.filters_hash != test_hash assert tile.last_refresh.isoformat(), "2021-08-25T22:09:14.252000+00:00" + @freeze_time("2021-08-25T22:09:14.252Z") + def test_cache_key_that_matches_no_assets_still_counts_as_a_refresh_attempt_for_dashboard_tiles(self) -> None: + test_hash = "märg koer lamab parimal tekil" + insight = self._create_insight_with_known_cache_key(test_hash) + dashboard, tile = self._create_dashboard_tile_with_known_cache_key( + insight, test_hash, dashboard_filters={"date_from": "-30d"} + ) + + assert insight.refresh_attempt is None + assert tile.refresh_attempt is None + + filter_dict: Dict[str, Any] = { + "events": [{"id": "$pageview"}], + "properties": [{"key": "$browser", "value": "Mac OS X"}], + } + + update_cache_item( + key="a key that does not match", + cache_type=CacheType.TRENDS, + payload={ + "filter": Filter(data=filter_dict).toJSON(), + "team_id": self.team.id, + "insight_id": insight.id, + "dashboard_id": dashboard.id, + }, + ) + + insight.refresh_from_db() + tile.refresh_from_db() + assert insight.refresh_attempt is None + assert tile.refresh_attempt == 1 + + @freeze_time("2021-08-25T22:09:14.252Z") + def test_cache_key_that_matches_no_assets_still_counts_as_a_refresh_attempt_for_insights(self) -> None: + test_hash = "märg koer lamab parimal tekil" + insight = self._create_insight_with_known_cache_key(test_hash) + + assert insight.refresh_attempt is None + + filter_dict: Dict[str, Any] = { + "events": [{"id": "$pageview"}], + "properties": [{"key": "$browser", "value": "Mac OS X"}], + } + + update_cache_item( + key="a key that does not match", + cache_type=CacheType.TRENDS, + payload={ + "filter": Filter(data=filter_dict).toJSON(), + "team_id": self.team.id, + "insight_id": insight.id, + "dashboard_id": None, + }, + ) + + insight.refresh_from_db() + assert insight.refresh_attempt == 1 + def _create_insight_with_known_cache_key(self, test_hash: str) -> Insight: filter_dict: Dict[str, Any] = { "events": [{"id": "$pageview"}], diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index c1ae9208642139..31dd08584bc601 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -70,15 +70,22 @@ def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Di elif insight_result is not None: result = insight_result else: + dashboard_id = payload.get("dashboard_id", None) + insight_id = payload.get("insight_id", "unknown") statsd.incr( "update_cache_item_no_results", - tags={ - "team": team_id, - "cache_key": key, - "insight_id": payload.get("insight_id", "unknown"), - "dashboard_id": payload.get("dashboard_id", None), - }, + tags={"team": team_id, "cache_key": key, "insight_id": insight_id, "dashboard_id": dashboard_id,}, ) + # there is strong likelihood these querysets match no insights or dashboard tiles + _mark_refresh_attempt_for(insights_queryset) + _mark_refresh_attempt_for(dashboard_tiles_queryset) + # so mark the item that triggered the update + if insight_id != "unknown": + _mark_refresh_attempt_for( + Insight.objects.filter(id=insight_id) + if not dashboard_id + else DashboardTile.objects.filter(insight_id=insight_id, dashboard_id=dashboard_id) + ) return [] return result @@ -101,15 +108,23 @@ def _update_cache_for_queryset( ) except Exception as e: statsd.incr("update_cache_item_error", tags={"team": team.id}) - queryset.filter(refresh_attempt=None).update(refresh_attempt=0) - queryset.update(refreshing=False, refresh_attempt=F("refresh_attempt") + 1) + _mark_refresh_attempt_for(queryset) raise e - statsd.incr("update_cache_item_success", tags={"team": team.id}) - queryset.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) + if result: + statsd.incr("update_cache_item_success", tags={"team": team.id}) + queryset.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) + else: + queryset.update(last_refresh=timezone.now(), refreshing=False) + return result +def _mark_refresh_attempt_for(queryset: QuerySet) -> None: + queryset.filter(refresh_attempt=None).update(refresh_attempt=0) + queryset.update(refreshing=False, refresh_attempt=F("refresh_attempt") + 1) + + def update_insight_cache(insight: Insight, dashboard: Optional[Dashboard]) -> List[Dict[str, Any]]: cache_key, cache_type, payload = insight_update_task_params(insight, dashboard) # check if the cache key has changed, usually because of a new default filter From 009175cb2424154480771d6133b81be6ad511bd4 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 10:36:57 +0100 Subject: [PATCH 069/213] fix: point users to the correct cohorts API to export more than limit (#10807) --- frontend/src/scenes/persons/Persons.tsx | 8 ++++---- frontend/src/scenes/persons/personsLogic.ts | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/scenes/persons/Persons.tsx b/frontend/src/scenes/persons/Persons.tsx index 44b74227107df2..e084d9a0b36f72 100644 --- a/frontend/src/scenes/persons/Persons.tsx +++ b/frontend/src/scenes/persons/Persons.tsx @@ -35,7 +35,8 @@ export function Persons({ cohort }: PersonsProps = {}): JSX.Element { export function PersonsScene(): JSX.Element { const { loadPersons, setListFilters, exportCsv } = useActions(personsLogic) - const { cohortId, persons, listFilters, personsLoading, exportUrl, exporterProps } = useValues(personsLogic) + const { cohortId, persons, listFilters, personsLoading, exportUrl, exporterProps, apiDocsURL } = + useValues(personsLogic) const { featureFlags } = useValues(featureFlagLogic) const newExportButtonActive = !!featureFlags[FEATURE_FLAGS.ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS] @@ -53,9 +54,8 @@ export function PersonsScene(): JSX.Element { <> Exporting by csv is limited to 10,000 users.
- To return more, please use{' '} - the API. Do you want to export by - CSV? + To return more, please use the API. Do you want to export + by CSV? } onConfirm={() => (newExportButtonActive ? triggerExport(exporterProps[0]) : exportCsv())} diff --git a/frontend/src/scenes/persons/personsLogic.ts b/frontend/src/scenes/persons/personsLogic.ts index 9c0dc669a624d3..907105fbcd1708 100644 --- a/frontend/src/scenes/persons/personsLogic.ts +++ b/frontend/src/scenes/persons/personsLogic.ts @@ -97,6 +97,13 @@ export const personsLogic = kea({ }, }, selectors: () => ({ + apiDocsURL: [ + () => [(_, props) => props.cohort], + (cohort: PersonLogicProps['cohort']) => + !!cohort + ? 'https://posthog.com/docs/api/cohorts#get-api-projects-project_id-cohorts-id-persons' + : 'https://posthog.com/docs/api/persons', + ], cohortId: [() => [(_, props) => props.cohort], (cohort: PersonLogicProps['cohort']) => cohort], showSessionRecordings: [ (s) => [s.currentTeam], From 206322178dff9a50aec826eb641d31a3744cb33e Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 11:16:39 +0100 Subject: [PATCH 070/213] feat: write image exports to object storage (#10809) * feat: write image exports to object storage * setup_method was confusing things * correct patch in test * fixes test --- posthog/models/exported_asset.py | 44 +++++++++- posthog/tasks/exports/csv_exporter.py | 37 +------- posthog/tasks/exports/image_exporter.py | 77 +++++++++------- .../tasks/exports/test/test_csv_exporter.py | 42 ++++----- .../exports/test/test_csv_exporter_renders.py | 2 +- .../tasks/exports/test/test_image_exporter.py | 87 +++++++++++++++++++ posthog/tasks/test/test_exporter.py | 7 +- 7 files changed, 208 insertions(+), 88 deletions(-) create mode 100644 posthog/tasks/exports/test/test_image_exporter.py diff --git a/posthog/models/exported_asset.py b/posthog/models/exported_asset.py index d35bb76dc6ecd3..9764d870957cc4 100644 --- a/posthog/models/exported_asset.py +++ b/posthog/models/exported_asset.py @@ -1,16 +1,23 @@ import secrets from datetime import timedelta -from typing import Optional +from typing import List, Optional +import structlog +from django.conf import settings from django.db import models from django.http import HttpResponse from django.utils.text import slugify +from sentry_sdk import capture_exception from posthog.jwt import PosthogJwtAudience, decode_jwt, encode_jwt +from posthog.models.utils import UUIDT from posthog.settings import DEBUG from posthog.storage import object_storage +from posthog.storage.object_storage import ObjectStorageError from posthog.utils import absolute_uri +logger = structlog.get_logger(__name__) + PUBLIC_ACCESS_TOKEN_EXP_DAYS = 365 MAX_AGE_CONTENT = 86400 # 1 day @@ -94,7 +101,7 @@ def asset_for_token(token: str) -> ExportedAsset: def get_content_response(asset: ExportedAsset, download: bool = False): content = asset.content if not content and asset.content_location: - content = object_storage.read(asset.content_location) + content = object_storage.read_bytes(asset.content_location) res = HttpResponse(content, content_type=asset.export_format) if download: @@ -104,3 +111,36 @@ def get_content_response(asset: ExportedAsset, download: bool = False): res["Cache-Control"] = f"max-age={MAX_AGE_CONTENT}" return res + + +def save_content(exported_asset: ExportedAsset, content: bytes) -> None: + try: + if settings.OBJECT_STORAGE_ENABLED: + save_content_to_object_storage(exported_asset, content) + else: + save_content_to_exported_asset(exported_asset, content) + except ObjectStorageError as ose: + capture_exception(ose) + logger.error( + "exported_asset.object-storage-error", exported_asset_id=exported_asset.id, exception=ose, exc_info=True + ) + save_content_to_exported_asset(exported_asset, content) + + +def save_content_to_exported_asset(exported_asset: ExportedAsset, content: bytes) -> None: + exported_asset.content = content + exported_asset.save(update_fields=["content"]) + + +def save_content_to_object_storage(exported_asset: ExportedAsset, content: bytes) -> None: + path_parts: List[str] = [ + settings.OBJECT_STORAGE_EXPORTS_FOLDER, + exported_asset.export_format.split("/")[1], + f"team-{exported_asset.team.id}", + f"task-{exported_asset.id}", + str(UUIDT()), + ] + object_path = f'/{"/".join(path_parts)}' + object_storage.write(object_path, content) + exported_asset.content_location = object_path + exported_asset.save(update_fields=["content_location"]) diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index 7522eb23f7e197..ab2f0cb6d3b1a1 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -4,17 +4,13 @@ import requests import structlog -from django.conf import settings from rest_framework_csv import renderers as csvrenderers from sentry_sdk import capture_exception, push_scope from statshog.defaults.django import statsd from posthog.jwt import PosthogJwtAudience, encode_jwt from posthog.logging.timing import timed -from posthog.models.exported_asset import ExportedAsset -from posthog.models.utils import UUIDT -from posthog.storage import object_storage -from posthog.storage.object_storage import ObjectStorageError +from posthog.models.exported_asset import ExportedAsset, save_content from posthog.utils import absolute_uri logger = structlog.get_logger(__name__) @@ -177,36 +173,7 @@ def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: rendered_csv_content = renderer.render(all_csv_rows) - try: - if settings.OBJECT_STORAGE_ENABLED: - _write_to_object_storage(exported_asset, rendered_csv_content) - else: - _write_to_exported_asset(exported_asset, rendered_csv_content) - except ObjectStorageError as ose: - with push_scope() as scope: - scope.set_tag("celery_task", "csv_export") - capture_exception(ose) - logger.error("csv_exporter.object-storage-error", exception=ose, exc_info=True) - _write_to_exported_asset(exported_asset, rendered_csv_content) - - -def _write_to_exported_asset(exported_asset: ExportedAsset, rendered_csv_content: bytes) -> None: - exported_asset.content = rendered_csv_content - exported_asset.save(update_fields=["content"]) - - -def _write_to_object_storage(exported_asset: ExportedAsset, rendered_csv_content: bytes) -> None: - path_parts: List[str] = [ - settings.OBJECT_STORAGE_EXPORTS_FOLDER, - "csvs", - f"team-{exported_asset.team.id}", - f"task-{exported_asset.id}", - str(UUIDT()), - ] - object_path = f'/{"/".join(path_parts)}' - object_storage.write(object_path, rendered_csv_content) - exported_asset.content_location = object_path - exported_asset.save(update_fields=["content_location"]) + save_content(exported_asset, rendered_csv_content) @timed("csv_exporter") diff --git a/posthog/tasks/exports/image_exporter.py b/posthog/tasks/exports/image_exporter.py index 74041399d3263c..37bc0eaba47f24 100644 --- a/posthog/tasks/exports/image_exporter.py +++ b/posthog/tasks/exports/image_exporter.py @@ -3,6 +3,7 @@ import time import uuid from datetime import timedelta +from typing import Literal, Optional import structlog from django.conf import settings @@ -11,12 +12,14 @@ from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support.wait import WebDriverWait +from sentry_sdk import capture_exception +from statshog.defaults.django import statsd from webdriver_manager.chrome import ChromeDriverManager from webdriver_manager.utils import ChromeType from posthog.internal_metrics import incr, timing from posthog.logging.timing import timed -from posthog.models.exported_asset import ExportedAsset, get_public_access_token +from posthog.models.exported_asset import ExportedAsset, get_public_access_token, save_content from posthog.tasks.update_cache import update_insight_cache from posthog.utils import absolute_uri @@ -24,6 +27,8 @@ TMP_DIR = "/tmp" # NOTE: Externalise this to ENV var +ScreenWidth = Literal[800, 1920] +CSSSelector = Literal[".InsightCard", ".ExportedInsight"] # NOTE: We purporsefully DONT re-use the driver. It would be slightly faster but would keep an in-memory browser # window permanently around which is unnecessary @@ -57,7 +62,6 @@ def _export_to_png(exported_asset: ExportedAsset) -> None: _start = time.time() - driver = None image_path = None try: @@ -69,14 +73,14 @@ def _export_to_png(exported_asset: ExportedAsset) -> None: image_id = str(uuid.uuid4()) image_path = os.path.join(TMP_DIR, f"{image_id}.png") - url_to_render = None - screenshot_width = 800 - if not os.path.exists(TMP_DIR): os.makedirs(TMP_DIR) access_token = get_public_access_token(exported_asset, timedelta(minutes=15)) + screenshot_width: ScreenWidth + wait_for_css_selector: CSSSelector + if exported_asset.insight is not None: url_to_render = absolute_uri(f"/exporter?token={access_token}&legend") wait_for_css_selector = ".ExportedInsight" @@ -88,24 +92,14 @@ def _export_to_png(exported_asset: ExportedAsset) -> None: else: raise Exception(f"Export is missing required dashboard or insight ID") - logger.info(f"Exporting {exported_asset.id} from {url_to_render}") - - driver = get_driver() - driver.set_window_size(screenshot_width, screenshot_width * 0.5) - driver.get(url_to_render) - - WebDriverWait(driver, 10).until(lambda x: x.find_element(By.CSS_SELECTOR, wait_for_css_selector)) + logger.info("exporting_asset", asset_id=exported_asset.id, render_url=url_to_render) - height = driver.execute_script("return document.body.scrollHeight") - - driver.set_window_size(screenshot_width, height) - driver.save_screenshot(image_path) + _screenshot_asset(image_path, url_to_render, screenshot_width, wait_for_css_selector) with open(image_path, "rb") as image_file: image_data = image_file.read() - exported_asset.content = image_data - exported_asset.save() + save_content(exported_asset, image_data) os.remove(image_path) timing("exporter_task_success", time.time() - _start) @@ -115,9 +109,21 @@ def _export_to_png(exported_asset: ExportedAsset) -> None: if image_path and os.path.exists(image_path): os.remove(image_path) - incr("exporter_task_failure") - logger.error(f"Error: {err}") raise err + + +def _screenshot_asset( + image_path: str, url_to_render: str, screenshot_width: ScreenWidth, wait_for_css_selector: CSSSelector, +) -> None: + driver: Optional[webdriver.Chrome] = None + try: + driver = get_driver() + driver.set_window_size(screenshot_width, screenshot_width * 0.5) + driver.get(url_to_render) + WebDriverWait(driver, 10).until(lambda x: x.find_element(By.CSS_SELECTOR, wait_for_css_selector)) + height = driver.execute_script("return document.body.scrollHeight") + driver.set_window_size(screenshot_width, height) + driver.save_screenshot(image_path) finally: if driver: driver.close() @@ -125,12 +131,25 @@ def _export_to_png(exported_asset: ExportedAsset) -> None: @timed("image_exporter") def export_image(exported_asset: ExportedAsset) -> None: - if exported_asset.insight: - # NOTE: Dashboards are regularly updated but insights are not - # so, we need to trigger a manual update to ensure the results are good - update_insight_cache(exported_asset.insight, dashboard=exported_asset.dashboard) - - if exported_asset.export_format == "image/png": - return _export_to_png(exported_asset) - else: - raise NotImplementedError(f"Export to format {exported_asset.export_format} is not supported for insights") + try: + if exported_asset.insight: + # NOTE: Dashboards are regularly updated but insights are not + # so, we need to trigger a manual update to ensure the results are good + update_insight_cache(exported_asset.insight, dashboard=exported_asset.dashboard) + + if exported_asset.export_format == "image/png": + _export_to_png(exported_asset) + statsd.incr("image_exporter.succeeded", tags={"team_id": exported_asset.team.id}) + else: + raise NotImplementedError(f"Export to format {exported_asset.export_format} is not supported for insights") + except Exception as e: + if exported_asset: + team_id = str(exported_asset.team.id) + else: + team_id = "unknown" + + capture_exception(e) + + logger.error("image_exporter.failed", exception=e, exc_info=True) + incr("exporter_task_failure", tags={"team_id": team_id}) + raise e diff --git a/posthog/tasks/exports/test/test_csv_exporter.py b/posthog/tasks/exports/test/test_csv_exporter.py index db495d498d2320..343cf6899f5f76 100644 --- a/posthog/tasks/exports/test/test_csv_exporter.py +++ b/posthog/tasks/exports/test/test_csv_exporter.py @@ -75,16 +75,14 @@ def patched_request(self): patched_request.return_value = mock_response yield patched_request - exported_asset: ExportedAsset - - def setup_method(self, method): + def _create_asset(self) -> ExportedAsset: asset = ExportedAsset( team=self.team, export_format=ExportedAsset.ExportFormat.CSV, export_context={"path": "/api/literally/anything"}, ) asset.save() - self.exported_asset = asset + return asset def teardown_method(self, method): s3 = resource( @@ -99,48 +97,52 @@ def teardown_method(self, method): bucket.objects.filter(Prefix=TEST_BUCKET).delete() def test_csv_exporter_writes_to_asset_when_object_storage_is_disabled(self) -> None: + exported_asset = self._create_asset() with self.settings(OBJECT_STORAGE_ENABLED=False): - csv_exporter.export_csv(self.exported_asset) + csv_exporter.export_csv(exported_asset) assert ( - self.exported_asset.content + exported_asset.content == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" ) - assert self.exported_asset.content_location is None + assert exported_asset.content_location is None - @patch("posthog.tasks.exports.csv_exporter.UUIDT") + @patch("posthog.models.exported_asset.UUIDT") def test_csv_exporter_writes_to_object_storage_when_object_storage_is_enabled(self, mocked_uuidt) -> None: + exported_asset = self._create_asset() mocked_uuidt.return_value = "a-guid" + with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): - csv_exporter.export_csv(self.exported_asset) + csv_exporter.export_csv(exported_asset) assert ( - self.exported_asset.content_location - == f"/{TEST_BUCKET}/csvs/team-{self.team.id}/task-{self.exported_asset.id}/a-guid" + exported_asset.content_location + == f"/{TEST_BUCKET}/csv/team-{self.team.id}/task-{exported_asset.id}/a-guid" ) - content = object_storage.read(self.exported_asset.content_location) + content = object_storage.read(exported_asset.content_location) assert ( content == "distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" ) - assert self.exported_asset.content is None + assert exported_asset.content is None - @patch("posthog.tasks.exports.csv_exporter.UUIDT") - @patch("posthog.tasks.exports.csv_exporter.object_storage.write") - def test_csv_exporter_writes_to_object_storage_when_object_storage_write_fails( - self, mocked_uuidt, mocked_object_storage_write + @patch("posthog.models.exported_asset.UUIDT") + @patch("posthog.models.exported_asset.object_storage.write") + def test_csv_exporter_writes_to_asset_when_object_storage_write_fails( + self, mocked_object_storage_write, mocked_uuidt ) -> None: + exported_asset = self._create_asset() mocked_uuidt.return_value = "a-guid" mocked_object_storage_write.side_effect = ObjectStorageError("mock write failed") with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): - csv_exporter.export_csv(self.exported_asset) + csv_exporter.export_csv(exported_asset) - assert self.exported_asset.content_location is None + assert exported_asset.content_location is None assert ( - self.exported_asset.content + exported_asset.content == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" ) diff --git a/posthog/tasks/exports/test/test_csv_exporter_renders.py b/posthog/tasks/exports/test/test_csv_exporter_renders.py index 417c8643992b25..4b9f3d7c5be195 100644 --- a/posthog/tasks/exports/test/test_csv_exporter_renders.py +++ b/posthog/tasks/exports/test/test_csv_exporter_renders.py @@ -23,7 +23,7 @@ @pytest.mark.parametrize("filename", fixtures) @pytest.mark.django_db @patch("posthog.tasks.exports.csv_exporter.requests.request") -@patch("posthog.tasks.exports.csv_exporter.settings") +@patch("posthog.models.exported_asset.settings") def test_csv_rendering(mock_settings, mock_request, filename): mock_settings.OBJECT_STORAGE_ENABLED = False org = Organization.objects.create(name="org") diff --git a/posthog/tasks/exports/test/test_image_exporter.py b/posthog/tasks/exports/test/test_image_exporter.py new file mode 100644 index 00000000000000..69368df883ccf6 --- /dev/null +++ b/posthog/tasks/exports/test/test_image_exporter.py @@ -0,0 +1,87 @@ +from unittest.mock import mock_open, patch + +from boto3 import resource +from botocore.client import Config + +from posthog.models import ExportedAsset, Insight +from posthog.settings import ( + OBJECT_STORAGE_ACCESS_KEY_ID, + OBJECT_STORAGE_BUCKET, + OBJECT_STORAGE_ENDPOINT, + OBJECT_STORAGE_SECRET_ACCESS_KEY, +) +from posthog.storage import object_storage +from posthog.storage.object_storage import ObjectStorageError +from posthog.tasks.exports import image_exporter +from posthog.test.base import APIBaseTest + +TEST_BUCKET = "Test-Exports" + + +@patch("posthog.tasks.exports.image_exporter.update_insight_cache") +@patch("posthog.tasks.exports.image_exporter._screenshot_asset") +@patch("builtins.open", new_callable=mock_open, read_data=b"image_data") +@patch("os.remove") +class TestImageExporter(APIBaseTest): + exported_asset: ExportedAsset + + def setup_method(self, method): + insight = Insight.objects.create(team=self.team) + asset = ExportedAsset.objects.create( + team=self.team, export_format=ExportedAsset.ExportFormat.PNG, insight=insight, + ) + self.exported_asset = asset + + def teardown_method(self, method): + s3 = resource( + "s3", + endpoint_url=OBJECT_STORAGE_ENDPOINT, + aws_access_key_id=OBJECT_STORAGE_ACCESS_KEY_ID, + aws_secret_access_key=OBJECT_STORAGE_SECRET_ACCESS_KEY, + config=Config(signature_version="s3v4"), + region_name="us-east-1", + ) + bucket = s3.Bucket(OBJECT_STORAGE_BUCKET) + bucket.objects.filter(Prefix=TEST_BUCKET).delete() + + def test_image_exporter_writes_to_asset_when_object_storage_is_disabled( + self, mock_update_cache, mock_screenshot, mock_file_read, mock_remove + ) -> None: + with self.settings(OBJECT_STORAGE_ENABLED=False): + image_exporter.export_image(self.exported_asset) + + assert self.exported_asset.content == b"image_data" + assert self.exported_asset.content_location is None + + @patch("posthog.models.exported_asset.UUIDT") + def test_image_exporter_writes_to_object_storage_when_object_storage_is_enabled( + self, mocked_uuidt, mock_update_cache, mock_screenshot, mock_file_read, mock_remove + ) -> None: + mocked_uuidt.return_value = "a-guid" + with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): + image_exporter.export_image(self.exported_asset) + + assert ( + self.exported_asset.content_location + == f"/{TEST_BUCKET}/png/team-{self.team.id}/task-{self.exported_asset.id}/a-guid" + ) + + content = object_storage.read_bytes(self.exported_asset.content_location) + assert content == b"image_data" + + assert self.exported_asset.content is None + + @patch("posthog.models.exported_asset.UUIDT") + @patch("posthog.models.exported_asset.object_storage.write") + def test_image_exporter_writes_to_object_storage_when_object_storage_write_fails( + self, mocked_object_storage_write, mocked_uuidt, mock_update_cache, mock_screenshot, mock_file_read, mock_remove + ) -> None: + mocked_uuidt.return_value = "a-guid" + mocked_object_storage_write.side_effect = ObjectStorageError("mock write failed") + + with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): + image_exporter.export_image(self.exported_asset) + + assert self.exported_asset.content_location is None + + assert self.exported_asset.content == b"image_data" diff --git a/posthog/tasks/test/test_exporter.py b/posthog/tasks/test/test_exporter.py index 9b0fc995c13f00..137987b9159122 100644 --- a/posthog/tasks/test/test_exporter.py +++ b/posthog/tasks/test/test_exporter.py @@ -28,10 +28,15 @@ def setUpTestData(cls) -> None: @patch("posthog.tasks.exports.image_exporter.get_driver") def test_exporter_runs(self, mock_get_driver: MagicMock, mock_uuid: MagicMock) -> None: mock_uuid.uuid4.return_value = "posthog_test_exporter" + assert self.exported_asset.content is None + assert self.exported_asset.content_location is None + exporter.export_asset(self.exported_asset.id) self.exported_asset.refresh_from_db() - assert self.exported_asset.content is not None + + assert self.exported_asset.content is None + assert self.exported_asset.content_location is not None def test_exporter_setsup_selenium(self, mock_uuid: MagicMock) -> None: driver = get_driver() From 663ac01baaeb3d3698dd1596f37d7f2676fe5881 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 14:46:07 +0100 Subject: [PATCH 071/213] fix: csv_exporter reports API errors as errors (#10823) --- posthog/tasks/exports/csv_exporter.py | 3 +++ posthog/tasks/exports/test/test_csv_exporter.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index ab2f0cb6d3b1a1..d8e1c53fd2dd6f 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -151,6 +151,9 @@ def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: response = requests.request( method=method.lower(), url=url, json=body, headers={"Authorization": f"Bearer {access_token}"}, ) + if not response.ok: + raise Exception(f"export API call failed with status_code: {response.status_code}") + # Figure out how to handle funnel polling.... data = response.json() csv_rows = _convert_response_to_csv_data(data) diff --git a/posthog/tasks/exports/test/test_csv_exporter.py b/posthog/tasks/exports/test/test_csv_exporter.py index 343cf6899f5f76..842476f71cbde7 100644 --- a/posthog/tasks/exports/test/test_csv_exporter.py +++ b/posthog/tasks/exports/test/test_csv_exporter.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest from boto3 import resource @@ -146,3 +146,16 @@ def test_csv_exporter_writes_to_asset_when_object_storage_write_fails( exported_asset.content == b"distinct_id,elements_chain,event,id,person,properties.$browser,timestamp\r\n2,,event_name,e9ca132e-400f-4854-a83c-16c151b2f145,,Safari,2022-07-06T19:37:43.095295+00:00\r\n2,,event_name,1624228e-a4f1-48cd-aabc-6baa3ddb22e4,,Safari,2022-07-06T19:37:43.095279+00:00\r\n2,,event_name,66d45914-bdf5-4980-a54a-7dc699bdcce9,,Safari,2022-07-06T19:37:43.095262+00:00\r\n" ) + + @patch("posthog.tasks.exports.csv_exporter.logger") + @patch("posthog.tasks.exports.csv_exporter.statsd") + def test_failing_export_api_is_reported(self, mock_statsd, mock_logger) -> None: + with patch("posthog.tasks.exports.csv_exporter.requests.request") as patched_request: + exported_asset = self._create_asset() + mock_response = MagicMock() + mock_response.status_code = 403 + mock_response.ok = False + patched_request.return_value = mock_response + + with pytest.raises(Exception, match="export API call failed with status_code: 403"): + csv_exporter.export_csv(exported_asset) From d5f6ef7e4efb1afb69300428d836bcb533ce0b97 Mon Sep 17 00:00:00 2001 From: timgl Date: Fri, 15 Jul 2022 16:41:14 +0200 Subject: [PATCH 072/213] perf(Decide): Execute all feature flag matching in single query (#10804) * perf(Decide): Execute all feature flag matching in single query * fix test * fix decide test * add failing test * minor test fixes * fix test * Update snapshots * Update posthog/test/test_feature_flag.py Co-authored-by: Neil Kakkar * fix formatting Co-authored-by: Neil Kakkar Co-authored-by: timgl --- posthog/api/feature_flag.py | 10 +- posthog/api/test/test_decide.py | 4 +- posthog/models/feature_flag.py | 181 +++++----- posthog/models/filters/test/test_filter.py | 11 +- posthog/models/property/property.py | 16 +- .../test/__snapshots__/test_feature_flag.ambr | 312 +++++------------- posthog/test/test_feature_flag.py | 202 +++++++++--- 7 files changed, 357 insertions(+), 379 deletions(-) diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index 41f7d98b54adfd..0326becddd54d3 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -16,6 +16,7 @@ from posthog.models.activity_logging.activity_log import Detail, changes_between, load_activity, log_activity from posthog.models.activity_logging.activity_page import activity_page_response from posthog.models.cohort import Cohort +from posthog.models.feature_flag import FeatureFlagMatcher from posthog.models.property import Property from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission @@ -203,12 +204,13 @@ def my_flags(self, request: request.Request, **kwargs): ) groups = json.loads(request.GET.get("groups", "{}")) flags = [] + matches = FeatureFlagMatcher(list(feature_flags), request.user.distinct_id, groups).get_matches() for feature_flag in feature_flags: - match = feature_flag.matches(request.user.distinct_id, groups) - value = (match.variant or True) if match else False - flags.append( - {"feature_flag": FeatureFlagSerializer(feature_flag).data, "value": value,} + { + "feature_flag": FeatureFlagSerializer(feature_flag).data, + "value": matches.get(feature_flag.key, False), + } ) return Response(flags) diff --git a/posthog/api/test/test_decide.py b/posthog/api/test/test_decide.py index 7fb8f7dba3eeaf..d7737dd41ad455 100644 --- a/posthog/api/test/test_decide.py +++ b/posthog/api/test/test_decide.py @@ -139,14 +139,14 @@ def test_feature_flags(self): created_by=self.user, ) - with self.assertNumQueries(4): + with self.assertNumQueries(3): response = self._post_decide() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertIn("default-flag", response.json()["featureFlags"]) self.assertIn("beta-feature", response.json()["featureFlags"]) self.assertIn("filer-by-property-2", response.json()["featureFlags"]) - with self.assertNumQueries(4): + with self.assertNumQueries(3): response = self._post_decide({"token": self.team.api_token, "distinct_id": "another_id"}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["featureFlags"], ["default-flag"]) diff --git a/posthog/models/feature_flag.py b/posthog/models/feature_flag.py index 659682cfb2115b..e902534a9162a7 100644 --- a/posthog/models/feature_flag.py +++ b/posthog/models/feature_flag.py @@ -4,6 +4,7 @@ from django.core.cache import cache from django.db import models +from django.db.models import Q from django.db.models.expressions import ExpressionWrapper, RawSQL from django.db.models.fields import BooleanField from django.db.models.query import QuerySet @@ -51,9 +52,6 @@ class Meta: ensure_experience_continuity: models.BooleanField = models.BooleanField(default=False, null=True, blank=True) - def matches(self, *args, **kwargs) -> Optional[FeatureFlagMatch]: - return FeatureFlagMatcher(self, *args, **kwargs).get_match() - def get_analytics_metadata(self) -> Dict: filter_count = sum(len(condition.get("properties", [])) for condition in self.conditions) variants_count = len(self.variants) @@ -122,6 +120,9 @@ def update_cohorts(self) -> None: for cohort in Cohort.objects.filter(pk__in=self.cohort_ids): update_cohort(cohort) + def __str__(self): + return f"{self.key} ({self.pk})" + @mutable_receiver(pre_delete, sender=Experiment) def delete_experiment_flags(sender, instance, **kwargs): @@ -164,101 +165,131 @@ def group_type_index_to_name(self) -> Dict[GroupTypeIndex, GroupTypeName]: class FeatureFlagMatcher: def __init__( self, - feature_flag: FeatureFlag, + feature_flags: List[FeatureFlag], distinct_id: str, groups: Dict[GroupTypeName, str] = {}, cache: Optional[FlagsMatcherCache] = None, hash_key_overrides: Dict[str, str] = {}, ): - self.feature_flag = feature_flag + self.feature_flags = feature_flags self.distinct_id = distinct_id self.groups = groups - self.cache = cache or FlagsMatcherCache(self.feature_flag.team_id) + self.cache = cache or FlagsMatcherCache(self.feature_flags[0].team_id) self.hash_key_overrides = hash_key_overrides - def get_match(self) -> Optional[FeatureFlagMatch]: + def get_match(self, feature_flag: FeatureFlag) -> Optional[FeatureFlagMatch]: # If aggregating flag by groups and relevant group type is not passed - flag is off! - if self.hashed_identifier is None: + if self.hashed_identifier(feature_flag) is None: return None is_match = any( - self.is_condition_match(condition, index) for index, condition in enumerate(self.feature_flag.conditions) + self.is_condition_match(feature_flag, condition, index) + for index, condition in enumerate(feature_flag.conditions) ) if is_match: - return FeatureFlagMatch(variant=self.get_matching_variant()) + return FeatureFlagMatch(variant=self.get_matching_variant(feature_flag)) else: return None - def get_matching_variant(self) -> Optional[str]: - for variant in self.variant_lookup_table: - if self._variant_hash >= variant["value_min"] and self._variant_hash < variant["value_max"]: + def get_matches(self) -> Dict[str, Union[str, bool]]: + flags_enabled = {} + for feature_flag in self.feature_flags: + try: + match = self.get_match(feature_flag) + if match: + flags_enabled[feature_flag.key] = match.variant or True + except Exception as err: + capture_exception(err) + return flags_enabled + + def get_matching_variant(self, feature_flag: FeatureFlag) -> Optional[str]: + for variant in self.variant_lookup_table(feature_flag): + if ( + self.get_hash(feature_flag, salt="variant") >= variant["value_min"] + and self.get_hash(feature_flag, salt="variant") < variant["value_max"] + ): return variant["key"] return None - def is_condition_match(self, condition: Dict, condition_index: int): + def is_condition_match(self, feature_flag: FeatureFlag, condition: Dict, condition_index: int): rollout_percentage = condition.get("rollout_percentage") if len(condition.get("properties", [])) > 0: - if not self._condition_matches(condition_index): + if not self._condition_matches(feature_flag, condition_index): return False elif rollout_percentage is None: return True - if rollout_percentage is not None and self._hash > (rollout_percentage / 100): + if rollout_percentage is not None and self.get_hash(feature_flag) > (rollout_percentage / 100): return False return True - def _condition_matches(self, condition_index: int) -> bool: - return len(self.query_conditions) > 0 and self.query_conditions[0][condition_index] + def _condition_matches(self, feature_flag: FeatureFlag, condition_index: int) -> bool: + return self.query_conditions.get(f"flag_{feature_flag.pk}_condition_{condition_index}", False) # Define contiguous sub-domains within [0, 1]. # By looking up a random hash value, you can find the associated variant key. # e.g. the first of two variants with 50% rollout percentage will have value_max: 0.5 # and the second will have value_min: 0.5 and value_max: 1.0 - @property - def variant_lookup_table(self): + def variant_lookup_table(self, feature_flag: FeatureFlag): lookup_table = [] value_min = 0 - for variant in self.feature_flag.variants: + for variant in feature_flag.variants: value_max = value_min + variant["rollout_percentage"] / 100 lookup_table.append({"value_min": value_min, "value_max": value_max, "key": variant["key"]}) value_min = value_max return lookup_table @cached_property - def query_conditions(self) -> List[List[bool]]: - if self.feature_flag.aggregation_group_type_index is None: - query: QuerySet = Person.objects.filter( - team_id=self.feature_flag.team_id, - persondistinctid__distinct_id=self.distinct_id, - persondistinctid__team_id=self.feature_flag.team_id, - ) - else: - query = Group.objects.filter( - team_id=self.feature_flag.team_id, - group_type_index=self.feature_flag.aggregation_group_type_index, - group_key=self.hashed_identifier, - ) - - fields = [] - for index, condition in enumerate(self.feature_flag.conditions): - key = f"condition_{index}" - - if len(condition.get("properties", {})) > 0: - # Feature Flags don't support OR filtering yet - expr: Any = properties_to_Q( - Filter(data=condition).property_groups.flat, team_id=self.feature_flag.team_id, is_direct_query=True - ) - else: - expr = RawSQL("true", []) - - query = query.annotate(**{key: ExpressionWrapper(expr, output_field=BooleanField())}) - fields.append(key) - - return list(query.values_list(*fields)) - - @cached_property - def hashed_identifier(self) -> Optional[str]: + def query_conditions(self) -> Dict[str, bool]: + team_id = self.feature_flags[0].team_id + person_query: QuerySet = Person.objects.filter( + team_id=team_id, persondistinctid__distinct_id=self.distinct_id, persondistinctid__team_id=team_id, + ) + group_query: QuerySet = Group.objects.filter(team_id=team_id,) + person_fields = [] + group_fields = [] + + for feature_flag in self.feature_flags: + for index, condition in enumerate(feature_flag.conditions): + key = f"flag_{feature_flag.pk}_condition_{index}" + expr: Any = None + if len(condition.get("properties", {})) > 0: + # Feature Flags don't support OR filtering yet + expr = properties_to_Q( + Filter(data=condition).property_groups.flat, team_id=team_id, is_direct_query=True + ) + + if feature_flag.aggregation_group_type_index is None: + person_query = person_query.annotate( + **{key: ExpressionWrapper(expr if expr else RawSQL("true", []), output_field=BooleanField())} + ) + person_fields.append(key) + else: + group_filter = Q( + group_type_index=feature_flag.aggregation_group_type_index, + group_key=self.hashed_identifier(feature_flag), + ) + if expr: + expr = expr & group_filter + else: + expr = group_filter + group_query = group_query.annotate(**{key: ExpressionWrapper(expr, output_field=BooleanField())}) + group_fields.append(key) + + all_conditions = {} + if len(person_fields) > 0: + person_query = person_query.values(*person_fields) + if len(person_query) > 0: + all_conditions = {**person_query[0]} + if len(group_fields) > 0: + group_query = group_query.values(*group_fields) + if len(group_query) > 0: + all_conditions = {**all_conditions, **group_query[0]} + + return all_conditions + + def hashed_identifier(self, feature_flag: FeatureFlag) -> Optional[str]: """ If aggregating by people, returns distinct_id. @@ -266,15 +297,15 @@ def hashed_identifier(self) -> Optional[str]: If relevant group is not passed to the flag, None is returned and handled in get_match. """ - if self.feature_flag.aggregation_group_type_index is None: - if self.feature_flag.ensure_experience_continuity: + if feature_flag.aggregation_group_type_index is None: + if feature_flag.ensure_experience_continuity: # TODO: Try a global cache - if self.feature_flag.key in self.hash_key_overrides: - return self.hash_key_overrides[self.feature_flag.key] + if feature_flag.key in self.hash_key_overrides: + return self.hash_key_overrides[feature_flag.key] return self.distinct_id else: # :TRICKY: If aggregating by groups - group_type_name = self.cache.group_type_index_to_name.get(self.feature_flag.aggregation_group_type_index) + group_type_name = self.cache.group_type_index_to_name.get(feature_flag.aggregation_group_type_index) group_key = self.groups.get(group_type_name) # type: ignore return group_key @@ -282,19 +313,11 @@ def hashed_identifier(self) -> Optional[str]: # Given the same identifier and key, it'll always return the same float. These floats are # uniformly distributed between 0 and 1, so if we want to show this feature to 20% of traffic # we can do _hash(key, identifier) < 0.2 - def get_hash(self, salt="") -> float: - hash_key = f"{self.feature_flag.key}.{self.hashed_identifier}{salt}" + def get_hash(self, feature_flag: FeatureFlag, salt="") -> float: + hash_key = f"{feature_flag.key}.{self.hashed_identifier(feature_flag)}{salt}" hash_val = int(hashlib.sha1(hash_key.encode("utf-8")).hexdigest()[:15], 16) return hash_val / __LONG_SCALE__ - @cached_property - def _hash(self): - return self.get_hash() - - @cached_property - def _variant_hash(self) -> float: - return self.get_hash(salt="variant") - def hash_key_overrides(team_id: int, person_id: int) -> Dict[str, str]: feature_flag_to_key_overrides = {} @@ -308,34 +331,26 @@ def hash_key_overrides(team_id: int, person_id: int) -> Dict[str, str]: # Return a Dict with all active flags and their values def _get_active_feature_flags( - feature_flags: QuerySet, + feature_flags: List[FeatureFlag], team_id: int, distinct_id: str, person_id: Optional[int] = None, groups: Dict[GroupTypeName, str] = {}, -) -> Dict[str, Union[bool, str, None]]: +) -> Dict[str, Union[bool, str]]: cache = FlagsMatcherCache(team_id) - flags_enabled: Dict[str, Union[bool, str, None]] = {} if person_id is not None: overrides = hash_key_overrides(team_id, person_id) else: overrides = {} - for feature_flag in feature_flags: - try: - match = feature_flag.matches(distinct_id, groups, cache, overrides) - if match: - flags_enabled[feature_flag.key] = match.variant or True - except Exception as err: - capture_exception(err) - return flags_enabled + return FeatureFlagMatcher(feature_flags, distinct_id, groups, cache, overrides).get_matches() # Return feature flags def get_active_feature_flags( team_id: int, distinct_id: str, groups: Dict[GroupTypeName, str] = {}, hash_key_override: Optional[str] = None -) -> Dict[str, Union[bool, str, None]]: +) -> Dict[str, Union[bool, str]]: all_feature_flags = FeatureFlag.objects.filter(team_id=team_id, active=True, deleted=False).only( "id", "team_id", "filters", "key", "rollout_percentage", "ensure_experience_continuity" @@ -346,7 +361,7 @@ def get_active_feature_flags( ) if not flags_have_experience_continuity_enabled: - return _get_active_feature_flags(all_feature_flags, team_id, distinct_id, groups=groups) + return _get_active_feature_flags(list(all_feature_flags), team_id, distinct_id, groups=groups) person_id = ( PersonDistinctId.objects.filter(distinct_id=distinct_id, team_id=team_id) @@ -380,7 +395,7 @@ def get_active_feature_flags( # as overrides are stored on personIDs. # We can optimise by not going down this path when person_id doesn't exist, or # no flags have experience continuity enabled - return _get_active_feature_flags(all_feature_flags, team_id, distinct_id, person_id, groups=groups) + return _get_active_feature_flags(list(all_feature_flags), team_id, distinct_id, person_id, groups=groups) def set_feature_flag_hash_key_overrides( diff --git a/posthog/models/filters/test/test_filter.py b/posthog/models/filters/test/test_filter.py index 2484f15561a0ba..27d81b1dab5cac 100644 --- a/posthog/models/filters/test/test_filter.py +++ b/posthog/models/filters/test/test_filter.py @@ -357,11 +357,12 @@ def test_person_cohort_properties(self): filter = Filter(data={"properties": [{"key": "id", "value": cohort1.pk, "type": "cohort"}],}) - matched_person = ( - Person.objects.filter(team_id=self.team.pk, persondistinctid__distinct_id=person1_distinct_id) - .filter(properties_to_Q(filter.property_groups.flat, team_id=self.team.pk, is_direct_query=True)) - .exists() - ) + with self.assertNumQueries(1): + matched_person = ( + Person.objects.filter(team_id=self.team.pk, persondistinctid__distinct_id=person1_distinct_id) + .filter(properties_to_Q(filter.property_groups.flat, team_id=self.team.pk, is_direct_query=True)) + .exists() + ) self.assertTrue(matched_person) def test_group_property_filters_direct(self): diff --git a/posthog/models/property/property.py b/posthog/models/property/property.py index e089d3572db574..ae26b77a3989d6 100644 --- a/posthog/models/property/property.py +++ b/posthog/models/property/property.py @@ -11,7 +11,7 @@ cast, ) -from django.db.models import Exists, OuterRef, Q +from django.db.models import Exists, F, OuterRef, Q, Subquery from posthog.constants import PropertyOperatorType from posthog.models.filters.mixins.utils import cached_property @@ -263,12 +263,18 @@ def property_to_Q(self) -> Q: cohort_id = int(cast(Union[str, int], value)) - cohort = Cohort.objects.only("version").get(pk=cohort_id) + cohort = Cohort.objects.only("version").filter(pk=cohort_id) return Q( Exists( - CohortPeople.objects.filter( - cohort_id=cohort.pk, person_id=OuterRef("id"), version=cohort.version - ).only("id") + CohortPeople.objects.annotate(cohort_version=Subquery(cohort.values("version")[:1])) + .filter(cohort_id=cohort_id, person_id=OuterRef("id"), cohort__id=cohort_id) + .filter( + # bit of a hack. if cohort_version is NULL, the query is still `version = cohort_version`, which doesn't match + # So just explicitly ask if cohort_version is null + Q(cohort_version=F("version")) + | Q(cohort_version__isnull=True) + ) + .only("id") ) ) diff --git a/posthog/test/__snapshots__/test_feature_flag.ambr b/posthog/test/__snapshots__/test_feature_flag.ambr index 5c9bb91ee490be..c74a4fe6587816 100644 --- a/posthog/test/__snapshots__/test_feature_flag.ambr +++ b/posthog/test/__snapshots__/test_feature_flag.ambr @@ -1,148 +1,64 @@ -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override - ' - SELECT "posthog_featureflag"."id", - "posthog_featureflag"."key", - "posthog_featureflag"."filters", - "posthog_featureflag"."rollout_percentage", - "posthog_featureflag"."team_id", - "posthog_featureflag"."ensure_experience_continuity" - FROM "posthog_featureflag" - WHERE ("posthog_featureflag"."active" - AND NOT "posthog_featureflag"."deleted" - AND "posthog_featureflag"."team_id" = 2) - ' ---- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.1 - ' - SELECT "posthog_persondistinctid"."person_id" - FROM "posthog_persondistinctid" - WHERE ("posthog_persondistinctid"."distinct_id" = 'other_id' - AND "posthog_persondistinctid"."team_id" = 2) +# name: TestFeatureFlagMatcher.test_multiple_flags + ' + SELECT "posthog_team"."id", + "posthog_team"."uuid", + "posthog_team"."organization_id", + "posthog_team"."api_token", + "posthog_team"."app_urls", + "posthog_team"."name", + "posthog_team"."slack_incoming_webhook", + "posthog_team"."created_at", + "posthog_team"."updated_at", + "posthog_team"."anonymize_ips", + "posthog_team"."completed_snippet_onboarding", + "posthog_team"."ingested_event", + "posthog_team"."session_recording_opt_in", + "posthog_team"."signup_token", + "posthog_team"."is_demo", + "posthog_team"."access_control", + "posthog_team"."test_account_filters", + "posthog_team"."path_cleaning_filters", + "posthog_team"."timezone", + "posthog_team"."data_attributes", + "posthog_team"."person_display_name_properties", + "posthog_team"."live_events_columns", + "posthog_team"."primary_dashboard_id", + "posthog_team"."correlation_config", + "posthog_team"."session_recording_retention_period_days", + "posthog_team"."plugins_opt_in", + "posthog_team"."opt_out_capture", + "posthog_team"."event_names", + "posthog_team"."event_names_with_usage", + "posthog_team"."event_properties", + "posthog_team"."event_properties_with_usage", + "posthog_team"."event_properties_numerical" + FROM "posthog_team" + WHERE "posthog_team"."id" = 2 + LIMIT 21 + ' +--- +# name: TestFeatureFlagMatcher.test_multiple_flags.1 + ' + SELECT "posthog_asyncmigration"."id", + "posthog_asyncmigration"."name", + "posthog_asyncmigration"."description", + "posthog_asyncmigration"."progress", + "posthog_asyncmigration"."status", + "posthog_asyncmigration"."current_operation_index", + "posthog_asyncmigration"."current_query_id", + "posthog_asyncmigration"."celery_task_id", + "posthog_asyncmigration"."started_at", + "posthog_asyncmigration"."finished_at", + "posthog_asyncmigration"."posthog_min_version", + "posthog_asyncmigration"."posthog_max_version" + FROM "posthog_asyncmigration" + WHERE ("posthog_asyncmigration"."name" = '0003_fill_person_distinct_id2' + AND "posthog_asyncmigration"."status" = 2) + ORDER BY "posthog_asyncmigration"."id" ASC LIMIT 1 ' --- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.2 - ' - SELECT "posthog_persondistinctid"."person_id" - FROM "posthog_persondistinctid" - WHERE ("posthog_persondistinctid"."distinct_id" = 'example_id' - AND "posthog_persondistinctid"."team_id" = 2) - ORDER BY "posthog_persondistinctid"."id" ASC - LIMIT 1 - ' ---- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.3 - ' - SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key" - FROM "posthog_featureflaghashkeyoverride" - WHERE ("posthog_featureflaghashkeyoverride"."person_id" = 2 - AND "posthog_featureflaghashkeyoverride"."team_id" = 2) - ' ---- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.4 - ' - INSERT INTO "posthog_featureflaghashkeyoverride" ("feature_flag_key", - "person_id", - "team_id", - "hash_key") - VALUES ('beta-feature', 2925, 763, 'example_id'), - ('multivariate-flag', 2925, 763, 'example_id') RETURNING "posthog_featureflaghashkeyoverride"."id" - ' ---- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.5 - ' - SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key", - "posthog_featureflaghashkeyoverride"."hash_key" - FROM "posthog_featureflaghashkeyoverride" - WHERE ("posthog_featureflaghashkeyoverride"."person_id" = 2 - AND "posthog_featureflaghashkeyoverride"."team_id" = 2) - ' ---- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.6 - ' - SELECT "posthog_featureflagoverride"."id", - "posthog_featureflagoverride"."feature_flag_id", - "posthog_featureflagoverride"."override_value", - "posthog_featureflag"."id", - "posthog_featureflag"."key" - FROM "posthog_featureflagoverride" - INNER JOIN "posthog_featureflag" ON ("posthog_featureflagoverride"."feature_flag_id" = "posthog_featureflag"."id") - WHERE ("posthog_featureflagoverride"."team_id" = 2 - AND "posthog_featureflagoverride"."user_id" IN - (SELECT W0."id" - FROM "posthog_user" W0 - WHERE W0."distinct_id" IN - (SELECT V0."distinct_id" - FROM "posthog_persondistinctid" V0 - WHERE V0."person_id" IN - (SELECT U0."person_id" - FROM "posthog_persondistinctid" U0 - WHERE (U0."distinct_id" = 'other_id' - AND U0."team_id" = 2) - LIMIT 1)) - LIMIT 1)) - ' ---- -# name: TestFeatureFlagHashKeyOverrides.test_entire_flow_with_hash_key_override.7 - ' - SELECT "posthog_featureflagoverride"."id", - "posthog_featureflagoverride"."feature_flag_id", - "posthog_featureflagoverride"."override_value", - "posthog_featureflag"."id", - "posthog_featureflag"."key" - FROM "posthog_featureflagoverride" - INNER JOIN "posthog_featureflag" ON ("posthog_featureflagoverride"."feature_flag_id" = "posthog_featureflag"."id") - WHERE ("posthog_featureflagoverride"."team_id" = 2 - AND "posthog_featureflagoverride"."user_id" IN - (SELECT W0."id" - FROM "posthog_user" W0 - WHERE W0."distinct_id" IN - (SELECT V0."distinct_id" - FROM "posthog_persondistinctid" V0 - WHERE V0."person_id" IN - (SELECT U0."person_id" - FROM "posthog_persondistinctid" U0 - WHERE (U0."distinct_id" = 'other_id' - AND U0."team_id" = 2) - LIMIT 1)) - LIMIT 1)) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_group_flags_with_overrides - ' - SELECT "posthog_featureflag"."id", - "posthog_featureflag"."key", - "posthog_featureflag"."filters", - "posthog_featureflag"."rollout_percentage", - "posthog_featureflag"."team_id", - "posthog_featureflag"."ensure_experience_continuity" - FROM "posthog_featureflag" - WHERE ("posthog_featureflag"."active" - AND NOT "posthog_featureflag"."deleted" - AND "posthog_featureflag"."team_id" = 2) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_group_flags_with_overrides.1 - ' - SELECT UPPER(("posthog_person"."properties" ->> 'email')::text) LIKE UPPER('%posthog.com%') AS "condition_0" - FROM "posthog_person" - INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") - WHERE ("posthog_persondistinctid"."distinct_id" = 'distinct_id' - AND "posthog_persondistinctid"."team_id" = 2 - AND "posthog_person"."team_id" = 2) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_group_flags_with_overrides.2 - ' - SELECT ("posthog_person"."properties" -> 'email') = '"tim@posthog.com"' AS "condition_0" - FROM "posthog_person" - INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") - WHERE ("posthog_persondistinctid"."distinct_id" = 'distinct_id' - AND "posthog_persondistinctid"."team_id" = 2 - AND "posthog_person"."team_id" = 2) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_group_flags_with_overrides.3 +# name: TestFeatureFlagMatcher.test_multiple_flags.2 ' SELECT "posthog_grouptypemapping"."id", "posthog_grouptypemapping"."team_id", @@ -154,108 +70,30 @@ WHERE "posthog_grouptypemapping"."team_id" = 2 ' --- -# name: TestFeatureFlagsWithOverrides.test_group_flags_with_overrides.4 - ' - SELECT ("posthog_group"."group_properties" -> 'name') = '"foo.inc"' AS "condition_0" - FROM "posthog_group" - WHERE ("posthog_group"."group_key" = 'PostHog' - AND "posthog_group"."group_type_index" = 2 - AND "posthog_group"."team_id" = 2) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_group_flags_with_overrides.5 - ' - SELECT "posthog_featureflagoverride"."id", - "posthog_featureflagoverride"."feature_flag_id", - "posthog_featureflagoverride"."override_value", - "posthog_featureflag"."id", - "posthog_featureflag"."key" - FROM "posthog_featureflagoverride" - INNER JOIN "posthog_featureflag" ON ("posthog_featureflagoverride"."feature_flag_id" = "posthog_featureflag"."id") - WHERE ("posthog_featureflagoverride"."team_id" = 2 - AND "posthog_featureflagoverride"."user_id" IN - (SELECT W0."id" - FROM "posthog_user" W0 - WHERE W0."distinct_id" IN - (SELECT V0."distinct_id" - FROM "posthog_persondistinctid" V0 - WHERE V0."person_id" IN - (SELECT U0."person_id" - FROM "posthog_persondistinctid" U0 - WHERE (U0."distinct_id" = 'distinct_id' - AND U0."team_id" = 2) - LIMIT 1)) - LIMIT 1)) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_person_flags_with_overrides - ' - SELECT "posthog_featureflag"."id", - "posthog_featureflag"."key", - "posthog_featureflag"."filters", - "posthog_featureflag"."rollout_percentage", - "posthog_featureflag"."team_id", - "posthog_featureflag"."ensure_experience_continuity" - FROM "posthog_featureflag" - WHERE ("posthog_featureflag"."active" - AND NOT "posthog_featureflag"."deleted" - AND "posthog_featureflag"."team_id" = 2) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_person_flags_with_overrides.1 - ' - SELECT UPPER(("posthog_person"."properties" ->> 'email')::text) LIKE UPPER('%posthog.com%') AS "condition_0" - FROM "posthog_person" - INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") - WHERE ("posthog_persondistinctid"."distinct_id" = 'distinct_id' - AND "posthog_persondistinctid"."team_id" = 2 - AND "posthog_person"."team_id" = 2) - ' ---- -# name: TestFeatureFlagsWithOverrides.test_person_flags_with_overrides.2 +# name: TestFeatureFlagMatcher.test_multiple_flags.3 ' - SELECT ("posthog_person"."properties" -> 'email') = '"tim@posthog.com"' AS "condition_0" + SELECT ("posthog_person"."properties" -> 'email') = '"test@posthog.com"' AS "flag_87_condition_0", + (true) AS "flag_87_condition_1", + (true) AS "flag_88_condition_0", + (true) AS "flag_89_condition_0", + (true) AS "flag_93_condition_0" FROM "posthog_person" INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") - WHERE ("posthog_persondistinctid"."distinct_id" = 'distinct_id' + WHERE ("posthog_persondistinctid"."distinct_id" = 'test_id' AND "posthog_persondistinctid"."team_id" = 2 AND "posthog_person"."team_id" = 2) ' --- -# name: TestFeatureFlagsWithOverrides.test_person_flags_with_overrides.3 +# name: TestFeatureFlagMatcher.test_multiple_flags.4 ' - SELECT "posthog_grouptypemapping"."id", - "posthog_grouptypemapping"."team_id", - "posthog_grouptypemapping"."group_type", - "posthog_grouptypemapping"."group_type_index", - "posthog_grouptypemapping"."name_singular", - "posthog_grouptypemapping"."name_plural" - FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 2 - ' ---- -# name: TestFeatureFlagsWithOverrides.test_person_flags_with_overrides.4 - ' - SELECT "posthog_featureflagoverride"."id", - "posthog_featureflagoverride"."feature_flag_id", - "posthog_featureflagoverride"."override_value", - "posthog_featureflag"."id", - "posthog_featureflag"."key" - FROM "posthog_featureflagoverride" - INNER JOIN "posthog_featureflag" ON ("posthog_featureflagoverride"."feature_flag_id" = "posthog_featureflag"."id") - WHERE ("posthog_featureflagoverride"."team_id" = 2 - AND "posthog_featureflagoverride"."user_id" IN - (SELECT W0."id" - FROM "posthog_user" W0 - WHERE W0."distinct_id" IN - (SELECT V0."distinct_id" - FROM "posthog_persondistinctid" V0 - WHERE V0."person_id" IN - (SELECT U0."person_id" - FROM "posthog_persondistinctid" U0 - WHERE (U0."distinct_id" = 'distinct_id' - AND U0."team_id" = 2) - LIMIT 1)) - LIMIT 1)) + SELECT ("posthog_group"."group_key" = 'group_key' + AND "posthog_group"."group_type_index" = 2) AS "flag_90_condition_0", + ("posthog_group"."group_key" = 'group_key' + AND "posthog_group"."group_type_index" = 2) AS "flag_91_condition_0", + (("posthog_group"."group_properties" -> 'name') IN ('"foo.inc"') + AND "posthog_group"."group_key" = 'foo' + AND "posthog_group"."group_type_index" = 2) AS "flag_92_condition_0" + FROM "posthog_group" + WHERE "posthog_group"."team_id" = 2 ' --- diff --git a/posthog/test/test_feature_flag.py b/posthog/test/test_feature_flag.py index 41f1278ba4ea4e..d4420311a71031 100644 --- a/posthog/test/test_feature_flag.py +++ b/posthog/test/test_feature_flag.py @@ -5,38 +5,39 @@ FeatureFlagHashKeyOverride, FeatureFlagMatch, FeatureFlagMatcher, + FlagsMatcherCache, get_active_feature_flags, hash_key_overrides, set_feature_flag_hash_key_overrides, ) from posthog.models.group import Group -from posthog.test.base import BaseTest, QueryMatchingTest +from posthog.test.base import BaseTest, QueryMatchingTest, snapshot_postgres_queries -class TestFeatureFlagMatcher(BaseTest): +class TestFeatureFlagMatcher(BaseTest, QueryMatchingTest): def test_blank_flag(self): # Blank feature flags now default to be released for everyone feature_flag = self.create_feature_flag() - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertEqual(FeatureFlagMatcher(feature_flag, "another_id").get_match(), FeatureFlagMatch()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag), FeatureFlagMatch()) def test_rollout_percentage(self): feature_flag = self.create_feature_flag(filters={"groups": [{"rollout_percentage": 50}]}) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) def test_empty_group(self): feature_flag = self.create_feature_flag(filters={"groups": [{}]}) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertEqual(FeatureFlagMatcher(feature_flag, "another_id").get_match(), FeatureFlagMatch()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag), FeatureFlagMatch()) def test_null_rollout_percentage(self): feature_flag = self.create_feature_flag(filters={"groups": [{"properties": [], "rollout_percentage": None}]}) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) def test_zero_rollout_percentage(self): feature_flag = self.create_feature_flag(filters={"groups": [{"properties": [], "rollout_percentage": 0}]}) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), None) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), None) def test_complicated_flag(self): Person.objects.create( @@ -57,9 +58,106 @@ def test_complicated_flag(self): } ) - self.assertEqual(FeatureFlagMatcher(feature_flag, "test_id").get_match(), FeatureFlagMatch()) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "test_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) + + @snapshot_postgres_queries + def test_multiple_flags(self): + Person.objects.create( + team=self.team, distinct_ids=["test_id"], properties={"email": "test@posthog.com"}, + ) + self.create_groups() + feature_flag_one = self.create_feature_flag( + filters={ + "groups": [ + { + "properties": [ + {"key": "email", "type": "person", "value": "test@posthog.com", "operator": "exact"} + ], + "rollout_percentage": 100, + }, + {"rollout_percentage": 50}, + ] + }, + key="one", + ) + feature_flag_always_match = self.create_feature_flag( + filters={"groups": [{"rollout_percentage": 100}]}, key="always_match" + ) + feature_flag_never_match = self.create_feature_flag( + filters={"groups": [{"rollout_percentage": 0}]}, key="never_match" + ) + feature_flag_group_match = self.create_feature_flag( + filters={"aggregation_group_type_index": 1, "groups": [{"rollout_percentage": 100}],}, key="group_match" + ) + feature_flag_group_no_match = self.create_feature_flag( + filters={"aggregation_group_type_index": 1, "groups": [{"rollout_percentage": 0}],}, key="group_no_match" + ) + feature_flag_group_property_match = self.create_feature_flag( + filters={ + "aggregation_group_type_index": 0, + "groups": [ + { + "rollout_percentage": 100, + "properties": [ + { + "key": "name", + "value": ["foo.inc"], + "operator": "exact", + "type": "group", + "group_type_index": 0, + } + ], + } + ], + }, + key="group_property_match", + ) + feature_flag_variant = self.create_feature_flag( + filters={ + "groups": [{"properties": [], "rollout_percentage": None}], + "multivariate": { + "variants": [ + {"key": "first-variant", "name": "First Variant", "rollout_percentage": 50}, + {"key": "second-variant", "name": "Second Variant", "rollout_percentage": 25}, + {"key": "third-variant", "name": "Third Variant", "rollout_percentage": 25}, + ], + }, + }, + key="variant", + ) + + with self.assertNumQueries( + 3 + ): # 1 to fill group cache, 1 to match feature flags with group properties, 1 to match feature flags with person properties + + matches = FeatureFlagMatcher( + [ + feature_flag_one, + feature_flag_always_match, + feature_flag_never_match, + feature_flag_group_match, + feature_flag_group_no_match, + feature_flag_variant, + feature_flag_group_property_match, + ], + "test_id", + {"project": "group_key", "organization": "foo"}, + FlagsMatcherCache(self.team.id), + ).get_matches() + + self.assertEqual( + matches, + { + "one": True, + "always_match": True, + "group_match": True, + "variant": "first-variant", + "group_property_match": True, + # never_match and group_no_match don't match + }, + ) def test_multi_property_filters(self): Person.objects.create( @@ -77,10 +175,14 @@ def test_multi_property_filters(self): } ) with self.assertNumQueries(1): - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch() + ) with self.assertNumQueries(1): - self.assertEqual(FeatureFlagMatcher(feature_flag, "another_id").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "false_id").get_match()) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag), FeatureFlagMatch() + ) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "false_id").get_match(feature_flag)) def test_user_in_cohort(self): Person.objects.create(team=self.team, distinct_ids=["example_id_1"], properties={"$some_prop_1": "something_1"}) @@ -97,13 +199,13 @@ def test_user_in_cohort(self): feature_flag.update_cohorts() - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id_1").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id_1").get_match(feature_flag), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) def test_legacy_rollout_percentage(self): feature_flag = self.create_feature_flag(rollout_percentage=50) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) def test_legacy_property_filters(self): Person.objects.create( @@ -113,8 +215,8 @@ def test_legacy_property_filters(self): team=self.team, distinct_ids=["another_id"], properties={"email": "example@example.com"}, ) feature_flag = self.create_feature_flag(filters={"properties": [{"key": "email", "value": "tim@posthog.com"}]},) - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) def test_legacy_rollout_and_property_filter(self): Person.objects.create( @@ -131,9 +233,11 @@ def test_legacy_rollout_and_property_filter(self): filters={"properties": [{"key": "email", "value": "tim@posthog.com", "type": "person"}]}, ) with self.assertNumQueries(1): - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "id_number_3").get_match()) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), FeatureFlagMatch() + ) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "id_number_3").get_match(feature_flag)) def test_legacy_user_in_cohort(self): Person.objects.create(team=self.team, distinct_ids=["example_id_2"], properties={"$some_prop_2": "something_2"}) @@ -150,8 +254,8 @@ def test_legacy_user_in_cohort(self): feature_flag.update_cohorts() - self.assertEqual(FeatureFlagMatcher(feature_flag, "example_id_2").get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "another_id").get_match()) + self.assertEqual(FeatureFlagMatcher([feature_flag], "example_id_2").get_match(feature_flag), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "another_id").get_match(feature_flag)) def test_variants(self): feature_flag = self.create_feature_flag( @@ -167,11 +271,16 @@ def test_variants(self): } ) - self.assertEqual(FeatureFlagMatcher(feature_flag, "11").get_match(), FeatureFlagMatch(variant="first-variant")) self.assertEqual( - FeatureFlagMatcher(feature_flag, "example_id").get_match(), FeatureFlagMatch(variant="second-variant") + FeatureFlagMatcher([feature_flag], "11").get_match(feature_flag), FeatureFlagMatch(variant="first-variant") + ) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "example_id").get_match(feature_flag), + FeatureFlagMatch(variant="second-variant"), + ) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "3").get_match(feature_flag), FeatureFlagMatch(variant="third-variant") ) - self.assertEqual(FeatureFlagMatcher(feature_flag, "3").get_match(), FeatureFlagMatch(variant="third-variant")) def test_flag_by_groups_with_rollout_100(self): self.create_groups() @@ -179,10 +288,12 @@ def test_flag_by_groups_with_rollout_100(self): filters={"aggregation_group_type_index": 1, "groups": [{"rollout_percentage": 100}],} ) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "").get_match()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "", {"unknown": "group_key"}).get_match()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "", {"organization": "group_key"}).get_match()) - self.assertEqual(FeatureFlagMatcher(feature_flag, "", {"project": "group_key"}).get_match(), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "").get_match(feature_flag)) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "", {"unknown": "group_key"}).get_match(feature_flag)) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "", {"organization": "group_key"}).get_match(feature_flag)) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "", {"project": "group_key"}).get_match(feature_flag), FeatureFlagMatch() + ) def test_flag_by_groups_with_rollout_50(self): self.create_groups() @@ -190,8 +301,10 @@ def test_flag_by_groups_with_rollout_50(self): filters={"aggregation_group_type_index": 1, "groups": [{"rollout_percentage": 50}],} ) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "", {"project": "1"}).get_match()) - self.assertEqual(FeatureFlagMatcher(feature_flag, "", {"project": "4"}).get_match(), FeatureFlagMatch()) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "", {"project": "1"}).get_match(feature_flag)) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "", {"project": "4"}).get_match(feature_flag), FeatureFlagMatch() + ) def test_flag_by_group_properties(self): self.create_groups() @@ -199,12 +312,14 @@ def test_flag_by_group_properties(self): filters={ "aggregation_group_type_index": 0, "groups": [ - {"properties": [{"key": "name", "value": "foo.inc", "type": "group", "group_type_index": 0}],} + {"properties": [{"key": "name", "value": ["foo.inc"], "type": "group", "group_type_index": 0}],} ], } ) - self.assertEqual(FeatureFlagMatcher(feature_flag, "", {"organization": "foo"}).get_match(), FeatureFlagMatch()) - self.assertIsNone(FeatureFlagMatcher(feature_flag, "", {"organization": "bar"}).get_match()) + self.assertEqual( + FeatureFlagMatcher([feature_flag], "", {"organization": "foo"}).get_match(feature_flag), FeatureFlagMatch() + ) + self.assertIsNone(FeatureFlagMatcher([feature_flag], "", {"organization": "bar"}).get_match(feature_flag)) def create_groups(self): GroupTypeMapping.objects.create(team=self.team, group_type="organization", group_type_index=0) @@ -215,12 +330,13 @@ def create_groups(self): Group.objects.create( team=self.team, group_type_index=0, group_key="bar", group_properties={"name": "var.inc"}, version=1 ) - - def create_feature_flag(self, **kwargs): - return FeatureFlag.objects.create( - team=self.team, name="Beta feature", key="beta-feature", created_by=self.user, **kwargs + Group.objects.create( + team=self.team, group_type_index=1, group_key="group_key", group_properties={"name": "var.inc"}, version=1 ) + def create_feature_flag(self, key="beta-feature", **kwargs): + return FeatureFlag.objects.create(team=self.team, name="Beta feature", key=key, created_by=self.user, **kwargs) + class TestFeatureFlagHashKeyOverrides(BaseTest, QueryMatchingTest): From d531d595c3de024a8c60c301789adf6d7549d313 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 15:45:18 +0100 Subject: [PATCH 073/213] chore: turn the celery sentry trace sample rate down (#10814) --- posthog/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/celery.py b/posthog/celery.py index 9bc6b8777344a4..16463ac473b122 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -58,7 +58,7 @@ def receiver_setup_logging(loglevel, logfile, format, colorize, **kwargs) -> Non def on_worker_start(**kwargs) -> None: from posthog.settings import sentry_init - sentry_init(traces_sample_rate=0.5) + sentry_init(traces_sample_rate=0.05) @app.on_after_configure.connect From 16e0979d7f265c3546ab99854bab07e9c1a56121 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 15:56:32 +0100 Subject: [PATCH 074/213] chore: limit live events CSV export to 3500 events (#10806) --- frontend/src/scenes/events/EventsTable.tsx | 19 +++++++++++++++---- posthog/tasks/exports/csv_exporter.py | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/frontend/src/scenes/events/EventsTable.tsx b/frontend/src/scenes/events/EventsTable.tsx index 0280b90912a0ee..eb58ace04b4c43 100644 --- a/frontend/src/scenes/events/EventsTable.tsx +++ b/frontend/src/scenes/events/EventsTable.tsx @@ -2,7 +2,7 @@ import React, { useMemo } from 'react' import { useActions, useValues } from 'kea' import { EventDetails } from 'scenes/events/EventDetails' import { Link } from 'lib/components/Link' -import { Button } from 'antd' +import { Button, Popconfirm } from 'antd' import { FilterPropertyLink } from 'lib/components/FilterPropertyLink' import { Property } from 'lib/components/Property' import { autoCaptureEventToDescription } from 'lib/utils' @@ -448,15 +448,26 @@ export function EventsTable({ /> )} {showExport && exportUrl && ( - + + Exporting by csv is limited to 3,500 events. +
+ To return more, please use{' '} + the API. Do you want to + export by CSV? + + } + onConfirm={startDownload} + > } - onClick={startDownload} > Export - + )} diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index d8e1c53fd2dd6f..7385fe4b50c054 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -130,7 +130,7 @@ def _convert_response_to_csv_data(data: Any) -> List[Any]: return [] -def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: int = 10_000,) -> None: +def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: int = 3_500,) -> None: resource = exported_asset.export_context path: str = resource["path"] @@ -180,7 +180,7 @@ def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: @timed("csv_exporter") -def export_csv(exported_asset: ExportedAsset, limit: Optional[int] = None, max_limit: int = 10_000,) -> None: +def export_csv(exported_asset: ExportedAsset, limit: Optional[int] = None, max_limit: int = 3_500,) -> None: if not limit: limit = 1000 From da427d9a84879406c00c8ba2f44a4837f311f333 Mon Sep 17 00:00:00 2001 From: timgl Date: Fri, 15 Jul 2022 17:19:26 +0200 Subject: [PATCH 075/213] chore: Release property filter on dashboard (#10825) --- frontend/src/lib/constants.tsx | 1 - frontend/src/scenes/dashboard/Dashboard.tsx | 18 ++++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index a9484312802b25..164cad87518a39 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -110,7 +110,6 @@ export const FEATURE_FLAGS = { MULTI_DASHBOARD_INSIGHTS: 'multi-dashboard-insights', // owner: @pauldambra INSIGHT_ACTIVITY_LOG: '8545-insight-activity-log', // owner: @pauldambra FRONTEND_APPS: '9618-frontend-apps', // owner: @mariusandra - PROPERTY_FILTER_ON_DASHBOARD: 'property-filter-on-dashboard', // owner: @edscode EXPORT_DASHBOARD_INSIGHTS: 'export-dashboard-insights', // owner: @benjackwhite ONBOARDING_1_5: 'onboarding-1_5', // owner: @liyiy BREAKDOWN_ATTRIBUTION: 'breakdown-attribution', // owner: @neilkakkar diff --git a/frontend/src/scenes/dashboard/Dashboard.tsx b/frontend/src/scenes/dashboard/Dashboard.tsx index 97620994d87cc2..a40cc391d75915 100644 --- a/frontend/src/scenes/dashboard/Dashboard.tsx +++ b/frontend/src/scenes/dashboard/Dashboard.tsx @@ -16,8 +16,6 @@ import { SceneExport } from 'scenes/sceneTypes' import { InsightErrorState } from 'scenes/insights/EmptyStates' import { DashboardHeader } from './DashboardHeader' import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import { FEATURE_FLAGS } from 'lib/constants' interface Props { id?: string @@ -54,7 +52,6 @@ function DashboardScene(): JSX.Element { receivedErrorsFromAPI, } = useValues(dashboardLogic) const { setDashboardMode, setDates, reportDashboardViewed, setProperties } = useActions(dashboardLogic) - const { featureFlags } = useValues(featureFlagLogic) useEffect(() => { reportDashboardViewed() @@ -131,15 +128,12 @@ function DashboardScene(): JSX.Element { )} /> - {(featureFlags[FEATURE_FLAGS.PROPERTY_FILTER_ON_DASHBOARD] || - dashboard?.filters.properties) && ( - - )} + )} {placement !== DashboardPlacement.Export && ( From 144e1ba056842ad566b65855312786994bc15a3b Mon Sep 17 00:00:00 2001 From: Ben White Date: Fri, 15 Jul 2022 17:31:29 +0200 Subject: [PATCH 076/213] fix: Export endpoint content location (#10826) --- posthog/api/exports.py | 1 + posthog/api/sharing.py | 3 --- posthog/models/exported_asset.py | 4 ++++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/posthog/api/exports.py b/posthog/api/exports.py index f7b34ced641836..646e00af54c2cc 100644 --- a/posthog/api/exports.py +++ b/posthog/api/exports.py @@ -122,6 +122,7 @@ class ExportedAssetViewSet( ] permission_classes = [IsAuthenticated, ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission] + # TODO: This should be removed as it is only used by frontend exporter and can instead use the api/sharing.py endpoint @action(methods=["GET"], detail=True) def content(self, request: Request, *args: Any, **kwargs: Any) -> HttpResponse: instance = self.get_object() diff --git a/posthog/api/sharing.py b/posthog/api/sharing.py index 2f881d85341db3..432bcb949c6098 100644 --- a/posthog/api/sharing.py +++ b/posthog/api/sharing.py @@ -165,9 +165,6 @@ def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Any: if asset: if request.path.endswith(f".{asset.file_ext}"): - if not asset.content: - raise NotFound() - return get_content_response(asset, request.query_params.get("download") == "true") exported_data["type"] = "image" diff --git a/posthog/models/exported_asset.py b/posthog/models/exported_asset.py index 9764d870957cc4..6f2090cf4a1fcb 100644 --- a/posthog/models/exported_asset.py +++ b/posthog/models/exported_asset.py @@ -7,6 +7,7 @@ from django.db import models from django.http import HttpResponse from django.utils.text import slugify +from rest_framework.exceptions import NotFound from sentry_sdk import capture_exception from posthog.jwt import PosthogJwtAudience, decode_jwt, encode_jwt @@ -103,6 +104,9 @@ def get_content_response(asset: ExportedAsset, download: bool = False): if not content and asset.content_location: content = object_storage.read_bytes(asset.content_location) + if not content: + raise NotFound() + res = HttpResponse(content, content_type=asset.export_format) if download: res["Content-Disposition"] = f'attachment; filename="{asset.filename}"' From 88fe512e780c6fc8f981c5cd36d136722b24865e Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Fri, 15 Jul 2022 17:08:58 +0100 Subject: [PATCH 077/213] chore: Revert disabling optimize move to prewhere (#10766) --- posthog/client.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/posthog/client.py b/posthog/client.py index ab65596ef76bad..af8012e454864e 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -55,12 +55,6 @@ _request_information: Optional[Dict] = None -# Optimize_move_to_prewhere setting is set because of this regression test -# test_ilike_regression_with_current_clickhouse_version -# https://github.com/PostHog/posthog/blob/master/ee/clickhouse/queries/test/test_trends.py#L1566 -settings_override = {"optimize_move_to_prewhere": 0} - - def default_client(): """ Return a bare bones client for use in places where we are only interested in general ClickHouse state @@ -150,8 +144,6 @@ def sync_execute(query, args=None, settings=None, with_column_types=False, flush timeout_task = QUERY_TIMEOUT_THREAD.schedule(_notify_of_slow_query_failure, tags) - settings = {**settings_override, **(settings or {})} - try: result = client.execute( prepared_sql, params=prepared_args, settings=settings, with_column_types=with_column_types, From 353b5a41a521f777299d812800d0f3d623b2480c Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Fri, 15 Jul 2022 17:34:44 +0100 Subject: [PATCH 078/213] fix(my-flags): Ensure flags work for empty list (#10828) * fix(my-flags): Ensure flags work for empty list * fix mypy --- posthog/api/feature_flag.py | 12 +++++++++--- posthog/api/test/test_feature_flag.py | 10 ++++++++++ posthog/models/feature_flag.py | 5 ++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index 0326becddd54d3..a21f1521bafb22 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict, Optional, cast +from typing import Any, Dict, List, Optional, cast from django.db.models import QuerySet from rest_framework import authentication, exceptions, request, serializers, status, viewsets @@ -203,8 +203,14 @@ def my_flags(self, request: request.Request, **kwargs): .order_by("-created_at") ) groups = json.loads(request.GET.get("groups", "{}")) - flags = [] - matches = FeatureFlagMatcher(list(feature_flags), request.user.distinct_id, groups).get_matches() + flags: List[dict] = [] + + feature_flag_list = list(feature_flags) + + if not feature_flag_list: + return Response(flags) + + matches = FeatureFlagMatcher(feature_flag_list, request.user.distinct_id, groups).get_matches() for feature_flag in feature_flags: flags.append( { diff --git a/posthog/api/test/test_feature_flag.py b/posthog/api/test/test_feature_flag.py index 07d8c588bdcb4a..dee14659c0629e 100644 --- a/posthog/api/test/test_feature_flag.py +++ b/posthog/api/test/test_feature_flag.py @@ -876,6 +876,16 @@ def test_my_flags(self, mock_capture): self.assertEqual(first_flag["feature_flag"]["key"], "alpha-feature") self.assertEqual(first_flag["value"], False) + @patch("posthog.api.feature_flag.report_user_action") + def test_my_flags_empty_flags(self, mock_capture): + # Ensure empty feature flag list + FeatureFlag.objects.all().delete() + + response = self.client.get(f"/api/projects/{self.team.id}/feature_flags/my_flags") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_data = response.json() + self.assertEqual(len(response_data), 0) + @patch("posthoganalytics.capture") def test_my_flags_groups(self, mock_capture): self.client.post( diff --git a/posthog/models/feature_flag.py b/posthog/models/feature_flag.py index e902534a9162a7..d0334597874c81 100644 --- a/posthog/models/feature_flag.py +++ b/posthog/models/feature_flag.py @@ -344,7 +344,10 @@ def _get_active_feature_flags( else: overrides = {} - return FeatureFlagMatcher(feature_flags, distinct_id, groups, cache, overrides).get_matches() + if feature_flags: + return FeatureFlagMatcher(feature_flags, distinct_id, groups, cache, overrides).get_matches() + + return {} # Return feature flags From d56364dcdeaa5f217d52c99670864f253ba56731 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 15 Jul 2022 18:57:09 +0100 Subject: [PATCH 079/213] fix: response doesn't always have OK (#10831) --- posthog/tasks/exports/csv_exporter.py | 2 +- posthog/tasks/exports/test/test_csv_exporter.py | 1 + posthog/tasks/exports/test/test_csv_exporter_renders.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index 7385fe4b50c054..ab3d77d7c54014 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -151,7 +151,7 @@ def _export_to_csv(exported_asset: ExportedAsset, limit: int = 1000, max_limit: response = requests.request( method=method.lower(), url=url, json=body, headers={"Authorization": f"Bearer {access_token}"}, ) - if not response.ok: + if response.status_code != 200: raise Exception(f"export API call failed with status_code: {response.status_code}") # Figure out how to handle funnel polling.... diff --git a/posthog/tasks/exports/test/test_csv_exporter.py b/posthog/tasks/exports/test/test_csv_exporter.py index 842476f71cbde7..33142e6b305172 100644 --- a/posthog/tasks/exports/test/test_csv_exporter.py +++ b/posthog/tasks/exports/test/test_csv_exporter.py @@ -24,6 +24,7 @@ class TestCSVExporter(APIBaseTest): def patched_request(self): with patch("posthog.tasks.exports.csv_exporter.requests.request") as patched_request: mock_response = Mock() + mock_response.status_code = 200 # API responses copied from https://github.com/PostHog/posthog/runs/7221634689?check_suite_focus=true mock_response.json.side_effect = [ { diff --git a/posthog/tasks/exports/test/test_csv_exporter_renders.py b/posthog/tasks/exports/test/test_csv_exporter_renders.py index 4b9f3d7c5be195..a5fc2befc2e4bd 100644 --- a/posthog/tasks/exports/test/test_csv_exporter_renders.py +++ b/posthog/tasks/exports/test/test_csv_exporter_renders.py @@ -38,6 +38,7 @@ def test_csv_rendering(mock_settings, mock_request, filename): asset.save() mock = Mock() + mock.status_code = 200 mock.json.return_value = fixture["response"] mock_request.return_value = mock csv_exporter.export_csv(asset) From 93fb633e95a4108e25992553681e0adbffebc0c3 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 18 Jul 2022 07:56:21 +0100 Subject: [PATCH 080/213] chore: add regression test for reading assets from object storage (#10834) --- posthog/api/test/test_sharing.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/posthog/api/test/test_sharing.py b/posthog/api/test/test_sharing.py index d16d0c2b47690d..ad871c82a6316e 100644 --- a/posthog/api/test/test_sharing.py +++ b/posthog/api/test/test_sharing.py @@ -1,6 +1,9 @@ +from unittest.mock import patch + from freezegun import freeze_time from rest_framework import status +from posthog.models import ExportedAsset from posthog.models.dashboard import Dashboard from posthog.models.filters.filter import Filter from posthog.models.insight import Insight @@ -111,3 +114,29 @@ def test_should_not_get_deleted_item(self): response = self.client.patch(f"/api/projects/{self.team.id}/dashboards/{dashboard.id}", {"deleted": True}) response = self.client.get(f"/shared_dashboard/my_test_token") assert response.status_code == 404 + + @patch("posthog.models.exported_asset.object_storage.read_bytes") + @patch("posthog.api.sharing.asset_for_token") + def test_can_get_shared_dashboard_asset_with_no_content_but_content_location( + self, patched_asset_for_token, patched_object_storage + ) -> None: + asset = ExportedAsset.objects.create( + team_id=self.team.id, + export_format=ExportedAsset.ExportFormat.PNG, + content=None, + content_location="some object url", + ) + patched_asset_for_token.return_value = asset + + patched_object_storage.return_value = b"the image bytes" + + # pytest parameterize doesn't work in unittest.TestCase classes :'( + for url in [ + "/exporter/something.png?token=my_test_token", + "/shared_dashboard/something.png?token=my_test_token", + ]: + response = self.client.get(url) + + assert response.status_code == 200 + assert response.headers.get("Content-Type") == "image/png" + assert response.content == b"the image bytes" From 09c212cca9beb1019bc5e22c44fa40d2b310b529 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 18 Jul 2022 09:08:02 +0100 Subject: [PATCH 081/213] chore: measure the age of the dashboard queue (#10805) * chore: measure the age of the dashboard queue * tags so we know if the oldest item is changing * the same but working --- posthog/tasks/test/test_update_cache.py | 40 +++++++++++++++++++++++++ posthog/tasks/update_cache.py | 24 ++++++++++++++- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/posthog/tasks/test/test_update_cache.py b/posthog/tasks/test/test_update_cache.py index ec4231b3e07682..1d6f89c51d511e 100644 --- a/posthog/tasks/test/test_update_cache.py +++ b/posthog/tasks/test/test_update_cache.py @@ -738,6 +738,46 @@ def test_cache_key_that_matches_no_assets_still_counts_as_a_refresh_attempt_for_ insight.refresh_from_db() assert insight.refresh_attempt == 1 + @patch("posthog.tasks.update_cache.statsd.gauge") + def test_never_refreshed_tiles_are_gauged(self, statsd_gauge: MagicMock) -> None: + dashboard = create_shared_dashboard(team=self.team, is_shared=True) + filter = {"events": [{"id": "$pageview"}]} + item = Insight.objects.create(filters=filter, team=self.team) + tile: DashboardTile = DashboardTile.objects.create(insight=item, dashboard=dashboard) + + assert tile.last_refresh is None + + update_cached_items() + + statsd_gauge.assert_any_call("update_cache_queue.never_refreshed", 1) + + @freeze_time("2022-12-01T13:54:00.000Z") + @patch("posthog.tasks.update_cache.statsd.gauge") + def test_refresh_age_of_tiles_is_gauged(self, statsd_gauge: MagicMock) -> None: + tile_one = self._a_dashboard_tile_with_known_last_refresh(datetime.now(pytz.utc) - timedelta(hours=1)) + self._a_dashboard_tile_with_known_last_refresh(datetime.now(pytz.utc) - timedelta(hours=0.5)) + + update_cached_items() + + statsd_gauge.assert_any_call( + "update_cache_queue.dashboards_lag", + 3600, + tags={ + "insight_id": tile_one.insight_id, + "dashboard_id": tile_one.dashboard_id, + "cache_key": tile_one.filters_hash, + }, + ) + + def _a_dashboard_tile_with_known_last_refresh(self, last_refresh_date: datetime) -> DashboardTile: + dashboard = create_shared_dashboard(team=self.team, is_shared=True) + filter = {"events": [{"id": "$pageview"}]} + item = Insight.objects.create(filters=filter, team=self.team) + tile: DashboardTile = DashboardTile.objects.create(insight=item, dashboard=dashboard) + tile.last_refresh = last_refresh_date + tile.save(update_fields=["last_refresh"]) + return tile + def _create_insight_with_known_cache_key(self, test_hash: str) -> Insight: filter_dict: Dict[str, Any] = { "events": [{"id": "$pageview"}], diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index 31dd08584bc601..22d6c18a98163a 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -1,3 +1,4 @@ +import datetime import json import os from typing import Any, Dict, List, Optional, Tuple @@ -242,9 +243,30 @@ def update_cached_items() -> Tuple[int, int]: insight.save(update_fields=["refresh_attempt"]) capture_exception(e) - logger.info("Found {} items to refresh".format(len(tasks))) + statsd.gauge("update_cache_queue.never_refreshed", dashboard_tiles.filter(last_refresh=None).count()) + + # how old is the next to be refreshed + oldest_previously_refreshed_tile: Optional[DashboardTile] = dashboard_tiles.exclude(last_refresh=None).first() + if oldest_previously_refreshed_tile and oldest_previously_refreshed_tile.last_refresh: + dashboard_cache_age = ( + datetime.datetime.now(timezone.utc) - oldest_previously_refreshed_tile.last_refresh + ).total_seconds() + + statsd.gauge( + "update_cache_queue.dashboards_lag", + round(dashboard_cache_age), + tags={ + "insight_id": oldest_previously_refreshed_tile.insight_id, + "dashboard_id": oldest_previously_refreshed_tile.dashboard_id, + "cache_key": oldest_previously_refreshed_tile.filters_hash, + }, + ) + + logger.info("update_cache_queue", length=len(tasks)) taskset = group(tasks) taskset.apply_async() + + # this is the number of cacheable items that match the query queue_depth = dashboard_tiles.count() + shared_insights.count() statsd.gauge("update_cache_queue_depth.shared_insights", shared_insights.count()) statsd.gauge("update_cache_queue_depth.dashboards", dashboard_tiles.count()) From 118861462a280a3fd68f2f2836da5baae15b8670 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 18 Jul 2022 09:09:17 +0100 Subject: [PATCH 082/213] feat: allow exports to specify their max row limit (#10830) * feat: allow exports to specify their max row limit * There are more places that export 10000 rows --- frontend/src/models/cohortsModel.ts | 1 + frontend/src/scenes/events/eventsTableLogic.ts | 1 + frontend/src/scenes/persons/personsLogic.ts | 1 + frontend/src/scenes/retention/RetentionModal.tsx | 1 + frontend/src/types.ts | 1 + posthog/tasks/exporter.py | 7 +++++-- 6 files changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/models/cohortsModel.ts b/frontend/src/models/cohortsModel.ts index 1ebeb782fb1368..cf5ed85035424a 100644 --- a/frontend/src/models/cohortsModel.ts +++ b/frontend/src/models/cohortsModel.ts @@ -85,6 +85,7 @@ export const cohortsModel = kea({ export_format: ExporterFormat.CSV, export_context: { path: `/api/cohort/${id}/persons`, + max_limit: 10000, }, }) } else { diff --git a/frontend/src/scenes/events/eventsTableLogic.ts b/frontend/src/scenes/events/eventsTableLogic.ts index 1b05d086e5a0e7..f6af26a420e247 100644 --- a/frontend/src/scenes/events/eventsTableLogic.ts +++ b/frontend/src/scenes/events/eventsTableLogic.ts @@ -301,6 +301,7 @@ export const eventsTableLogic = kea({ export_format: ExporterFormat.CSV, export_context: { path: values.eventsUrl(), + max_limit: 3500, }, }) } else { diff --git a/frontend/src/scenes/persons/personsLogic.ts b/frontend/src/scenes/persons/personsLogic.ts index 907105fbcd1708..89a88271c6c782 100644 --- a/frontend/src/scenes/persons/personsLogic.ts +++ b/frontend/src/scenes/persons/personsLogic.ts @@ -152,6 +152,7 @@ export const personsLogic = kea({ export_format: ExporterFormat.CSV, export_context: { path: (cohort ? `/api/cohort/${cohort}/persons` : `api/person/`) + `?${toParams(listFilters)}`, + max_limit: 10000, }, }, ], diff --git a/frontend/src/scenes/retention/RetentionModal.tsx b/frontend/src/scenes/retention/RetentionModal.tsx index a94446e92df098..7392d4f9c1bc6b 100644 --- a/frontend/src/scenes/retention/RetentionModal.tsx +++ b/frontend/src/scenes/retention/RetentionModal.tsx @@ -61,6 +61,7 @@ export function RetentionModal({ export_format: ExporterFormat.CSV, export_context: { path: results[selectedRow]?.people_url, + max_limit: 10000, }, }) } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 53869af81730cb..0d15cfb4d75c49 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1984,6 +1984,7 @@ export interface ExportedAssetType { query?: any body?: any filename?: string + max_limit?: number } has_content: boolean filename: string diff --git a/posthog/tasks/exporter.py b/posthog/tasks/exporter.py index fe7bc305968f83..4c171e50ab2e5e 100644 --- a/posthog/tasks/exporter.py +++ b/posthog/tasks/exporter.py @@ -10,11 +10,14 @@ def export_asset(exported_asset_id: int, limit: Optional[int] = None,) -> None: from posthog.tasks.exports import csv_exporter, image_exporter - exported_asset = ExportedAsset.objects.select_related("insight", "dashboard").get(pk=exported_asset_id) + exported_asset: ExportedAsset = ExportedAsset.objects.select_related("insight", "dashboard").get( + pk=exported_asset_id + ) is_csv_export = exported_asset.export_format == ExportedAsset.ExportFormat.CSV if is_csv_export: - csv_exporter.export_csv(exported_asset, limit=limit) + max_limit = exported_asset.export_context.get("max_limit", 10000) + csv_exporter.export_csv(exported_asset, limit=limit, max_limit=max_limit) statsd.incr("csv_exporter.queued", tags={"team_id": str(exported_asset.team_id)}) else: image_exporter.export_image(exported_asset) From 9d699dde06012a427ecd618636205d05a6c5e3ae Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 18 Jul 2022 09:31:36 +0100 Subject: [PATCH 083/213] chore: remove the multi dashboard insight flag (#10836) --- .../components/InsightCard/InsightCard.tsx | 70 +++++++----------- .../SaveToDashboard/SaveToDashboard.tsx | 74 +++++-------------- frontend/src/lib/constants.tsx | 1 - frontend/src/models/insightsModel.tsx | 45 +---------- .../src/scenes/dashboard/DashboardItems.tsx | 3 - frontend/src/scenes/insights/Insight.tsx | 5 +- posthog/settings/feature_flags.py | 1 - 7 files changed, 49 insertions(+), 150 deletions(-) diff --git a/frontend/src/lib/components/InsightCard/InsightCard.tsx b/frontend/src/lib/components/InsightCard/InsightCard.tsx index 6cfa3c90951105..53694e4d062490 100644 --- a/frontend/src/lib/components/InsightCard/InsightCard.tsx +++ b/frontend/src/lib/components/InsightCard/InsightCard.tsx @@ -36,7 +36,7 @@ import { IconSubtitles, IconSubtitlesOff } from '../icons' import { CSSTransition, Transition } from 'react-transition-group' import { InsightDetails } from './InsightDetails' import { INSIGHT_TYPES_METADATA } from 'scenes/saved-insights/SavedInsights' -import { DashboardPrivilegeLevel, FEATURE_FLAGS } from 'lib/constants' +import { DashboardPrivilegeLevel } from 'lib/constants' import { funnelLogic } from 'scenes/funnels/funnelLogic' import { ActionsHorizontalBar, ActionsLineGraph, ActionsPie } from 'scenes/trends/viz' import { DashboardInsightsTable } from 'scenes/insights/views/InsightsTable/InsightsTable' @@ -50,7 +50,6 @@ import { cohortsModel } from '~/models/cohortsModel' import { mathsLogic } from 'scenes/trends/mathsLogic' import { WorldMap } from 'scenes/insights/views/WorldMap' import { AlertMessage } from '../AlertMessage' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { UserActivityIndicator } from '../UserActivityIndicator/UserActivityIndicator' // TODO: Add support for Retention to InsightDetails @@ -146,7 +145,6 @@ export interface InsightCardProps extends React.HTMLAttributes { refresh?: () => void rename?: () => void duplicate?: () => void - moveToDashboardOld?: (dashboardId: DashboardType['id']) => void moveToDashboard?: (dashboard: DashboardType) => void } @@ -160,7 +158,6 @@ interface InsightMetaProps | 'refresh' | 'rename' | 'duplicate' - | 'moveToDashboardOld' | 'moveToDashboard' | 'showEditingControls' | 'showDetailsControls' @@ -182,7 +179,6 @@ function InsightMeta({ refresh, rename, duplicate, - moveToDashboardOld, moveToDashboard, setPrimaryHeight, areDetailsShown, @@ -201,8 +197,6 @@ function InsightMeta({ (d: DashboardType) => !dashboards?.includes(d.id) ) - const { featureFlags } = useValues(featureFlagLogic) - const { ref: primaryRef, height: primaryHeight, width: primaryWidth } = useResizeObserver() const { ref: detailsRef, height: detailsHeight } = useResizeObserver() @@ -331,41 +325,31 @@ function InsightMeta({ Set color )} - {editable && - moveToDashboardOld && - moveToDashboard && - otherDashboards.length > 0 && ( - ( - { - !!featureFlags[ - FEATURE_FLAGS - .MULTI_DASHBOARD_INSIGHTS - ] - ? moveToDashboard(otherDashboard) - : moveToDashboardOld( - otherDashboard.id - ) - }} - fullWidth - > - {otherDashboard.name || Untitled} - - )), - placement: 'right-start', - fallbackPlacements: ['left-start'], - actionable: true, - }} - fullWidth - > - Move to - - )} + {editable && moveToDashboard && otherDashboards.length > 0 && ( + ( + { + moveToDashboard(otherDashboard) + }} + fullWidth + > + {otherDashboard.name || Untitled} + + )), + placement: 'right-start', + fallbackPlacements: ['left-start'], + actionable: true, + }} + fullWidth + > + Move to + + )} {editable && ( @@ -19,60 +16,25 @@ export function SaveToDashboard({ insight, canEditInsight }: SaveToDashboardProp const { rawDashboards } = useValues(dashboardsModel) const dashboards = insight.dashboards?.map((dashboard) => rawDashboards[dashboard]).filter((d) => !!d) || [] - const { featureFlags } = useValues(featureFlagLogic) - const multiDashboardInsights = featureFlags[FEATURE_FLAGS.MULTI_DASHBOARD_INSIGHTS] - return ( - {multiDashboardInsights ? ( - <> - setOpenModal(false)} - insight={insight} - canEditInsight={canEditInsight} - /> - setOpenModal(true)} - type="secondary" - icon={ - - - - } - > - Add to dashboard - - - ) : ( - <> - setOpenModal(false)} - insight={insight} - canEditInsight={canEditInsight} - /> - {dashboards.length > 0 ? ( - } - > - {dashboards.length > 1 ? 'On multiple dashboards' : `On dashboard: ${dashboards[0]?.name}`} - - ) : ( - setOpenModal(true)} - type="secondary" - icon={} - > - Add to dashboard - - )} - - )} + setOpenModal(false)} + insight={insight} + canEditInsight={canEditInsight} + /> + setOpenModal(true)} + type="secondary" + icon={ + + + + } + > + Add to dashboard + ) } diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 164cad87518a39..2534821cc21548 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -107,7 +107,6 @@ export const FEATURE_FLAGS = { BILLING_LIMIT: 'billing-limit', // owner: @timgl KAFKA_INSPECTOR: 'kafka-inspector', // owner: @yakkomajuri INSIGHT_EDITOR_PANELS: '8929-insight-editor-panels', // owner: @mariusandra - MULTI_DASHBOARD_INSIGHTS: 'multi-dashboard-insights', // owner: @pauldambra INSIGHT_ACTIVITY_LOG: '8545-insight-activity-log', // owner: @pauldambra FRONTEND_APPS: '9618-frontend-apps', // owner: @mariusandra EXPORT_DASHBOARD_INSIGHTS: 'export-dashboard-insights', // owner: @benjackwhite diff --git a/frontend/src/models/insightsModel.tsx b/frontend/src/models/insightsModel.tsx index 9afc48aacf4a38..11233e7234c579 100644 --- a/frontend/src/models/insightsModel.tsx +++ b/frontend/src/models/insightsModel.tsx @@ -17,10 +17,9 @@ export const insightsModel = kea({ actions: () => ({ renameInsight: (item: InsightModel) => ({ item }), renameInsightSuccess: (item: InsightModel) => ({ item }), - duplicateInsight: (item: InsightModel, dashboardId?: number, move: boolean = false) => ({ + duplicateInsight: (item: InsightModel, dashboardId?: number) => ({ item, dashboardId, - move, }), moveToDashboard: (item: InsightModel, fromDashboard: number, toDashboard: number, toDashboardName: string) => ({ item, @@ -90,7 +89,7 @@ export const insightsModel = kea({ } ) }, - duplicateInsight: async ({ item, dashboardId, move }) => { + duplicateInsight: async ({ item, dashboardId }) => { if (!item) { return } @@ -106,45 +105,7 @@ export const insightsModel = kea({ const dashboard = dashboardId ? dashboardsModel.values.rawDashboards[dashboardId] : null - if (move && dashboard) { - const deletedItem = await api.update( - `api/projects/${teamLogic.values.currentTeamId}/insights/${item.id}`, - { - deleted: true, - } - ) - dashboardsModel.actions.updateDashboardItem(deletedItem) - - lemonToast.success( - <> - Insight moved to{' '} - - {dashboard.name || 'Untitled'} - - , - { - button: { - label: 'Undo', - action: async () => { - const [restoredItem, removedItem] = await Promise.all([ - api.update(`api/projects/${teamLogic.values.currentTeamId}/insights/${item.id}`, { - deleted: false, - }), - api.update( - `api/projects/${teamLogic.values.currentTeamId}/insights/${addedItem.id}`, - { - deleted: true, - } - ), - ]) - lemonToast.success('Panel move reverted') - dashboardsModel.actions.updateDashboardItem(restoredItem) - dashboardsModel.actions.updateDashboardItem(removedItem) - }, - }, - } - ) - } else if (!move && dashboardId && dashboard) { + if (dashboardId && dashboard) { lemonToast.success('Insight copied', { button: { label: `View ${dashboard.name}`, diff --git a/frontend/src/scenes/dashboard/DashboardItems.tsx b/frontend/src/scenes/dashboard/DashboardItems.tsx index 1e586ce9f05605..8d1d857908c773 100644 --- a/frontend/src/scenes/dashboard/DashboardItems.tsx +++ b/frontend/src/scenes/dashboard/DashboardItems.tsx @@ -92,9 +92,6 @@ export function DashboardItems(): JSX.Element { refresh={() => refreshAllDashboardItems([item])} rename={() => renameInsight(item)} duplicate={() => duplicateInsight(item)} - moveToDashboardOld={(dashboardId: DashboardType['id']) => - duplicateInsight(item, dashboardId, true) - } moveToDashboard={({ id, name }: Pick) => { if (!dashboard) { throw new Error('must be on a dashboard to move an insight') diff --git a/frontend/src/scenes/insights/Insight.tsx b/frontend/src/scenes/insights/Insight.tsx index 62344714434d99..227dc4b1452119 100644 --- a/frontend/src/scenes/insights/Insight.tsx +++ b/frontend/src/scenes/insights/Insight.tsx @@ -138,9 +138,8 @@ export function Insight({ insightId }: { insightId: InsightShortId | 'new' }): J !canEditInsight ? { icon: , - tooltip: featureFlags[FEATURE_FLAGS.MULTI_DASHBOARD_INSIGHTS] - ? "You don't have edit permissions on any of the dashboards this insight belongs to. Ask a dashboard collaborator with edit access to add you." - : "You don't have edit permissions in the dashboard this insight belongs to. Ask a dashboard collaborator with edit access to add you.", + tooltip: + "You don't have edit permissions on any of the dashboards this insight belongs to. Ask a dashboard collaborator with edit access to add you.", } : undefined } diff --git a/posthog/settings/feature_flags.py b/posthog/settings/feature_flags.py index 80a8fd1f86c49d..277c3a4db3e76c 100644 --- a/posthog/settings/feature_flags.py +++ b/posthog/settings/feature_flags.py @@ -7,5 +7,4 @@ PERSISTED_FEATURE_FLAGS = get_list(os.getenv("PERSISTED_FEATURE_FLAGS", "")) + [ "invite-teammates-prompt", "insight-legends", - "multi-dashboard-insights", ] From e62995ef51b702dc668eb2987edda93e07b36710 Mon Sep 17 00:00:00 2001 From: Ben White Date: Mon, 18 Jul 2022 10:44:38 +0200 Subject: [PATCH 084/213] feat: Removed feature flags for exports/subscriptions/embeds (#10837) --- frontend/src/lib/constants.tsx | 3 -- .../src/scenes/dashboard/DashboardHeader.tsx | 8 ++--- frontend/src/scenes/insights/Insight.tsx | 33 ++++++++----------- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 2534821cc21548..3d665c4af72182 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -109,16 +109,13 @@ export const FEATURE_FLAGS = { INSIGHT_EDITOR_PANELS: '8929-insight-editor-panels', // owner: @mariusandra INSIGHT_ACTIVITY_LOG: '8545-insight-activity-log', // owner: @pauldambra FRONTEND_APPS: '9618-frontend-apps', // owner: @mariusandra - EXPORT_DASHBOARD_INSIGHTS: 'export-dashboard-insights', // owner: @benjackwhite ONBOARDING_1_5: 'onboarding-1_5', // owner: @liyiy BREAKDOWN_ATTRIBUTION: 'breakdown-attribution', // owner: @neilkakkar - INSIGHT_SUBSCRIPTIONS: 'insight-subscriptions', // owner: @benjackwhite SIMPLIFY_ACTIONS: 'simplify-actions', // owner: @alexkim205, SUBSCRIPTIONS_SLACK: 'subscriptions-slack', // owner: @benjackwhite SESSION_ANALYSIS: 'session-analysis', // owner: @rcmarron TOOLBAR_LAUNCH_SIDE_ACTION: 'toolbar-launch-side-action', // owner: @pauldambra, FEATURE_FLAG_EXPERIENCE_CONTINUITY: 'feature-flag-experience-continuity', // owner: @neilkakkar - EMBED_INSIGHTS: 'embed-insights', // owner: @mariusandra ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS: 'ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS', // owner: @pauldambra } diff --git a/frontend/src/scenes/dashboard/DashboardHeader.tsx b/frontend/src/scenes/dashboard/DashboardHeader.tsx index 2364b89b82b69d..379e93b34d30cd 100644 --- a/frontend/src/scenes/dashboard/DashboardHeader.tsx +++ b/frontend/src/scenes/dashboard/DashboardHeader.tsx @@ -39,8 +39,6 @@ export function DashboardHeader(): JSX.Element | null { const { hasAvailableFeature } = useValues(userLogic) const { featureFlags } = useValues(featureFlagLogic) - const usingExportFeature = featureFlags[FEATURE_FLAGS.EXPORT_DASHBOARD_INSIGHTS] - const usingSubscriptionFeature = featureFlags[FEATURE_FLAGS.INSIGHT_SUBSCRIPTIONS] const { push } = useActions(router) const exportOptions: ExportButtonItem[] = [ @@ -199,10 +197,8 @@ export function DashboardHeader(): JSX.Element | null { Pin dashboard ))} - {usingSubscriptionFeature && } - {usingExportFeature && ( - - )} + + diff --git a/frontend/src/scenes/insights/Insight.tsx b/frontend/src/scenes/insights/Insight.tsx index 227dc4b1452119..a1b57859cd0e18 100644 --- a/frontend/src/scenes/insights/Insight.tsx +++ b/frontend/src/scenes/insights/Insight.tsx @@ -79,9 +79,6 @@ export function Insight({ insightId }: { insightId: InsightShortId | 'new' }): J // const screens = useBreakpoint() const usingEditorPanels = featureFlags[FEATURE_FLAGS.INSIGHT_EDITOR_PANELS] - const usingExportFeature = featureFlags[FEATURE_FLAGS.EXPORT_DASHBOARD_INSIGHTS] - const usingEmbedFeature = featureFlags[FEATURE_FLAGS.EMBED_INSIGHTS] - const usingSubscriptionFeature = featureFlags[FEATURE_FLAGS.INSIGHT_SUBSCRIPTIONS] // Show the skeleton if loading an insight for which we only know the id // This helps with the UX flickering and showing placeholder "name" text. @@ -172,24 +169,20 @@ export function Insight({ insightId }: { insightId: InsightShortId | 'new' }): J - {usingEmbedFeature && ( - - insight.short_id - ? push(urls.insightSharing(insight.short_id)) - : null - } - fullWidth - > - Share or embed - - )} - {usingExportFeature && insight.short_id && ( + + insight.short_id + ? push(urls.insightSharing(insight.short_id)) + : null + } + fullWidth + > + Share or embed + + {insight.short_id && ( <> - {usingSubscriptionFeature && ( - - )} + {exporterResourceParams ? ( Date: Mon, 18 Jul 2022 14:46:31 +0100 Subject: [PATCH 085/213] chore: update insight cache logging (#10842) --- posthog/tasks/test/test_update_cache.py | 23 +++++++++++++-- posthog/tasks/update_cache.py | 39 ++++++++++++++----------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/posthog/tasks/test/test_update_cache.py b/posthog/tasks/test/test_update_cache.py index 1d6f89c51d511e..1492b02293866d 100644 --- a/posthog/tasks/test/test_update_cache.py +++ b/posthog/tasks/test/test_update_cache.py @@ -752,10 +752,11 @@ def test_never_refreshed_tiles_are_gauged(self, statsd_gauge: MagicMock) -> None statsd_gauge.assert_any_call("update_cache_queue.never_refreshed", 1) @freeze_time("2022-12-01T13:54:00.000Z") + @patch("posthog.tasks.update_cache.logger") @patch("posthog.tasks.update_cache.statsd.gauge") - def test_refresh_age_of_tiles_is_gauged(self, statsd_gauge: MagicMock) -> None: + def test_refresh_age_of_tiles_is_gauged(self, statsd_gauge: MagicMock, logger: MagicMock,) -> None: tile_one = self._a_dashboard_tile_with_known_last_refresh(datetime.now(pytz.utc) - timedelta(hours=1)) - self._a_dashboard_tile_with_known_last_refresh(datetime.now(pytz.utc) - timedelta(hours=0.5)) + tile_two = self._a_dashboard_tile_with_known_last_refresh(datetime.now(pytz.utc) - timedelta(hours=0.5)) update_cached_items() @@ -769,6 +770,24 @@ def test_refresh_age_of_tiles_is_gauged(self, statsd_gauge: MagicMock) -> None: }, ) + statsd_gauge.assert_any_call( + "update_cache_queue.dashboards_lag", + 1800, + tags={ + "insight_id": tile_two.insight_id, + "dashboard_id": tile_two.dashboard_id, + "cache_key": tile_two.filters_hash, + }, + ) + + logger.error.assert_called_with( + "insight_cache.waiting_for_more_than_thirty_minutes", + insight_id=tile_one.insight_id, + dashboard_id=tile_one.dashboard_id, + cache_key=tile_one.filters_hash, + team_id=self.team.id, + ) + def _a_dashboard_tile_with_known_last_refresh(self, last_refresh_date: datetime) -> DashboardTile: dashboard = create_shared_dashboard(team=self.team, is_shared=True) filter = {"events": [{"id": "$pageview"}]} diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index 22d6c18a98163a..5f35bbd436218e 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -225,6 +225,28 @@ def update_cached_items() -> Tuple[int, int]: capture_exception(e) + if dashboard_tile.last_refresh: + dashboard_cache_age = (datetime.datetime.now(timezone.utc) - dashboard_tile.last_refresh).total_seconds() + + statsd.gauge( + "update_cache_queue.dashboards_lag", + round(dashboard_cache_age), + tags={ + "insight_id": dashboard_tile.insight_id, + "dashboard_id": dashboard_tile.dashboard_id, + "cache_key": dashboard_tile.filters_hash, + }, + ) + + if dashboard_cache_age > 1800: + logger.error( + "insight_cache.waiting_for_more_than_thirty_minutes", + insight_id=dashboard_tile.insight.id, + dashboard_id=dashboard_tile.dashboard.id, + cache_key=dashboard_tile.filters_hash, + team_id=dashboard_tile.insight.team.id, + ) + shared_insights = ( Insight.objects.filter(sharingconfiguration__enabled=True) .exclude(deleted=True) @@ -245,23 +267,6 @@ def update_cached_items() -> Tuple[int, int]: statsd.gauge("update_cache_queue.never_refreshed", dashboard_tiles.filter(last_refresh=None).count()) - # how old is the next to be refreshed - oldest_previously_refreshed_tile: Optional[DashboardTile] = dashboard_tiles.exclude(last_refresh=None).first() - if oldest_previously_refreshed_tile and oldest_previously_refreshed_tile.last_refresh: - dashboard_cache_age = ( - datetime.datetime.now(timezone.utc) - oldest_previously_refreshed_tile.last_refresh - ).total_seconds() - - statsd.gauge( - "update_cache_queue.dashboards_lag", - round(dashboard_cache_age), - tags={ - "insight_id": oldest_previously_refreshed_tile.insight_id, - "dashboard_id": oldest_previously_refreshed_tile.dashboard_id, - "cache_key": oldest_previously_refreshed_tile.filters_hash, - }, - ) - logger.info("update_cache_queue", length=len(tasks)) taskset = group(tasks) taskset.apply_async() From 2837c497d6e5b7c2a8a03eececfd62d795d46a88 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 18 Jul 2022 14:56:49 +0100 Subject: [PATCH 086/213] chore: update cache runs in a group so should not ignore result (#10844) --- posthog/celery.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/posthog/celery.py b/posthog/celery.py index 16463ac473b122..0700bf5dfdca9d 100644 --- a/posthog/celery.py +++ b/posthog/celery.py @@ -422,8 +422,12 @@ def check_cached_items(): update_cached_items() -@app.task(ignore_result=True) +@app.task(ignore_result=False) def update_cache_item_task(key: str, cache_type, payload: dict) -> None: + """ + Tasks used in a group (as this is) must not ignore their results + https://docs.celeryq.dev/en/latest/userguide/canvas.html#groups:~:text=Similarly%20to%20chords%2C%20tasks%20used%20in%20a%20group%20must%20not%20ignore%20their%20results. + """ from posthog.tasks.update_cache import update_cache_item update_cache_item(key, cache_type, payload) From 8ff436f6e13c07dc7782709edf0166b6d544ed3d Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Mon, 18 Jul 2022 15:31:01 +0100 Subject: [PATCH 087/213] fix: makes the persons modal work with the new export api (#10824) * fix: makes the persons modal work with the new export api without fixing the URL generation * don't use CSV API for new exports * corrects auth so new exporter can get people * teach CSV exporter to process persons modal * check temporary token before jwt auth * add a second flag to cover turning exports on without turning person modal back on for all * maintain context --- frontend/src/lib/api.ts | 9 +- frontend/src/lib/constants.tsx | 2 + frontend/src/scenes/trends/PersonsModal.tsx | 42 +++-- posthog/api/action.py | 3 +- posthog/tasks/exports/csv_exporter.py | 8 +- .../test/csv_renders/persons_modal.json | 158 ++++++++++++++++++ 6 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 posthog/tasks/exports/test/csv_renders/persons_modal.json diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index aaf79ec284035a..80713b8c9afc0d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -402,10 +402,15 @@ const api = { determineDeleteEndpoint(): string { return new ApiRequest().actions().assembleEndpointUrl() }, - determinePeopleCsvUrl(peopleParams: PeopleParamType, filters: Partial): string { + determinePeopleCsvUrl( + peopleParams: PeopleParamType, + filters: Partial, + useBareAPIURL: boolean = false + ): string { + const apiAction = `people${useBareAPIURL ? '' : '.csv'}` return new ApiRequest() .actions() - .withAction('people.csv') + .withAction(apiAction) .withQueryString(parsePeopleParams(peopleParams, filters)) .assembleFullUrl(true) }, diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 3d665c4af72182..f1e61ddb2294ad 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -117,6 +117,8 @@ export const FEATURE_FLAGS = { TOOLBAR_LAUNCH_SIDE_ACTION: 'toolbar-launch-side-action', // owner: @pauldambra, FEATURE_FLAG_EXPERIENCE_CONTINUITY: 'feature-flag-experience-continuity', // owner: @neilkakkar ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS: 'ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS', // owner: @pauldambra + // Re-enable person modal CSV downloads when frontend can support new entity properties + PERSON_MODAL_EXPORTS: 'person-modal-exports', // hot potato see https://github.com/PostHog/posthog/pull/10824 } /** Which self-hosted plan's features are available with Cloud's "Standard" plan (aka card attached). */ diff --git a/frontend/src/scenes/trends/PersonsModal.tsx b/frontend/src/scenes/trends/PersonsModal.tsx index 01351be9797a94..377de8c4259d4f 100644 --- a/frontend/src/scenes/trends/PersonsModal.tsx +++ b/frontend/src/scenes/trends/PersonsModal.tsx @@ -1,8 +1,8 @@ import React, { useMemo } from 'react' import { useActions, useValues } from 'kea' import { DownloadOutlined, UsergroupAddOutlined } from '@ant-design/icons' -import { Modal, Button, Input, Skeleton, Select } from 'antd' -import { FilterType, InsightType, ActorType, ChartDisplayType } from '~/types' +import { Button, Input, Modal, Select, Skeleton } from 'antd' +import { ActorType, ChartDisplayType, ExporterFormat, FilterType, InsightType } from '~/types' import { personsModalLogic } from './personsModalLogic' import { CopyToClipboardInline } from 'lib/components/CopyToClipboard' import { capitalizeFirstLetter, isGroupType, midEllipsis, pluralize } from 'lib/utils' @@ -22,6 +22,7 @@ import { FEATURE_FLAGS } from 'lib/constants' import { SessionPlayerDrawer } from 'scenes/session-recordings/SessionPlayerDrawer' import { MultiRecordingButton } from 'scenes/session-recordings/multiRecordingButton/multiRecordingButton' import { countryCodeToFlag, countryCodeToName } from 'scenes/insights/views/WorldMap/countryCodes' +import { triggerExport } from 'lib/components/ExportButton/exporter' export interface PersonsModalProps { visible: boolean @@ -102,9 +103,12 @@ export function PersonsModal({ ) const flaggedInsights = featureFlags[FEATURE_FLAGS.NEW_INSIGHT_COHORTS] - // TODO: Re-enable CSV downloads when frontend can support new entity properties - // const isDownloadCsvAvailable: boolean = view === InsightType.TRENDS && showModalActions && !!people?.action - const isDownloadCsvAvailable: boolean = false + const isDownloadCsvAvailable: boolean = + !!featureFlags[FEATURE_FLAGS.ASYNC_EXPORT_CSV_FOR_LIVE_EVENTS] && + !!featureFlags[FEATURE_FLAGS.PERSON_MODAL_EXPORTS] && + InsightType.TRENDS && + showModalActions && + !!people?.action const isSaveAsCohortAvailable = (view === InsightType.TRENDS || view === InsightType.STICKINESS || @@ -128,16 +132,24 @@ export function PersonsModal({ {isDownloadCsvAvailable && ( - - ) -} - -function SubscriptionFailure(): JSX.Element { - const { sessionId } = useValues(billingSubscribedLogic) - return ( - <> - -

Something went wrong

-

- We couldn't start your subscription. Please try again with a{' '} - different payment method or contact us if the problem persists. -

- {sessionId && ( - /* Note we include PostHog Cloud specifically (app.posthog.com) in case a self-hosted user - ended up here for some reason. Should not happen as these should be processed by license.posthog.com */ - - )} -
- - Go to PostHog - -
- + + ) } diff --git a/frontend/src/scenes/billing/billingLogic.ts b/frontend/src/scenes/billing/billingLogic.ts index 3ab098abd11c48..71028943c4eacf 100644 --- a/frontend/src/scenes/billing/billingLogic.ts +++ b/frontend/src/scenes/billing/billingLogic.ts @@ -7,6 +7,9 @@ import posthog from 'posthog-js' import { sceneLogic } from 'scenes/sceneLogic' import { Scene } from 'scenes/sceneTypes' import { lemonToast } from 'lib/components/lemonToast' +import { router } from 'kea-router' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { FEATURE_FLAGS } from 'lib/constants' export const UTM_TAGS = 'utm_medium=in-product&utm_campaign=billing-management' export const ALLOCATION_THRESHOLD_ALERT = 0.85 // Threshold to show warning of event usage near limit @@ -22,7 +25,10 @@ export const billingLogic = kea({ actions: { registerInstrumentationProps: true, }, - loaders: ({ actions }) => ({ + connect: { + values: [featureFlagLogic, ['featureFlags']], + }, + loaders: ({ actions, values }) => ({ billing: [ null as BillingType | null, { @@ -31,12 +37,22 @@ export const billingLogic = kea({ if (!response?.plan) { actions.loadPlans() } + if ( + response.current_usage > 1000000 && + response.should_setup_billing && + router.values.location.pathname !== '/organization/billing/locked' && + values.featureFlags[FEATURE_FLAGS.BILLING_LOCK_EVERYTHING] + ) { + posthog.capture('billing locked screen shown') + router.actions.replace('/organization/billing/locked') + } actions.registerInstrumentationProps() return response as BillingType }, setBillingLimit: async (billing: BillingType) => { const res = await api.update('api/billing/', billing) lemonToast.success(`Billing limit set to $${billing.billing_limit} usd/month`) + return res as BillingType }, }, diff --git a/frontend/src/scenes/billing/billingSubscribedLogic.test.ts b/frontend/src/scenes/billing/billingSubscribedLogic.test.ts deleted file mode 100644 index 561f3e3c117e8b..00000000000000 --- a/frontend/src/scenes/billing/billingSubscribedLogic.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { expectLogic } from 'kea-test-utils' -import { initKeaTests } from '~/test/init' -import { router } from 'kea-router' -import { sceneLogic } from 'scenes/sceneLogic' -import { billingSubscribedLogic } from './billingSubscribedLogic' -import { billingLogic } from './billingLogic' - -describe('organizationLogic', () => { - let logic: ReturnType - - describe('if subscription was successful', () => { - beforeEach(() => { - initKeaTests() - logic = billingSubscribedLogic() - logic.mount() - }) - - it('mounts other logics', async () => { - await expectLogic(logic).toMount([sceneLogic, billingLogic]) - }) - - it('shows failed subscription by default', async () => { - await expectLogic(logic).toNotHaveDispatchedActions(['setStatus', 'setSubscriptionId']) - await expectLogic(logic).toMatchValues({ - status: 'failed', - }) - }) - - it('loads plan information and sets proper actions', async () => { - router.actions.push('/organization/billing/subscribed', { s: 'success' }) - await expectLogic(logic).toDispatchActions(['setStatus']) - await expectLogic(logic).toMatchValues({ - status: 'success', - sessionId: null, - }) - await expectLogic(logic).toNotHaveDispatchedActions(['setSessionId']) - }) - - it('loads failed page with session id', async () => { - router.actions.push('/organization/billing/subscribed', { session_id: 'cs_test_12345678' }) - await expectLogic(logic).toDispatchActions(['setSessionId']) - await expectLogic(logic).toMatchValues({ - status: 'failed', - sessionId: 'cs_test_12345678', - billing: null, - }) - await expectLogic(logic).toNotHaveDispatchedActions(['setStatus']) - }) - }) -}) diff --git a/frontend/src/scenes/billing/billingSubscribedLogic.ts b/frontend/src/scenes/billing/billingSubscribedLogic.ts deleted file mode 100644 index 1585a328aef95c..00000000000000 --- a/frontend/src/scenes/billing/billingSubscribedLogic.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { kea } from 'kea' -import { sceneLogic } from 'scenes/sceneLogic' -import { billingLogic } from './billingLogic' -import type { billingSubscribedLogicType } from './billingSubscribedLogicType' - -export enum SubscriptionStatus { - Success = 'success', - Failed = 'failed', -} - -export const billingSubscribedLogic = kea({ - path: ['scenes', 'billing', 'billingSubscribedLogic'], - connect: { - actions: [sceneLogic, ['setScene']], - values: [billingLogic, ['billing']], - }, - actions: { - setStatus: (status: SubscriptionStatus) => ({ status }), - setSessionId: (id: string) => ({ id }), - }, - reducers: { - status: [ - SubscriptionStatus.Failed as SubscriptionStatus, - { - setStatus: (_, { status }) => status, - }, - ], - sessionId: [ - null as string | null, - { - setSessionId: (_, { id }) => id, - }, - ], - }, - urlToAction: ({ actions }) => ({ - '/organization/billing/subscribed': (_, { s, session_id }) => { - if (s === 'success') { - actions.setStatus(SubscriptionStatus.Success) - } - if (session_id) { - actions.setSessionId(session_id) - } - }, - }), -}) diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index e8e8392fb88206..4bf7e499e5dc3e 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -41,6 +41,7 @@ export enum Scene { Annotations = 'Annotations', Billing = 'Billing', BillingSubscribed = 'BillingSubscribed', + BillingLocked = 'BillingLocked', Plugins = 'Plugins', FrontendAppScene = 'FrontendAppScene', SavedInsights = 'SavedInsights', diff --git a/frontend/src/scenes/scenes.ts b/frontend/src/scenes/scenes.ts index a3506ab0a19e56..bd179e618499a5 100644 --- a/frontend/src/scenes/scenes.ts +++ b/frontend/src/scenes/scenes.ts @@ -215,6 +215,10 @@ export const sceneConfigurations: Partial> = { plain: true, allowUnauthenticated: true, }, + [Scene.BillingLocked]: { + plain: true, + allowUnauthenticated: true, + }, [Scene.Unsubscribe]: { allowUnauthenticated: true, }, @@ -281,6 +285,7 @@ export const routes: Record = { [urls.organizationSettings()]: Scene.OrganizationSettings, [urls.organizationBilling()]: Scene.Billing, [urls.billingSubscribed()]: Scene.BillingSubscribed, + [urls.billingLocked()]: Scene.BillingLocked, [urls.organizationCreateFirst()]: Scene.OrganizationCreateFirst, [urls.organizationCreationConfirm()]: Scene.OrganizationCreationConfirm, [urls.instanceLicenses()]: Scene.Licenses, diff --git a/frontend/src/scenes/urls.ts b/frontend/src/scenes/urls.ts index c36376a732ac47..01f43f824b30ed 100644 --- a/frontend/src/scenes/urls.ts +++ b/frontend/src/scenes/urls.ts @@ -78,6 +78,7 @@ export const urls = { // Cloud only organizationBilling: (): string => '/organization/billing', billingSubscribed: (): string => '/organization/billing/subscribed', + billingLocked: (): string => '/organization/billing/locked', // Self-hosted only instanceLicenses: (): string => '/instance/licenses', instanceStatus: (): string => '/instance/status', From f1ebaf9a949b00f89e5762a14aefd8adbeb8f8ad Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 27 Jul 2022 13:09:12 +0300 Subject: [PATCH 178/213] feat(persons-on-events): async migration for persons-on-events (#10734) * feat(persons-on-events): add async migration for persons/groups on events backfill * Simplify run_optimize_table * run_optimize_table: explicit per_shard * Make execute_on_each_shard work * Add ON CLUSTER clauses everywhere * Reorder methods * Add first test, make backfill run successfully * Correct source database * Test for person columns * Test for deleted person behavior * Add test for person versioning * Dont leave garbage tables behind * Add test for groups * Update codec for person/groups columns on events * Code style improvement * Check disk usage before starting backfill/healthcheck * Document some constraints * Standardize a function * Fixup for disk util * Add test for rollbacks * Make per shard execution work for events backfill * Gate async migration on self-hosted * Fixup mutations_sync * Sleep until ALTER TABLE complete over mutations_sync * Make backfill step normal SQL * Attempt to minimize bleed between tests * Inter-test dependencies * Dont rely on default database * Use async IO * Update dictionary connection logic * update schema snapshots * ON CLUSTER * Ignore self query in _get_number_running_on_cluster * Reformat sql Co-authored-by: yakkomajuri --- posthog/async_migrations/disk_util.py | 36 ++ .../migrations/0002_events_sample_by.py | 28 +- ...6_persons_and_groups_on_events_backfill.py | 342 ++++++++++++++++++ .../test/test_0004_replicated_schema.py | 9 +- ...6_persons_and_groups_on_events_backfill.py | 299 +++++++++++++++ .../test/test_migrations_not_required.py | 2 +- posthog/async_migrations/utils.py | 52 ++- .../test/__snapshots__/test_schema.ambr | 84 ++--- posthog/models/event/sql.py | 12 +- posthog/models/group/util.py | 20 +- posthog/models/person/sql.py | 2 +- posthog/models/person/util.py | 10 +- 12 files changed, 791 insertions(+), 105 deletions(-) create mode 100644 posthog/async_migrations/disk_util.py create mode 100644 posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py create mode 100644 posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py diff --git a/posthog/async_migrations/disk_util.py b/posthog/async_migrations/disk_util.py new file mode 100644 index 00000000000000..559e4eeea9f005 --- /dev/null +++ b/posthog/async_migrations/disk_util.py @@ -0,0 +1,36 @@ +from posthog.client import sync_execute +from posthog.settings import CLICKHOUSE_DATABASE + + +def analyze_enough_disk_space_free_for_table(table_name: str, required_ratio: float): + """ + Analyzes whether there's enough disk space free for given async migration operation. + + This is done by checking whether there's at least ratio times space free to resize table_name with. + """ + + current_ratio, _, required_space_pretty = sync_execute( + f""" + WITH ( + SELECT free_space + FROM system.disks WHERE name = 'default' + ) AS free_disk_space,( + SELECT total_space + FROM system.disks WHERE name = 'default' + ) AS total_disk_space,( + SELECT sum(bytes) as size + FROM system.parts + WHERE table = %(table_name)s AND database = %(database)s + ) AS table_size + SELECT + free_disk_space / greatest(table_size, 1), + total_disk_space - (free_disk_space - %(ratio)s * table_size) AS required, + formatReadableSize(required) + """, + {"database": CLICKHOUSE_DATABASE, "table_name": table_name, "ratio": required_ratio,}, + )[0] + + if current_ratio >= required_ratio: + return (True, None) + else: + return (False, f"Upgrade your ClickHouse storage to at least {required_space_pretty}.") diff --git a/posthog/async_migrations/migrations/0002_events_sample_by.py b/posthog/async_migrations/migrations/0002_events_sample_by.py index a2e0b2c6af3535..b441c52c54e907 100644 --- a/posthog/async_migrations/migrations/0002_events_sample_by.py +++ b/posthog/async_migrations/migrations/0002_events_sample_by.py @@ -8,6 +8,7 @@ AsyncMigrationOperation, AsyncMigrationOperationSQL, ) +from posthog.async_migrations.disk_util import analyze_enough_disk_space_free_for_table from posthog.async_migrations.utils import run_optimize_table from posthog.client import sync_execute from posthog.constants import AnalyticsDBMS @@ -172,32 +173,7 @@ def precheck(self): ) events_table = "sharded_events" if CLICKHOUSE_REPLICATION else "events" - result = sync_execute( - f""" - SELECT (free_space.size / greatest(event_table_size.size, 1)) FROM - (SELECT 1 as jc, 'event_table_size', sum(bytes) as size FROM system.parts WHERE table = '{events_table}' AND database='{CLICKHOUSE_DATABASE}') event_table_size - JOIN - (SELECT 1 as jc, 'free_disk_space', free_space as size FROM system.disks WHERE name = 'default') free_space - ON event_table_size.jc=free_space.jc - """ - ) - event_size_to_free_space_ratio = result[0][0] - - # Require 1.5x the events table in free space to be available - if event_size_to_free_space_ratio > 1.5: - return (True, None) - else: - result = sync_execute( - f""" - SELECT formatReadableSize(free_space.size - (free_space.free_space - (1.5 * event_table_size.size ))) as required FROM - (SELECT 1 as jc, 'event_table_size', sum(bytes) as size FROM system.parts WHERE table = '{events_table}' AND database='{CLICKHOUSE_DATABASE}') event_table_size - JOIN - (SELECT 1 as jc, 'free_disk_space', free_space, total_space as size FROM system.disks WHERE name = 'default') free_space - ON event_table_size.jc=free_space.jc - """ - ) - required_space = result[0][0] - return (False, f"Upgrade your ClickHouse storage to at least {required_space}.") + return analyze_enough_disk_space_free_for_table(events_table, required_ratio=1.5) def healthcheck(self): result = sync_execute("SELECT free_space FROM system.disks") diff --git a/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py b/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py new file mode 100644 index 00000000000000..695f320a3b5790 --- /dev/null +++ b/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py @@ -0,0 +1,342 @@ +from functools import cached_property + +import structlog +from django.conf import settings + +from posthog.async_migrations.definition import ( + AsyncMigrationDefinition, + AsyncMigrationOperation, + AsyncMigrationOperationSQL, +) +from posthog.async_migrations.disk_util import analyze_enough_disk_space_free_for_table +from posthog.async_migrations.utils import execute_op_clickhouse, run_optimize_table, sleep_until_finished +from posthog.client import sync_execute +from posthog.models.event.sql import EVENTS_DATA_TABLE + +logger = structlog.get_logger(__name__) + +""" +Migration summary +================= + +Backfill the sharded_events table to add data for the following columns: + +- person_id +- person_properties +- person_created_at +- groupX_properties +- groupX_created_at + +This allows us to switch entirely to only querying the events table for insights, +without having to hit additional tables for persons, groups, and distinct IDs. + +Migration strategy +================== + +We will run the following operations on the cluster +(or on one node per shard if shard-level configuration is provided): + +1. Update `person_properties` and `groupX_properties` columns to use ZSTD(3) compression +2. Create temporary tables with the relevant columns from `person`, `person_distinct_id`, and `groups` +3. Copy data from the main tables into them +4. Optimize the temporary tables to remove duplicates and remove deleted data +5. Create a dictionary to query each temporary table with caching +6. Run an ALTER TABLE ... UPDATE to backfill all the data using the dictionaries + +Constraints +=========== + +1. The migration requires a lot of extra space for the new columns. At least 2x disk space is required to avoid issues while migrating. +2. We use ZSTD(3) compression on the new columns to save on space and speed up large reads. +3. New columns need to be populated for new rows before running this async migration. +""" + +TEMPORARY_PERSONS_TABLE_NAME = "tmp_person_0006" +TEMPORARY_PDI2_TABLE_NAME = "tmp_person_distinct_id2_0006" +TEMPORARY_GROUPS_TABLE_NAME = "tmp_groups_0006" + + +class Migration(AsyncMigrationDefinition): + description = "Backfill persons and groups data on the sharded_events table" + + depends_on = "0005_person_replacing_by_version" + + def precheck(self): + if not settings.MULTI_TENANCY: + return False, "This async migration is not yet ready for self-hosted users" + + return analyze_enough_disk_space_free_for_table(EVENTS_DATA_TABLE(), required_ratio=2.0) + + def is_required(self) -> bool: + compression_codec = sync_execute( + """ + SELECT compression_codec + FROM system.columns + WHERE database = %(database)s + AND table = %(events_data_table)s + AND name = 'person_properties' + """, + {"database": settings.CLICKHOUSE_DATABASE, "events_data_table": EVENTS_DATA_TABLE()}, + )[0][0] + + return compression_codec != "CODEC(ZSTD(3))" + + @cached_property + def operations(self): + return [ + AsyncMigrationOperation( + # See https://github.com/PostHog/posthog/issues/10616 for details on choice of codec + fn=lambda query_id: self._update_properties_column_compression_codec(query_id, "ZSTD(3)"), + rollback_fn=lambda query_id: self._update_properties_column_compression_codec(query_id, "LZ4"), + ), + AsyncMigrationOperationSQL( + sql=f""" + CREATE TABLE {TEMPORARY_PERSONS_TABLE_NAME} {{on_cluster_clause}} AS {settings.CLICKHOUSE_DATABASE}.person + ENGINE = ReplacingMergeTree(version) + ORDER BY (team_id, id) + SETTINGS index_granularity = 128 + """, + rollback=f"DROP TABLE IF EXISTS {TEMPORARY_PERSONS_TABLE_NAME} {{on_cluster_clause}}", + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + CREATE TABLE {TEMPORARY_PDI2_TABLE_NAME} {{on_cluster_clause}} AS {settings.CLICKHOUSE_DATABASE}.person_distinct_id2 + ENGINE = ReplacingMergeTree(version) + ORDER BY (team_id, distinct_id) + SETTINGS index_granularity = 128 + """, + rollback=f"DROP TABLE IF EXISTS {TEMPORARY_PDI2_TABLE_NAME} {{on_cluster_clause}}", + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + CREATE TABLE {TEMPORARY_GROUPS_TABLE_NAME} {{on_cluster_clause}} AS {settings.CLICKHOUSE_DATABASE}.groups + ENGINE = ReplacingMergeTree(_timestamp) + ORDER BY (team_id, group_type_index, group_key) + SETTINGS index_granularity = 128 + """, + rollback=f"DROP TABLE IF EXISTS {TEMPORARY_GROUPS_TABLE_NAME} {{on_cluster_clause}}", + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + ALTER TABLE {TEMPORARY_PERSONS_TABLE_NAME} {{on_cluster_clause}} + REPLACE PARTITION tuple() FROM person + """, + rollback=None, + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + ALTER TABLE {TEMPORARY_PDI2_TABLE_NAME} {{on_cluster_clause}} + REPLACE PARTITION tuple() FROM person_distinct_id2 + """, + rollback=None, + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + ALTER TABLE {TEMPORARY_GROUPS_TABLE_NAME} {{on_cluster_clause}} + REPLACE PARTITION tuple() FROM groups + """, + rollback=None, + per_shard=True, + ), + AsyncMigrationOperation( + fn=lambda query_id: run_optimize_table( + unique_name="0006_persons_and_groups_on_events_backfill_person", + query_id=query_id, + table_name=TEMPORARY_PERSONS_TABLE_NAME, + final=True, + deduplicate=True, + ) + ), + AsyncMigrationOperation( + fn=lambda query_id: run_optimize_table( + unique_name="0006_persons_and_groups_on_events_backfill_pdi2", + query_id=query_id, + table_name=TEMPORARY_PDI2_TABLE_NAME, + final=True, + deduplicate=True, + ) + ), + AsyncMigrationOperation( + fn=lambda query_id: run_optimize_table( + unique_name="0006_persons_and_groups_on_events_backfill_groups", + query_id=query_id, + table_name=TEMPORARY_GROUPS_TABLE_NAME, + final=True, + deduplicate=True, + per_shard=True, + ) + ), + AsyncMigrationOperationSQL( + sql=f""" + ALTER TABLE {TEMPORARY_PDI2_TABLE_NAME} {{on_cluster_clause}} + DELETE WHERE is_deleted = 1 OR person_id IN ( + SELECT id FROM {TEMPORARY_PERSONS_TABLE_NAME} WHERE is_deleted=1 + ) + """, + sql_settings={"mutations_sync": 2}, + rollback=None, + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + ALTER TABLE {TEMPORARY_PERSONS_TABLE_NAME} {{on_cluster_clause}} + DELETE WHERE is_deleted = 1 + """, + sql_settings={"mutations_sync": 2}, + rollback=None, + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + CREATE DICTIONARY IF NOT EXISTS person_dict {{on_cluster_clause}} + ( + team_id Int64, + id UUID, + properties String, + created_at DateTime + ) + PRIMARY KEY team_id, id + SOURCE(CLICKHOUSE(TABLE {TEMPORARY_PERSONS_TABLE_NAME} {self._dictionary_connection_string()})) + LAYOUT(complex_key_cache(size_in_cells 5000000 max_threads_for_updates 6 allow_read_expired_keys 1)) + Lifetime(60000) + """, + rollback=f"DROP DICTIONARY IF EXISTS person_dict {{on_cluster_clause}}", + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + CREATE DICTIONARY IF NOT EXISTS person_distinct_id2_dict {{on_cluster_clause}} + ( + team_id Int64, + distinct_id String, + person_id UUID + ) + PRIMARY KEY team_id, distinct_id + SOURCE(CLICKHOUSE(TABLE {TEMPORARY_PDI2_TABLE_NAME} {self._dictionary_connection_string()})) + LAYOUT(complex_key_cache(size_in_cells 50000000 max_threads_for_updates 6 allow_read_expired_keys 1)) + Lifetime(60000) + """, + rollback=f"DROP DICTIONARY IF EXISTS person_distinct_id2_dict {{on_cluster_clause}}", + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + CREATE DICTIONARY IF NOT EXISTS groups_dict {{on_cluster_clause}} + ( + team_id Int64, + group_type_index UInt8, + group_key String, + group_properties String, + created_at DateTime + ) + PRIMARY KEY team_id, group_type_index, group_key + SOURCE(CLICKHOUSE(TABLE {TEMPORARY_GROUPS_TABLE_NAME} {self._dictionary_connection_string()})) + LAYOUT(complex_key_cache(size_in_cells 1000000 max_threads_for_updates 6 allow_read_expired_keys 1)) + Lifetime(60000) + """, + rollback=f"DROP DICTIONARY IF EXISTS groups_dict {{on_cluster_clause}}", + per_shard=True, + ), + AsyncMigrationOperationSQL( + sql=f""" + ALTER TABLE {EVENTS_DATA_TABLE()} + {{on_cluster_clause}} + UPDATE + person_id = toUUID(dictGet('{settings.CLICKHOUSE_DATABASE}.person_distinct_id2_dict', 'person_id', tuple(team_id, distinct_id))), + person_properties = dictGetString( + '{settings.CLICKHOUSE_DATABASE}.person_dict', + 'properties', + tuple( + team_id, + toUUID(dictGet('{settings.CLICKHOUSE_DATABASE}.person_distinct_id2_dict', 'person_id', tuple(team_id, distinct_id))) + ) + ), + person_created_at = dictGetDateTime( + '{settings.CLICKHOUSE_DATABASE}.person_dict', + 'created_at', + tuple( + team_id, + toUUID(dictGet('{settings.CLICKHOUSE_DATABASE}.person_distinct_id2_dict', 'person_id', tuple(team_id, distinct_id))) + ) + ), + group0_properties = dictGetString('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'group_properties', tuple(team_id, 0, $group_0)), + group1_properties = dictGetString('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'group_properties', tuple(team_id, 1, $group_1)), + group2_properties = dictGetString('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'group_properties', tuple(team_id, 2, $group_2)), + group3_properties = dictGetString('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'group_properties', tuple(team_id, 3, $group_3)), + group4_properties = dictGetString('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'group_properties', tuple(team_id, 4, $group_4)), + group0_created_at = dictGetDateTime('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'created_at', tuple(team_id, 0, $group_0)), + group1_created_at = dictGetDateTime('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'created_at', tuple(team_id, 1, $group_1)), + group2_created_at = dictGetDateTime('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'created_at', tuple(team_id, 2, $group_2)), + group3_created_at = dictGetDateTime('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'created_at', tuple(team_id, 3, $group_3)), + group4_created_at = dictGetDateTime('{settings.CLICKHOUSE_DATABASE}.groups_dict', 'created_at', tuple(team_id, 4, $group_4)) + WHERE person_id = toUUIDOrZero('') + """, + sql_settings={"max_execution_time": 0}, + rollback=None, + per_shard=True, + ), + AsyncMigrationOperation(fn=self._wait_for_mutation_done,), + AsyncMigrationOperation(fn=self._clear_temporary_tables), + ] + + def _dictionary_connection_string(self): + result = f"DB '{settings.CLICKHOUSE_DATABASE}'" + if settings.CLICKHOUSE_USER: + result += f" USER '{settings.CLICKHOUSE_USER}'" + if settings.CLICKHOUSE_PASSWORD: + result += f" PASSWORD '{settings.CLICKHOUSE_PASSWORD}'" + return result + + def _update_properties_column_compression_codec(self, query_id, codec): + columns = [ + "person_properties", + "group0_properties", + "group1_properties", + "group2_properties", + "group3_properties", + "group4_properties", + ] + for column in columns: + execute_op_clickhouse( + query_id=query_id, + sql=f"ALTER TABLE {EVENTS_DATA_TABLE()} ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' MODIFY COLUMN {column} VARCHAR Codec({codec})", + ) + + def _wait_for_mutation_done(self, query_id): + # :KLUDGE: mutations_sync does not work with ON CLUSTER queries, causing race conditions with subsequent steps + sleep_until_finished("events table backill", lambda: self._count_running_mutations() > 0) + + def _count_running_mutations(self): + return sync_execute( + """ + SELECT count() + FROM clusterAllReplicas(%(cluster)s, system, 'mutations') + WHERE not is_done AND command LIKE %(pattern)s + """, + {"cluster": settings.CLICKHOUSE_CLUSTER, "pattern": "%person_properties = dictGetString%",}, + )[0][0] + + def _clear_temporary_tables(self, query_id): + queries = [ + f"DROP DICTIONARY IF EXISTS person_dict {{on_cluster_clause}}", + f"DROP DICTIONARY IF EXISTS person_distinct_id2_dict {{on_cluster_clause}}", + f"DROP DICTIONARY IF EXISTS groups_dict {{on_cluster_clause}}", + f"DROP TABLE IF EXISTS {TEMPORARY_PERSONS_TABLE_NAME} {{on_cluster_clause}}", + f"DROP TABLE IF EXISTS {TEMPORARY_PDI2_TABLE_NAME} {{on_cluster_clause}}", + f"DROP TABLE IF EXISTS {TEMPORARY_GROUPS_TABLE_NAME} {{on_cluster_clause}}", + ] + for query in queries: + execute_op_clickhouse(query_id=query_id, sql=query, per_shard=True) + + def healthcheck(self): + result = sync_execute("SELECT free_space FROM system.disks") + # 100mb or less left + if int(result[0][0]) < 100000000: + return (False, "ClickHouse available storage below 100MB") + + return (True, None) diff --git a/posthog/async_migrations/test/test_0004_replicated_schema.py b/posthog/async_migrations/test/test_0004_replicated_schema.py index 4ae1f8fbb43c01..ec8fba083350e0 100644 --- a/posthog/async_migrations/test/test_0004_replicated_schema.py +++ b/posthog/async_migrations/test/test_0004_replicated_schema.py @@ -30,7 +30,7 @@ def _create_event(**kwargs): class Test0004ReplicatedSchema(AsyncMigrationBaseTest, ClickhouseTestMixin): def setUp(self): - self.recreate_database() + self.recreate_database(replication=False) sync_execute(KAFKA_EVENTS_TABLE_SQL()) sync_execute(KAFKA_SESSION_RECORDING_EVENTS_TABLE_SQL()) sync_execute(KAFKA_GROUPS_TABLE_SQL()) @@ -40,12 +40,11 @@ def setUp(self): sync_execute(KAFKA_DEAD_LETTER_QUEUE_TABLE_SQL()) def tearDown(self): - self.recreate_database() - settings.CLICKHOUSE_REPLICATION = True + self.recreate_database(replication=True) super().tearDown() - def recreate_database(self): - settings.CLICKHOUSE_REPLICATION = False + def recreate_database(self, replication: bool): + settings.CLICKHOUSE_REPLICATION = replication sync_execute(f"DROP DATABASE {settings.CLICKHOUSE_DATABASE} SYNC") sync_execute(f"CREATE DATABASE {settings.CLICKHOUSE_DATABASE}") create_clickhouse_tables(0) diff --git a/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py b/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py new file mode 100644 index 00000000000000..d9a9b868f11d02 --- /dev/null +++ b/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py @@ -0,0 +1,299 @@ +import json +from typing import Dict, List + +import pytest +from django.test import override_settings + +from posthog.async_migrations.runner import start_async_migration +from posthog.async_migrations.setup import get_async_migration_definition, setup_async_migrations +from posthog.async_migrations.test.util import AsyncMigrationBaseTest +from posthog.client import query_with_columns, sync_execute +from posthog.models import Person +from posthog.models.async_migration import AsyncMigration, MigrationStatus +from posthog.models.event.util import create_event +from posthog.models.group.util import create_group +from posthog.models.person.util import create_person, create_person_distinct_id, delete_person +from posthog.models.utils import UUIDT +from posthog.test.base import ClickhouseTestMixin, run_clickhouse_statement_in_parallel + +MIGRATION_NAME = "0006_persons_and_groups_on_events_backfill" + +uuid1, uuid2, uuid3 = [UUIDT() for _ in range(3)] +# Clickhouse leaves behind blank/zero values for non-filled columns, these are checked against these constants +ZERO_UUID = UUIDT(uuid_str="00000000-0000-0000-0000-000000000000") +ZERO_DATE = "1970-01-01T00:00:00Z" + + +def run_migration(): + setup_async_migrations(ignore_posthog_version=True) + return start_async_migration(MIGRATION_NAME, ignore_posthog_version=True) + + +def query_events() -> List[Dict]: + return query_with_columns( + """ + SELECT + distinct_id, + person_id, + person_properties, + group0_properties, + group1_properties, + group2_properties, + group3_properties, + group4_properties, + $group_0, + $group_1, + $group_2, + $group_3, + $group_4, + formatDateTime(events.person_created_at, %(format)s) AS person_created_at, + formatDateTime(events.group0_created_at, %(format)s) AS group0_created_at, + formatDateTime(events.group1_created_at, %(format)s) AS group1_created_at, + formatDateTime(events.group2_created_at, %(format)s) AS group2_created_at, + formatDateTime(events.group3_created_at, %(format)s) AS group3_created_at, + formatDateTime(events.group4_created_at, %(format)s) AS group4_created_at + FROM events + ORDER BY distinct_id + """, + {"format": "%Y-%m-%dT%H:%M:%SZ"}, + ) + + +@pytest.mark.async_migrations +@override_settings(MULTI_TENANCY=True) +class Test0006PersonsAndGroupsOnEventsBackfill(AsyncMigrationBaseTest, ClickhouseTestMixin): + def setUp(self): + self.clear_tables() + super().setUp() + + def tearDown(self): + self.clear_tables() + super().tearDown() + + @classmethod + def tearDownClass(cls): + sync_execute("ALTER TABLE sharded_events MODIFY COLUMN person_properties VARCHAR CODEC(ZSTD(3))") + + def clear_tables(self): + run_clickhouse_statement_in_parallel( + [ + "TRUNCATE TABLE sharded_events", + "DROP TABLE IF EXISTS tmp_person_0006", + "DROP TABLE IF EXISTS tmp_person_distinct_id2_0006", + "DROP TABLE IF EXISTS tmp_groups_0006", + "DROP DICTIONARY IF EXISTS person_dict", + "DROP DICTIONARY IF EXISTS person_distinct_id2_dict", + "DROP DICTIONARY IF EXISTS groups_dict", + "ALTER TABLE sharded_events MODIFY COLUMN person_properties VARCHAR CODEC(LZ4)", + ] + ) + + def test_is_required(self): + definition = get_async_migration_definition(MIGRATION_NAME) + self.assertTrue(definition.is_required()) + + run_migration() + self.assertFalse(definition.is_required()) + + def test_completes_successfully(self): + self.assertTrue(run_migration()) + + def test_data_copy_persons(self): + create_event( + event_uuid=uuid1, team=self.team, distinct_id="1", event="$pageview", + ) + create_event( + event_uuid=uuid2, team=self.team, distinct_id="2", event="$pageview", + ) + create_event( + event_uuid=uuid3, team=self.team, distinct_id="3", event="$pageview", + ) + create_person( + team_id=self.team.pk, + version=0, + uuid=str(uuid1), + properties={"personprop": 1}, + timestamp="2022-01-01T00:00:00Z", + ) + create_person( + team_id=self.team.pk, + version=0, + uuid=str(uuid2), + properties={"personprop": 2}, + timestamp="2022-01-02T00:00:00Z", + ) + create_person_distinct_id(self.team.pk, "1", str(uuid1)) + create_person_distinct_id(self.team.pk, "2", str(uuid1)) + create_person_distinct_id(self.team.pk, "3", str(uuid2)) + + self.assertTrue(run_migration()) + + events = query_events() + + self.assertEqual(len(events), 3) + self.assertDictContainsSubset( + { + "distinct_id": "1", + "person_id": uuid1, + "person_properties": json.dumps({"personprop": 1}), + "person_created_at": "2022-01-01T00:00:00Z", + }, + events[0], + ) + self.assertDictContainsSubset( + { + "distinct_id": "2", + "person_id": uuid1, + "person_properties": json.dumps({"personprop": 1}), + "person_created_at": "2022-01-01T00:00:00Z", + }, + events[1], + ) + self.assertDictContainsSubset( + { + "distinct_id": "3", + "person_id": uuid2, + "person_properties": json.dumps({"personprop": 2}), + "person_created_at": "2022-01-02T00:00:00Z", + }, + events[2], + ) + + def test_duplicated_data_persons(self): + create_event( + event_uuid=uuid1, team=self.team, distinct_id="1", event="$pageview", + ) + create_person_distinct_id(self.team.pk, "1", str(uuid1)) + create_person( + team_id=self.team.pk, + version=1, + uuid=str(uuid1), + properties={"personprop": 2}, + timestamp="2022-01-02T00:00:00Z", + ) + create_person( + team_id=self.team.pk, + version=0, + uuid=str(uuid1), + properties={"personprop": 1}, + timestamp="2022-01-01T00:00:00Z", + ) + + self.assertTrue(run_migration()) + + events = query_events() + self.assertEqual(len(events), 1) + self.assertDictContainsSubset( + { + "distinct_id": "1", + "person_id": uuid1, + "person_properties": json.dumps({"personprop": 2}), + "person_created_at": "2022-01-02T00:00:00Z", + }, + events[0], + ) + + def test_deleted_data_persons(self): + create_event( + event_uuid=uuid1, team=self.team, distinct_id="1", event="$pageview", + ) + person = Person.objects.create( + team_id=self.team.pk, + distinct_ids=["1"], + properties={"$some_prop": "something", "$another_prop": "something"}, + ) + create_person_distinct_id(self.team.pk, "1", str(person.uuid)) + delete_person(person) + + self.assertTrue(run_migration()) + + events = query_events() + self.assertEqual(len(events), 1) + self.assertDictContainsSubset( + {"distinct_id": "1", "person_id": ZERO_UUID, "person_properties": "", "person_created_at": ZERO_DATE,}, + events[0], + ) + + def test_data_copy_groups(self): + create_group( + team_id=self.team.pk, + group_type_index=0, + group_key="org:5", + properties={"industry": "finance"}, + timestamp="2022-01-01T00:00:00Z", + ) + create_group( + team_id=self.team.pk, + group_type_index=0, + group_key="org:7", + properties={"industry": "IT"}, + timestamp="2022-01-02T00:00:00Z", + ) + create_group( + team_id=self.team.pk, + group_type_index=2, + group_key="77", + properties={"index": 2}, + timestamp="2022-01-03T00:00:00Z", + ) + create_group( + team_id=self.team.pk, + group_type_index=3, + group_key="77", + properties={"index": 3}, + timestamp="2022-01-04T00:00:00Z", + ) + + create_event( + event_uuid=uuid1, + team=self.team, + distinct_id="1", + event="$pageview", + properties={"$group_0": "org:7", "$group_1": "77", "$group_2": "77", "$group_3": "77",}, + ) + + self.assertTrue(run_migration()) + + events = query_events() + self.assertEqual(len(events), 1) + self.assertDictContainsSubset( + { + "$group_0": "org:7", + "group0_properties": json.dumps({"industry": "IT"}), + "group0_created_at": "2022-01-02T00:00:00Z", + "$group_1": "77", + "group1_properties": "", + "group1_created_at": ZERO_DATE, + "$group_2": "77", + "group2_properties": json.dumps({"index": 2}), + "group2_created_at": "2022-01-03T00:00:00Z", + "$group_3": "77", + "group3_properties": json.dumps({"index": 3}), + "group3_created_at": "2022-01-04T00:00:00Z", + "$group_4": "", + "group4_properties": "", + "group4_created_at": ZERO_DATE, + }, + events[0], + ) + + def test_no_extra_tables(self): + initial_table_count = sync_execute("SELECT count() FROM system.tables")[0][0] + initial_dictionary_count = sync_execute("SELECT count() FROM system.dictionaries")[0][0] + + run_migration() + + new_table_count = sync_execute("SELECT count() FROM system.tables")[0][0] + new_dictionary_count = sync_execute("SELECT count() FROM system.dictionaries")[0][0] + self.assertEqual(initial_table_count, new_table_count) + self.assertEqual(initial_dictionary_count, new_dictionary_count) + + def test_rollback(self): + migration = get_async_migration_definition(MIGRATION_NAME) + + self.assertEqual(len(migration.operations), 18) + migration.operations[-1].fn = lambda _: 0 / 0 # type: ignore + + migration_successful = run_migration() + self.assertFalse(migration_successful) + self.assertEqual(AsyncMigration.objects.get(name=MIGRATION_NAME).status, MigrationStatus.RolledBack) diff --git a/posthog/async_migrations/test/test_migrations_not_required.py b/posthog/async_migrations/test/test_migrations_not_required.py index 03a12caec22f26..5883f7ea405b27 100644 --- a/posthog/async_migrations/test/test_migrations_not_required.py +++ b/posthog/async_migrations/test/test_migrations_not_required.py @@ -15,4 +15,4 @@ def setUp(self): def test_async_migrations_not_required_on_fresh_instances(self): for name, migration in ALL_ASYNC_MIGRATIONS.items(): - self.assertFalse(migration.is_required()) + self.assertFalse(migration.is_required(), f"Async migration {name} is_required returned True") diff --git a/posthog/async_migrations/utils.py b/posthog/async_migrations/utils.py index cfc9ab00c061da..1af871525ddb81 100644 --- a/posthog/async_migrations/utils.py +++ b/posthog/async_migrations/utils.py @@ -1,5 +1,6 @@ +import asyncio from datetime import datetime -from typing import Optional +from typing import Callable, Optional import posthoganalytics import structlog @@ -25,7 +26,7 @@ logger = structlog.get_logger(__name__) -SLEEP_TIME_SECONDS = 20 +SLEEP_TIME_SECONDS = 20 if not TEST else 1 def send_analytics_to_posthog(event, data): @@ -74,22 +75,30 @@ def execute_op_clickhouse( client._request_information = None -def execute_on_each_shard(sql, args, settings=None) -> None: +def execute_on_each_shard(sql: str, args=None, settings=None) -> None: """ Executes query on each shard separately (if enabled) or on the cluster as a whole (if not enabled). Note that the shard selection is stable - subsequent queries are guaranteed to hit the same shards! """ - if "{on_cluster_clause}" not in sql: - raise Exception("SQL must include {on_cluster_clause} to allow execution on each shard") if CLICKHOUSE_ALLOW_PER_SHARD_EXECUTION: sql = sql.format(on_cluster_clause="") else: sql = sql.format(on_cluster_clause=f"ON CLUSTER '{CLICKHOUSE_CLUSTER}'") - for _, _, connection in _get_all_shard_connections(): - connection.execute(sql, args, settings=settings) + async def run_on_all_shards(): + tasks = [] + for _, _, connection_pool in _get_all_shard_connections(): + tasks.append(asyncio.create_task(run_on_connection(connection_pool))) + + await asyncio.wait(tasks) + + async def run_on_connection(connection_pool): + with connection_pool.get_client() as connection: + connection.execute(sql, args, settings=settings) + + asyncio.run(run_on_all_shards()) def _get_all_shard_connections(): @@ -108,11 +117,9 @@ def _get_all_shard_connections(): ) for shard, host in rows: ch_pool = make_ch_pool(host=host) - with ch_pool.get_client() as connection: - yield shard, host, connection + yield shard, host, ch_pool else: - with default_ch_pool.get_client() as connection: - yield None, None, connection + yield None, None, default_ch_pool def execute_op_postgres(sql: str, query_id: str): @@ -130,21 +137,23 @@ def _get_number_running_on_cluster(query_pattern: str) -> int: """ SELECT count() FROM clusterAllReplicas(%(cluster)s, system, 'processes') - WHERE query LIKE %(query_pattern)s + WHERE query LIKE %(query_pattern)s AND query NOT LIKE '%%clusterAllReplicas%%' """, {"cluster": CLICKHOUSE_CLUSTER, "query_pattern": query_pattern}, )[0][0] -def _sleep_until_finished(query_pattern: str) -> None: +def sleep_until_finished(name, is_running: Callable[[], bool]) -> None: from time import sleep - while _get_number_running_on_cluster(query_pattern) > 0: - logger.debug("Query still running, waiting until it's complete", query_pattern=query_pattern) + while is_running(): + logger.debug("Operation still running, waiting until it's complete", name=name) sleep(SLEEP_TIME_SECONDS) -def run_optimize_table(unique_name: str, query_id: str, table_name: str, deduplicate=False, final=False): +def run_optimize_table( + *, unique_name: str, query_id: str, table_name: str, deduplicate=False, final=False, per_shard=False +): """ Runs the passed OPTIMIZE TABLE query. @@ -152,17 +161,20 @@ def run_optimize_table(unique_name: str, query_id: str, table_name: str, dedupli we'll wait for that to complete first. """ if not TEST and _get_number_running_on_cluster(f"%%optimize:{unique_name}%%") > 0: - _sleep_until_finished(f"%%optimize:{unique_name}%%") + sleep_until_finished(unique_name, lambda: _get_number_running_on_cluster(f"%%optimize:{unique_name}%%") > 0) else: final_clause = "FINAL" if final else "" deduplicate_clause = "DEDUPLICATE" if deduplicate else "" - on_cluster_clause = "{on_cluster_clause}" - sql = f"OPTIMIZE TABLE {table_name} {on_cluster_clause} {final_clause} {deduplicate_clause}" + sql = f"OPTIMIZE TABLE {table_name} {{on_cluster_clause}} {final_clause} {deduplicate_clause}" + + if not per_shard: + sql = sql.format(on_cluster_clause=f"ON CLUSTER '{CLICKHOUSE_CLUSTER}'") + execute_op_clickhouse( sql, query_id=f"optimize:{unique_name}/{query_id}", settings={"max_execution_time": ASYNC_MIGRATIONS_DEFAULT_TIMEOUT_SECONDS, "mutations_sync": 2}, - per_shard=True, + per_shard=per_shard, ) diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index 1ebbf52b57e85e..7d4a12f1bd5a70 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -13,12 +13,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, @@ -71,12 +71,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, @@ -220,12 +220,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, @@ -419,12 +419,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, @@ -778,12 +778,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, @@ -853,12 +853,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, @@ -1086,12 +1086,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, diff --git a/posthog/models/event/sql.py b/posthog/models/event/sql.py index 6a52698a1ac23b..aa7b028e5effcd 100644 --- a/posthog/models/event/sql.py +++ b/posthog/models/event/sql.py @@ -31,12 +31,12 @@ created_at DateTime64(6, 'UTC'), person_id UUID, person_created_at DateTime64, - person_properties VARCHAR, - group0_properties VARCHAR, - group1_properties VARCHAR, - group2_properties VARCHAR, - group3_properties VARCHAR, - group4_properties VARCHAR, + person_properties VARCHAR Codec(ZSTD(3)), + group0_properties VARCHAR Codec(ZSTD(3)), + group1_properties VARCHAR Codec(ZSTD(3)), + group2_properties VARCHAR Codec(ZSTD(3)), + group3_properties VARCHAR Codec(ZSTD(3)), + group4_properties VARCHAR Codec(ZSTD(3)), group0_created_at DateTime64, group1_created_at DateTime64, group2_created_at DateTime64, diff --git a/posthog/models/group/util.py b/posthog/models/group/util.py index 707ae3d23c16e5..13bb42b0e3f439 100644 --- a/posthog/models/group/util.py +++ b/posthog/models/group/util.py @@ -1,7 +1,9 @@ import datetime import json -from typing import Dict, Optional +from typing import Dict, Optional, Union +import pytz +from dateutil.parser import isoparse from django.utils.timezone import now from posthog.kafka_client.client import ClickhouseProducer @@ -35,16 +37,28 @@ def create_group( group_type_index: GroupTypeIndex, group_key: str, properties: Optional[Dict] = None, - timestamp: Optional[datetime.datetime] = None, + timestamp: Optional[Union[datetime.datetime, str]] = None, ): """Create proper Group record (ClickHouse + Postgres).""" if not properties: properties = {} if not timestamp: timestamp = now() + + # clickhouse specific formatting + if isinstance(timestamp, str): + timestamp = isoparse(timestamp) + else: + timestamp = timestamp.astimezone(pytz.utc) + raw_create_group_ch(team_id, group_type_index, group_key, properties, timestamp) Group.objects.create( - team_id=team_id, group_type_index=group_type_index, group_key=group_key, group_properties=properties, version=0, + team_id=team_id, + group_type_index=group_type_index, + group_key=group_key, + group_properties=properties, + created_at=timestamp, + version=0, ) diff --git a/posthog/models/person/sql.py b/posthog/models/person/sql.py index 3c74108bb33c67..906a78356aa7a8 100644 --- a/posthog/models/person/sql.py +++ b/posthog/models/person/sql.py @@ -317,7 +317,7 @@ ) INSERT_PERSON_SQL = """ -INSERT INTO person (id, created_at, team_id, properties, is_identified, _timestamp, _offset, is_deleted, version) SELECT %(id)s, %(created_at)s, %(team_id)s, %(properties)s, %(is_identified)s, %(_timestamp)s, 0, 0, 0 +INSERT INTO person (id, created_at, team_id, properties, is_identified, _timestamp, _offset, is_deleted, version) SELECT %(id)s, %(created_at)s, %(team_id)s, %(properties)s, %(is_identified)s, %(_timestamp)s, 0, 0, %(version)s """ INSERT_PERSON_BULK_SQL = """ diff --git a/posthog/models/person/util.py b/posthog/models/person/util.py index 2e0542be5b8299..f40b40cc1cf651 100644 --- a/posthog/models/person/util.py +++ b/posthog/models/person/util.py @@ -3,6 +3,8 @@ from contextlib import ExitStack from typing import Dict, List, Optional, Union +import pytz +from dateutil.parser import isoparse from django.db.models.query import QuerySet from django.db.models.signals import post_delete, post_save from django.dispatch import receiver @@ -100,7 +102,7 @@ def create_person( properties: Optional[Dict] = {}, sync: bool = False, is_identified: bool = False, - timestamp: Optional[datetime.datetime] = None, + timestamp: Optional[Union[datetime.datetime, str]] = None, ) -> str: if uuid: uuid = str(uuid) @@ -109,6 +111,12 @@ def create_person( if not timestamp: timestamp = now() + # clickhouse specific formatting + if isinstance(timestamp, str): + timestamp = isoparse(timestamp) + else: + timestamp = timestamp.astimezone(pytz.utc) + data = { "id": str(uuid), "team_id": team_id, From 6457f0296b2b0cccefc7edeccf495a56c1c099d3 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 27 Jul 2022 13:26:50 +0300 Subject: [PATCH 179/213] fix(ingestion): Change overrides order when parsing Kafka messages (#10998) * Move formPluginEvent * Work around risky behavior Previously, users could override some important event fields by passing values in their payload. This bug was introduced way back in https://github.com/PostHog/plugin-server/pull/34 This bug indirectly caused the following sentry errors: - https://sentry.io/organizations/posthog2/issues/3289550563/?project=6423401&query=is%3Aunresolved+level%3Aerror - https://sentry.io/organizations/posthog2/issues/3455742732/?project=6423401&query=is%3Aunresolved+level%3Aerror - https://sentry.io/organizations/posthog2/issues/3382895905/?project=6423401&query=is%3Aunresolved+level%3Aerror One area I'm unsure on is specifically `ip` field and its expected behavior, but looking at old code from 2020 it seems we always took the ip from request rather than looking at event body. --- .../batch-processing/each-batch-ingestion.ts | 14 +-- plugin-server/src/utils/event.ts | 13 +++ plugin-server/tests/utils/event.test.ts | 95 ++++++++++++++++++- 3 files changed, 108 insertions(+), 14 deletions(-) diff --git a/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts b/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts index d2fa29d0f1f848..29f376b1fab936 100644 --- a/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts +++ b/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts @@ -2,23 +2,11 @@ import { PluginEvent } from '@posthog/plugin-scaffold' import { EachBatchPayload, KafkaMessage } from 'kafkajs' import { Hub, WorkerMethods } from '../../../types' -import { normalizeEvent } from '../../../utils/event' +import { formPluginEvent } from '../../../utils/event' import { status } from '../../../utils/status' import { KafkaQueue } from '../kafka-queue' import { eachBatch } from './each-batch' -export function formPluginEvent(message: KafkaMessage): PluginEvent { - // TODO: inefficient to do this twice? - const { data: dataStr, ...rawEvent } = JSON.parse(message.value!.toString()) - const combinedEvent = { ...rawEvent, ...JSON.parse(dataStr) } - const event: PluginEvent = normalizeEvent({ - ...combinedEvent, - site_url: combinedEvent.site_url || null, - ip: combinedEvent.ip || null, - }) - return event -} - export async function eachMessageIngestion(message: KafkaMessage, queue: KafkaQueue): Promise { await ingestEvent(queue.pluginsServer, queue.workerMethods, formPluginEvent(message)) } diff --git a/plugin-server/src/utils/event.ts b/plugin-server/src/utils/event.ts index d1881443f133e0..d8c97131f32307 100644 --- a/plugin-server/src/utils/event.ts +++ b/plugin-server/src/utils/event.ts @@ -1,4 +1,5 @@ import { PluginEvent, ProcessedPluginEvent } from '@posthog/plugin-scaffold' +import { KafkaMessage } from 'kafkajs' import { ClickhouseEventKafka, IngestionEvent } from '../types' import { chainToElements } from './db/elements-chain' @@ -52,3 +53,15 @@ export function normalizeEvent(event: PluginEvent): PluginEvent { event.properties = properties return event } + +export function formPluginEvent(message: KafkaMessage): PluginEvent { + // TODO: inefficient to do this twice? + const { data: dataStr, ...rawEvent } = JSON.parse(message.value!.toString()) + const combinedEvent = { ...JSON.parse(dataStr), ...rawEvent } + const event: PluginEvent = normalizeEvent({ + ...combinedEvent, + site_url: combinedEvent.site_url || null, + ip: combinedEvent.ip || null, + }) + return event +} diff --git a/plugin-server/tests/utils/event.test.ts b/plugin-server/tests/utils/event.test.ts index 756cb072364e44..33127aa433e904 100644 --- a/plugin-server/tests/utils/event.test.ts +++ b/plugin-server/tests/utils/event.test.ts @@ -1,4 +1,6 @@ -import { normalizeEvent } from '../../src/utils/event' +import { KafkaMessage } from 'kafkajs' + +import { formPluginEvent, normalizeEvent } from '../../src/utils/event' describe('normalizeEvent()', () => { describe('distinctId', () => { @@ -41,3 +43,94 @@ describe('normalizeEvent()', () => { }) }) }) + +describe('formPluginEvent()', () => { + it('forms pluginEvent from a raw message', () => { + const message = { + value: Buffer.from( + JSON.stringify({ + uuid: '01823e89-f75d-0000-0d4d-3d43e54f6de5', + distinct_id: 'some_distinct_id', + ip: null, + site_url: 'http://example.com', + team_id: 2, + now: '2020-02-23T02:15:00Z', + sent_at: '2020-02-23T02:15:00Z', + data: JSON.stringify({ + event: 'some-event', + properties: { foo: 123 }, + timestamp: '2020-02-24T02:15:00Z', + offset: 0, + $set: {}, + $set_once: {}, + }), + }) + ), + } as any as KafkaMessage + + expect(formPluginEvent(message)).toEqual({ + uuid: '01823e89-f75d-0000-0d4d-3d43e54f6de5', + distinct_id: 'some_distinct_id', + ip: null, + site_url: 'http://example.com', + team_id: 2, + now: '2020-02-23T02:15:00Z', + sent_at: '2020-02-23T02:15:00Z', + event: 'some-event', + properties: { foo: 123, $set: {}, $set_once: {} }, + timestamp: '2020-02-24T02:15:00Z', + offset: 0, + $set: {}, + $set_once: {}, + }) + }) + + it('does not override risky values', () => { + const message = { + value: Buffer.from( + JSON.stringify({ + uuid: '01823e89-f75d-0000-0d4d-3d43e54f6de5', + distinct_id: 'some_distinct_id', + ip: null, + site_url: 'http://example.com', + team_id: 2, + now: '2020-02-23T02:15:00Z', + sent_at: '2020-02-23T02:15:00Z', + data: JSON.stringify({ + // Risky overrides + uuid: 'bad-uuid', + distinct_id: 'bad long id', + ip: '192.168.0.1', + site_url: 'http://foo.com', + team_id: 456, + now: 'bad timestamp', + sent_at: 'bad timestamp', + // ... + event: 'some-event', + properties: { foo: 123 }, + timestamp: '2020-02-24T02:15:00Z', + offset: 0, + $set: {}, + $set_once: {}, + }), + }) + ), + } as any as KafkaMessage + + expect(formPluginEvent(message)).toEqual({ + uuid: '01823e89-f75d-0000-0d4d-3d43e54f6de5', + distinct_id: 'some_distinct_id', + ip: null, + site_url: 'http://example.com', + team_id: 2, + now: '2020-02-23T02:15:00Z', + sent_at: '2020-02-23T02:15:00Z', + event: 'some-event', + properties: { foo: 123, $set: {}, $set_once: {} }, + timestamp: '2020-02-24T02:15:00Z', + offset: 0, + $set: {}, + $set_once: {}, + }) + }) +}) From f35ef65ddcdf255eada34bd7af7c05485e5e8072 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 27 Jul 2022 13:29:41 +0300 Subject: [PATCH 180/213] feat(person-on-events): Add instance setting to show/hide 0006 async migration (#10758) * feat(person-on-events): Add instance setting to show/hide 0006 async migration We'll be doing a release soon, and the 0006 migration will not be ready for public consumption yet. This adds an instance setting for hiding the 0006 async migration * Update tests * Verify not skipping * Handle feedback * Fixup --- .../AsyncMigrations/asyncMigrationsLogic.ts | 3 +++ posthog/api/instance_settings.py | 13 ++++++++++- posthog/async_migrations/definition.py | 5 ++++ ...6_persons_and_groups_on_events_backfill.py | 7 +++--- posthog/async_migrations/setup.py | 23 ++++++++++++------- ...6_persons_and_groups_on_events_backfill.py | 2 -- .../commands/run_async_migrations.py | 12 ++++------ posthog/settings/dynamic_settings.py | 6 +++++ 8 files changed, 49 insertions(+), 22 deletions(-) diff --git a/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts b/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts index dc3f0e257ce989..31d04638ce170d 100644 --- a/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts +++ b/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts @@ -3,6 +3,7 @@ import { kea } from 'kea' import { userLogic } from 'scenes/userLogic' import type { asyncMigrationsLogicType } from './asyncMigrationsLogicType' +import { systemStatusLogic } from 'scenes/instance/SystemStatus/systemStatusLogic' import { InstanceSetting } from '~/types' import { lemonToast } from 'lib/components/lemonToast' export type TabName = 'overview' | 'internal_metrics' @@ -187,6 +188,8 @@ export const asyncMigrationsLogic = kea({ }) lemonToast.success(`Instance setting ${settingKey} updated`) actions.loadAsyncMigrationSettings() + actions.loadAsyncMigrations() + systemStatusLogic.actions.loadSystemStatus() } catch { lemonToast.error('Failed to trigger migration') } diff --git a/posthog/api/instance_settings.py b/posthog/api/instance_settings.py index ed4ca70be24aad..821f4546c61a76 100644 --- a/posthog/api/instance_settings.py +++ b/posthog/api/instance_settings.py @@ -6,7 +6,13 @@ from posthog.models.instance_setting import get_instance_setting as get_instance_setting_raw from posthog.models.instance_setting import set_instance_setting as set_instance_setting_raw from posthog.permissions import IsStaffUser -from posthog.settings import CONSTANCE_CONFIG, MULTI_TENANCY, SECRET_SETTINGS, SETTINGS_ALLOWING_API_OVERRIDE +from posthog.settings import ( + CONSTANCE_CONFIG, + MULTI_TENANCY, + SECRET_SETTINGS, + SETTINGS_ALLOWING_API_OVERRIDE, + SKIP_ASYNC_MIGRATIONS_SETUP, +) from posthog.utils import str_to_bool @@ -89,6 +95,11 @@ def update(self, instance: InstanceSettingHelper, validated_data: Dict[str, Any] from posthog.tasks.email import send_canary_email send_canary_email.apply_async(kwargs={"user_email": self.context["request"].user.email}) + elif instance.key.startswith("ASYNC_MIGRATION"): + from posthog.async_migrations.setup import setup_async_migrations + + if not SKIP_ASYNC_MIGRATIONS_SETUP: + setup_async_migrations() return instance diff --git a/posthog/async_migrations/definition.py b/posthog/async_migrations/definition.py index 23b7d108789aea..9e25430c5d8e83 100644 --- a/posthog/async_migrations/definition.py +++ b/posthog/async_migrations/definition.py @@ -98,6 +98,11 @@ class AsyncMigrationDefinition: # name of async migration this migration depends on depends_on: Optional[str] = None + # run before creating the migration model. Returns a boolean specifying if the instance should + # set up the AsyncMigration model and show this migration in the UI + def is_hidden(self) -> bool: + return False + # will be run before starting the migration, return a boolean specifying if the instance needs this migration # e.g. instances where fresh setups are already set up correctly def is_required(self) -> bool: diff --git a/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py b/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py index 695f320a3b5790..4a3bd9a2e1f7a6 100644 --- a/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py +++ b/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py @@ -12,6 +12,7 @@ from posthog.async_migrations.utils import execute_op_clickhouse, run_optimize_table, sleep_until_finished from posthog.client import sync_execute from posthog.models.event.sql import EVENTS_DATA_TABLE +from posthog.models.instance_setting import get_instance_setting logger = structlog.get_logger(__name__) @@ -61,10 +62,10 @@ class Migration(AsyncMigrationDefinition): depends_on = "0005_person_replacing_by_version" - def precheck(self): - if not settings.MULTI_TENANCY: - return False, "This async migration is not yet ready for self-hosted users" + def is_hidden(self) -> bool: + return not (get_instance_setting("ASYNC_MIGRATIONS_SHOW_PERSON_ON_EVENTS_MIGRATION") or settings.TEST) + def precheck(self): return analyze_enough_disk_space_free_for_table(EVENTS_DATA_TABLE(), required_ratio=2.0) def is_required(self) -> bool: diff --git a/posthog/async_migrations/setup.py b/posthog/async_migrations/setup.py index 4d2fc9e63386b1..eb7b175bb42192 100644 --- a/posthog/async_migrations/setup.py +++ b/posthog/async_migrations/setup.py @@ -47,14 +47,7 @@ def setup_async_migrations(ignore_posthog_version: bool = False): first_migration = None for migration_name, migration in ALL_ASYNC_MIGRATIONS.items(): - - sm = AsyncMigration.objects.get_or_create(name=migration_name)[0] - - sm.description = migration.description - sm.posthog_max_version = migration.posthog_max_version - sm.posthog_min_version = migration.posthog_min_version - - sm.save() + setup_model(migration_name, migration) dependency = migration.depends_on @@ -82,6 +75,20 @@ def setup_async_migrations(ignore_posthog_version: bool = False): kickstart_migration_if_possible(first_migration, applied_migrations) +def setup_model(migration_name: str, migration: AsyncMigrationDefinition) -> Optional[AsyncMigration]: + if migration.is_hidden(): + return None + + sm = AsyncMigration.objects.get_or_create(name=migration_name)[0] + + sm.description = migration.description + sm.posthog_max_version = migration.posthog_max_version + sm.posthog_min_version = migration.posthog_min_version + + sm.save() + return sm + + def kickstart_migration_if_possible(migration_name: str, applied_migrations: set): """ Find the last completed migration, look for a migration that depends on it, and try to run it diff --git a/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py b/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py index d9a9b868f11d02..5a0e1b1e522014 100644 --- a/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py +++ b/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py @@ -2,7 +2,6 @@ from typing import Dict, List import pytest -from django.test import override_settings from posthog.async_migrations.runner import start_async_migration from posthog.async_migrations.setup import get_async_migration_definition, setup_async_migrations @@ -60,7 +59,6 @@ def query_events() -> List[Dict]: @pytest.mark.async_migrations -@override_settings(MULTI_TENANCY=True) class Test0006PersonsAndGroupsOnEventsBackfill(AsyncMigrationBaseTest, ClickhouseTestMixin): def setUp(self): self.clear_tables() diff --git a/posthog/management/commands/run_async_migrations.py b/posthog/management/commands/run_async_migrations.py index 5d5c34441a921e..53f547180736f9 100644 --- a/posthog/management/commands/run_async_migrations.py +++ b/posthog/management/commands/run_async_migrations.py @@ -4,9 +4,8 @@ from semantic_version.base import Version from posthog.async_migrations.runner import complete_migration, is_migration_dependency_fulfilled, start_async_migration -from posthog.async_migrations.setup import ALL_ASYNC_MIGRATIONS, POSTHOG_VERSION, setup_async_migrations +from posthog.async_migrations.setup import ALL_ASYNC_MIGRATIONS, POSTHOG_VERSION, setup_async_migrations, setup_model from posthog.models.async_migration import ( - AsyncMigration, AsyncMigrationError, MigrationStatus, get_async_migrations_by_status, @@ -22,13 +21,10 @@ def get_necessary_migrations(): for migration_name, definition in sorted(ALL_ASYNC_MIGRATIONS.items()): if is_async_migration_complete(migration_name): continue - sm = AsyncMigration.objects.get_or_create(name=migration_name)[0] - sm.description = definition.description - sm.posthog_max_version = definition.posthog_max_version - sm.posthog_min_version = definition.posthog_min_version - - sm.save() + sm = setup_model(migration_name, definition) + if sm is None: + continue is_migration_required = ALL_ASYNC_MIGRATIONS[migration_name].is_required() diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.py index 9f38e2e6082247..9c9577e17ca650 100644 --- a/posthog/settings/dynamic_settings.py +++ b/posthog/settings/dynamic_settings.py @@ -61,6 +61,11 @@ "(Advanced) Whether having an async migration running, errored or required should prevent upgrades.", bool, ), + "ASYNC_MIGRATIONS_SHOW_PERSON_ON_EVENTS_MIGRATION": ( + get_from_env("ASYNC_MIGRATIONS_SHOW_PERSON_ON_EVENTS_MIGRATION", False, type_cast=str_to_bool), + "(Advanced) Whether to show the experimental 0006 async migration.", + bool, + ), "STRICT_CACHING_TEAMS": ( get_from_env("STRICT_CACHING_TEAMS", ""), "Whether to always try to find cached data for historical intervals on trends", @@ -144,6 +149,7 @@ "ASYNC_MIGRATIONS_DISABLE_AUTO_ROLLBACK", "ASYNC_MIGRATIONS_AUTO_CONTINUE", "ASYNC_MIGRATIONS_BLOCK_UPGRADE", + "ASYNC_MIGRATIONS_SHOW_PERSON_ON_EVENTS_MIGRATION", "EMAIL_ENABLED", "EMAIL_HOST", "EMAIL_PORT", From 5a080ebe4ef7f59aa4c559b6ecf4b2a80c50ef16 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 27 Jul 2022 13:32:54 +0300 Subject: [PATCH 181/213] chore(plugin-server): silence parseMemberAssignment errors (#10970) These errors cause sentry noise and dont cause any issues - lets ignore for now --- plugin-server/src/main/ingestion-queues/kafka-metrics.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugin-server/src/main/ingestion-queues/kafka-metrics.ts b/plugin-server/src/main/ingestion-queues/kafka-metrics.ts index c298c647a5d841..0cea8570e99ed6 100644 --- a/plugin-server/src/main/ingestion-queues/kafka-metrics.ts +++ b/plugin-server/src/main/ingestion-queues/kafka-metrics.ts @@ -1,4 +1,3 @@ -import * as Sentry from '@sentry/node' import { StatsD } from 'hot-shots' import { Consumer } from 'kafkajs' @@ -95,8 +94,6 @@ export async function emitConsumerGroupMetrics( memberId: consumerGroupMemberId || 'unknown', instanceId: pluginsServer.instanceId.toString(), }) - - Sentry.captureException(error) } } From 84b5ae364ffc30c89034c84c7d12703c35687639 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 27 Jul 2022 13:36:31 +0300 Subject: [PATCH 182/213] feat(async-migrations): Allow parameterizing async migrations (#10790) * Add async migration parameters * Make name of migration known to async migration * WIP parameterization system * Add missing type * Resolve circular imports elsehow * Update FE typing a bit * Update BE typing * Unify code in logic * Allow updating migration parameters in the UI * Solve typing-related issues * Use parameters from UI in async migration * Add tests for get_parameter * Add api tests * Ignore migration safety * update a test * resolve typing issue * Simplify a test --- .../AsyncMigrationParametersModal.tsx | 78 ++++++++++++ .../AsyncMigrations/AsyncMigrations.tsx | 26 ++-- .../asyncMigrationParameterFormLogic.ts | 40 ++++++ .../AsyncMigrations/asyncMigrationsLogic.ts | 92 ++++++++------ latest_migrations.manifest | 2 +- posthog/api/async_migration.py | 18 ++- posthog/api/test/test_async_migrations.py | 16 +-- posthog/async_migrations/definition.py | 43 ++++--- .../0005_person_replacing_by_version.py | 4 +- ...6_persons_and_groups_on_events_backfill.py | 118 ++++++++++-------- posthog/async_migrations/runner.py | 2 +- posthog/async_migrations/setup.py | 4 +- ...6_persons_and_groups_on_events_backfill.py | 1 - .../async_migrations/test/test_definition.py | 29 ++++- posthog/async_migrations/test/test_runner.py | 2 +- .../0253_add_async_migration_parameters.py | 20 +++ posthog/models/async_migration.py | 2 + posthog/tasks/test/test_async_migrations.py | 2 +- 18 files changed, 364 insertions(+), 135 deletions(-) create mode 100644 frontend/src/scenes/instance/AsyncMigrations/AsyncMigrationParametersModal.tsx create mode 100644 frontend/src/scenes/instance/AsyncMigrations/asyncMigrationParameterFormLogic.ts create mode 100644 posthog/migrations/0253_add_async_migration_parameters.py diff --git a/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrationParametersModal.tsx b/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrationParametersModal.tsx new file mode 100644 index 00000000000000..24ea62538c06ae --- /dev/null +++ b/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrationParametersModal.tsx @@ -0,0 +1,78 @@ +import React, { useState } from 'react' +import { useActions } from 'kea' +import { AsyncMigrationModalProps, asyncMigrationsLogic } from 'scenes/instance/AsyncMigrations/asyncMigrationsLogic' +import { LemonModal } from 'lib/components/LemonModal/LemonModal' +import { LemonButton } from 'lib/components/LemonButton' +import { asyncMigrationParameterFormLogic } from 'scenes/instance/AsyncMigrations/asyncMigrationParameterFormLogic' +import { Field } from 'kea-forms' +import { LemonInput } from 'lib/components/LemonInput/LemonInput' +import { VerticalForm } from 'lib/forms/VerticalForm' +import { AnimatedCollapsible } from 'lib/components/AnimatedCollapsible' + +export function AsyncMigrationParametersModal(props: AsyncMigrationModalProps): JSX.Element { + const { closeAsyncMigrationsModal } = useActions(asyncMigrationsLogic) + + const [collapsed, setCollapsed] = useState(true) + + return ( + + + + Cancel + + + Run migration + + + } + > +

+ This async migration allows tuning parameters used in the async migration. + {collapsed && ( + <> +
+ { + setCollapsed(!collapsed) + }} + > + Click here to show advanced configuration. + + + )} +

+ + + {Object.keys(props.migration.parameter_definitions).map((key) => ( + {props.migration.parameter_definitions[key][1]}}> + + + ))} + + + + ) +} diff --git a/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx b/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx index b6e142bc72a63a..e9494c3fed6ef8 100644 --- a/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx +++ b/frontend/src/scenes/instance/AsyncMigrations/AsyncMigrations.tsx @@ -22,6 +22,7 @@ import { More } from 'lib/components/LemonButton/More' import { LemonButton } from 'lib/components/LemonButton' import { LemonTag, LemonTagPropsType } from 'lib/components/LemonTag/LemonTag' import { IconRefresh, IconReplay } from 'lib/components/icons' +import { AsyncMigrationParametersModal } from 'scenes/instance/AsyncMigrations/AsyncMigrationParametersModal' const { TabPane } = Tabs @@ -34,8 +35,14 @@ const STATUS_RELOAD_INTERVAL_MS = 3000 export function AsyncMigrations(): JSX.Element { const { user } = useValues(userLogic) - const { asyncMigrations, asyncMigrationsLoading, activeTab, asyncMigrationSettings, isAnyMigrationRunning } = - useValues(asyncMigrationsLogic) + const { + asyncMigrations, + asyncMigrationsLoading, + activeTab, + asyncMigrationSettings, + isAnyMigrationRunning, + activeAsyncMigrationModal, + } = useValues(asyncMigrationsLogic) const { triggerMigration, resumeMigration, @@ -140,7 +147,7 @@ export function AsyncMigrations(): JSX.Element { @@ -151,14 +158,14 @@ export function AsyncMigrations(): JSX.Element { <> forceStopMigration(asyncMigration.id)} + onClick={() => forceStopMigration(asyncMigration)} fullWidth > Stop and rollback forceStopMigrationWithoutRollback(asyncMigration.id)} + onClick={() => forceStopMigrationWithoutRollback(asyncMigration)} fullWidth > Stop @@ -174,14 +181,14 @@ export function AsyncMigrations(): JSX.Element { <> resumeMigration(asyncMigration.id)} + onClick={() => resumeMigration(asyncMigration)} fullWidth > Resume rollbackMigration(asyncMigration.id)} + onClick={() => rollbackMigration(asyncMigration)} fullWidth > Rollback @@ -194,7 +201,7 @@ export function AsyncMigrations(): JSX.Element { } - onClick={() => triggerMigration(asyncMigration.id)} + onClick={() => triggerMigration(asyncMigration)} fullWidth /> @@ -260,6 +267,9 @@ export function AsyncMigrations(): JSX.Element { dataSource={asyncMigrations} expandable={rowExpansion} /> + {activeAsyncMigrationModal ? ( + + ) : null} ) : activeTab === AsyncMigrationsTab.Settings ? ( <> diff --git a/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationParameterFormLogic.ts b/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationParameterFormLogic.ts new file mode 100644 index 00000000000000..449af49a92dfcc --- /dev/null +++ b/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationParameterFormLogic.ts @@ -0,0 +1,40 @@ +import { kea, key, props, path } from 'kea' +import { forms } from 'kea-forms' +import { AsyncMigrationModalProps, asyncMigrationsLogic } from 'scenes/instance/AsyncMigrations/asyncMigrationsLogic' + +import type { asyncMigrationParameterFormLogicType } from './asyncMigrationParameterFormLogicType' + +export const asyncMigrationParameterFormLogic = kea([ + path(['scenes', 'instance', 'AsyncMigrations', 'asyncMigrationParameterFormLogic']), + props({} as AsyncMigrationModalProps), + key((props) => props.migration.id), + + forms(({ props }) => ({ + parameters: { + defaults: defaultParameters(props), + + submit: async (parameters: Record) => { + asyncMigrationsLogic.actions.updateMigrationStatus( + { + ...props.migration, + parameters, + }, + props.endpoint, + props.message + ) + }, + }, + })), +]) + +function defaultParameters(props: AsyncMigrationModalProps): Record { + const result = {} + Object.keys(props.migration.parameter_definitions).forEach((key) => { + if (props.migration.parameters[key]) { + result[key] = props.migration.parameters[key] + } else { + result[key] = props.migration.parameter_definitions[key][0] + } + }) + return result +} diff --git a/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts b/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts index 31d04638ce170d..fc3c76c9422548 100644 --- a/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts +++ b/frontend/src/scenes/instance/AsyncMigrations/asyncMigrationsLogic.ts @@ -53,16 +53,29 @@ export interface AsyncMigration { posthog_min_version: string posthog_max_version: string error_count: number + parameters: Record + parameter_definitions: Record +} + +export interface AsyncMigrationModalProps { + migration: AsyncMigration + endpoint: string + message: string } export const asyncMigrationsLogic = kea({ path: ['scenes', 'instance', 'AsyncMigrations', 'asyncMigrationsLogic'], actions: { - triggerMigration: (migrationId: number) => ({ migrationId }), - resumeMigration: (migrationId: number) => ({ migrationId }), - rollbackMigration: (migrationId: number) => ({ migrationId }), - forceStopMigration: (migrationId: number) => ({ migrationId }), - forceStopMigrationWithoutRollback: (migrationId: number) => ({ migrationId }), + triggerMigration: (migration: AsyncMigration) => ({ migration }), + resumeMigration: (migration: AsyncMigration) => ({ migration }), + rollbackMigration: (migration: AsyncMigration) => ({ migration }), + forceStopMigration: (migration: AsyncMigration) => ({ migration }), + forceStopMigrationWithoutRollback: (migration: AsyncMigration) => ({ migration }), + updateMigrationStatus: (migration: AsyncMigration, endpoint: string, message: string) => ({ + migration, + endpoint, + message, + }), setActiveTab: (tab: AsyncMigrationsTab) => ({ tab }), updateSetting: (settingKey: string, newValue: string) => ({ settingKey, newValue }), loadAsyncMigrationErrors: (migrationId: number) => ({ migrationId }), @@ -71,6 +84,12 @@ export const asyncMigrationsLogic = kea({ errors, }), loadAsyncMigrationErrorsFailure: (migrationId: number, error: any) => ({ migrationId, error }), + openAsyncMigrationsModal: (migration: AsyncMigration, endpoint: string, message: string) => ({ + migration, + endpoint, + message, + }), + closeAsyncMigrationsModal: true, }, reducers: { @@ -97,6 +116,14 @@ export const asyncMigrationsLogic = kea({ }, }, ], + activeAsyncMigrationModal: [ + null as AsyncMigrationModalProps | null, + { + openAsyncMigrationsModal: (_, payload) => payload, + closeAsyncMigrationsModal: () => null, + updateMigrationStatus: () => null, + }, + ], }, loaders: () => ({ asyncMigrations: [ @@ -135,46 +162,39 @@ export const asyncMigrationsLogic = kea({ }, listeners: ({ actions }) => ({ - triggerMigration: async ({ migrationId }) => { - const res = await api.create(`/api/async_migrations/${migrationId}/trigger`) - if (res.success) { - lemonToast.success('Migration triggered successfully') - actions.loadAsyncMigrations() + triggerMigration: async ({ migration }) => { + if (Object.keys(migration.parameter_definitions).length > 0) { + actions.openAsyncMigrationsModal(migration, 'trigger', 'Migration triggered successfully') } else { - lemonToast.error(res.error) + actions.updateMigrationStatus(migration, 'trigger', 'Migration triggered successfully') } }, - resumeMigration: async ({ migrationId }) => { - const res = await api.create(`/api/async_migrations/${migrationId}/resume`) - if (res.success) { - lemonToast.success('Migration resume triggered successfully') - actions.loadAsyncMigrations() + resumeMigration: async ({ migration }) => { + if (Object.keys(migration.parameter_definitions).length > 0) { + actions.openAsyncMigrationsModal(migration, 'resume', 'Migration resume triggered successfully') } else { - lemonToast.error(res.error) + actions.updateMigrationStatus(migration, 'resume', 'Migration resume triggered successfully') } }, - rollbackMigration: async ({ migrationId }) => { - const res = await api.create(`/api/async_migrations/${migrationId}/rollback`) - if (res.success) { - lemonToast.success('Migration rolledback triggered successfully') - actions.loadAsyncMigrations() - } else { - lemonToast.error(res.error) - } + rollbackMigration: async ({ migration }) => { + actions.updateMigrationStatus(migration, 'rollback', 'Migration rollback triggered successfully') }, - forceStopMigration: async ({ migrationId }) => { - const res = await api.create(`/api/async_migrations/${migrationId}/force_stop`) - if (res.success) { - lemonToast.success('Force stop triggered successfully') - actions.loadAsyncMigrations() - } else { - lemonToast.error(res.error) - } + forceStopMigration: async ({ migration }) => { + actions.updateMigrationStatus(migration, 'force_stop', 'Force stop triggered successfully') + }, + forceStopMigrationWithoutRollback: async ({ migration }) => { + actions.updateMigrationStatus( + migration, + 'force_stop_without_rollback', + 'Force stop without rollback triggered successfully' + ) }, - forceStopMigrationWithoutRollback: async ({ migrationId }) => { - const res = await api.create(`/api/async_migrations/${migrationId}/force_stop_without_rollback`) + updateMigrationStatus: async ({ migration, endpoint, message }) => { + const res = await api.create(`/api/async_migrations/${migration.id}/${endpoint}`, { + parameters: migration.parameters, + }) if (res.success) { - lemonToast.success('Force stop without rollback triggered successfully') + lemonToast.success(message) actions.loadAsyncMigrations() } else { lemonToast.error(res.error) diff --git a/latest_migrations.manifest b/latest_migrations.manifest index b28063bc88fa43..5ded1712ee24e4 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -3,7 +3,7 @@ auth: 0012_alter_user_first_name_max_length axes: 0006_remove_accesslog_trusted contenttypes: 0002_remove_content_type_name ee: 0013_silence_deprecated_tags_warnings -posthog: 0252_reset_insight_refreshing_status +posthog: 0253_add_async_migration_parameters rest_hooks: 0002_swappable_hook_model sessions: 0001_initial social_django: 0010_uid_db_index diff --git a/posthog/api/async_migration.py b/posthog/api/async_migration.py index 67ff542fb17eb3..4961f21751ad35 100644 --- a/posthog/api/async_migration.py +++ b/posthog/api/async_migration.py @@ -3,6 +3,7 @@ from posthog.api.routing import StructuredViewSetMixin from posthog.async_migrations.runner import MAX_CONCURRENT_ASYNC_MIGRATIONS, is_posthog_version_compatible +from posthog.async_migrations.setup import get_async_migration_definition from posthog.async_migrations.utils import force_stop_migration, rollback_migration, trigger_migration from posthog.models.async_migration import ( AsyncMigration, @@ -22,9 +23,7 @@ class Meta: class AsyncMigrationSerializer(serializers.ModelSerializer): error_count = serializers.SerializerMethodField() - - def get_error_count(self, async_migration: AsyncMigration): - return AsyncMigrationError.objects.filter(async_migration=async_migration).count() + parameter_definitions = serializers.SerializerMethodField() class Meta: model = AsyncMigration @@ -41,7 +40,9 @@ class Meta: "finished_at", "posthog_max_version", "posthog_min_version", + "parameters", "error_count", + "parameter_definitions", ] read_only_fields = [ "id", @@ -57,8 +58,17 @@ class Meta: "posthog_max_version", "posthog_min_version", "error_count", + "parameter_definitions", ] + def get_error_count(self, async_migration: AsyncMigration): + return AsyncMigrationError.objects.filter(async_migration=async_migration).count() + + def get_parameter_definitions(self, async_migration: AsyncMigration): + definition = get_async_migration_definition(async_migration.name) + # Ignore typecasting logic for parameters + return {key: param[:2] for key, param in definition.parameters.items()} + class AsyncMigrationsViewset(StructuredViewSetMixin, viewsets.ModelViewSet): queryset = AsyncMigration.objects.all().order_by("name") @@ -91,6 +101,7 @@ def trigger(self, request, **kwargs): ) migration_instance.status = MigrationStatus.Starting + migration_instance.parameters = request.data.get("parameters", {}) migration_instance.save() trigger_migration(migration_instance) @@ -105,6 +116,7 @@ def resume(self, request, **kwargs): ) migration_instance.status = MigrationStatus.Running + migration_instance.parameters = request.data.get("parameters", {}) migration_instance.save() trigger_migration(migration_instance, fresh_start=False) diff --git a/posthog/api/test/test_async_migrations.py b/posthog/api/test/test_async_migrations.py index 772fa383dd992a..cedf64f27a2c3b 100644 --- a/posthog/api/test/test_async_migrations.py +++ b/posthog/api/test/test_async_migrations.py @@ -41,26 +41,28 @@ def test_get_async_migrations_without_staff_status(self): self.assertEqual(response["detail"], "You are not a staff user, contact your instance admin.") def test_get_async_migrations(self): - create_async_migration() - create_async_migration(name="test2") + create_async_migration(name="0002_events_sample_by") + create_async_migration(name="0003_fill_person_distinct_id2") response = self.client.get(f"/api/async_migrations/").json() self.assertEqual(len(response["results"]), 2) - self.assertEqual(response["results"][0]["name"], "test1") - self.assertEqual(response["results"][1]["name"], "test2") + self.assertEqual(response["results"][0]["name"], "0002_events_sample_by") + self.assertEqual(response["results"][1]["name"], "0003_fill_person_distinct_id2") @patch("posthog.tasks.async_migrations.run_async_migration.delay") def test_trigger_endpoint(self, mock_run_async_migration): sm1 = create_async_migration() - # sm2 = create_async_migration(name="test2") - response = self.client.post(f"/api/async_migrations/{sm1.id}/trigger").json() + response = self.client.post( + f"/api/async_migrations/{sm1.id}/trigger", {"parameters": {"SOME_KEY": 1234}} + ).json() sm1.refresh_from_db() mock_run_async_migration.assert_called_once() self.assertEqual(response["success"], True) self.assertEqual(sm1.status, MigrationStatus.Starting) + self.assertEqual(sm1.parameters, {"SOME_KEY": 1234}) @patch("posthog.tasks.async_migrations.run_async_migration.delay") def test_trigger_with_another_migration_running(self, mock_run_async_migration): @@ -102,7 +104,7 @@ def test_force_stop_endpoint_non_running_migration(self, mock_run_async_migratio @patch("posthog.async_migrations.runner.get_async_migration_definition") def test_force_rollback_endpoint(self, mock_get_migration_definition): - mock_get_migration_definition.return_value = AsyncMigrationDefinition() + mock_get_migration_definition.return_value = AsyncMigrationDefinition(name="foo") sm1 = create_async_migration(status=MigrationStatus.CompletedSuccessfully) response = self.client.post(f"/api/async_migrations/{sm1.id}/force_rollback").json() diff --git a/posthog/async_migrations/definition.py b/posthog/async_migrations/definition.py index 9e25430c5d8e83..848eed4c2aa17f 100644 --- a/posthog/async_migrations/definition.py +++ b/posthog/async_migrations/definition.py @@ -1,26 +1,12 @@ -from datetime import datetime -from typing import Callable, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple from posthog.constants import AnalyticsDBMS from posthog.models.utils import sane_repr from posthog.settings import ASYNC_MIGRATIONS_DEFAULT_TIMEOUT_SECONDS from posthog.version_requirement import ServiceVersionRequirement - -# used to prevent circular imports -class AsyncMigrationType: - id: int - name: str - description: str - progress: int - status: int - current_operation_index: int - current_query_id: str - celery_task_id: str - started_at: datetime - finished_at: datetime - posthog_min_version: str - posthog_max_version: str +if TYPE_CHECKING: + from posthog.models.async_migration import AsyncMigration class AsyncMigrationOperation: @@ -79,6 +65,7 @@ def _execute_op(self, query_id: str, sql: str, settings: Optional[Dict]): class AsyncMigrationDefinition: + name: str # the migration cannot be run before this version posthog_min_version = "0.0.0" @@ -98,6 +85,12 @@ class AsyncMigrationDefinition: # name of async migration this migration depends on depends_on: Optional[str] = None + # optional parameters for this async migration. Shown in the UI when starting the migration + parameters: Dict[str, Tuple[(int, str, Callable[[Any], Any])]] = {} + + def __init__(self, name: str): + self.name = name + # run before creating the migration model. Returns a boolean specifying if the instance should # set up the AsyncMigration model and show this migration in the UI def is_hidden(self) -> bool: @@ -117,5 +110,19 @@ def healthcheck(self) -> Tuple[bool, Optional[str]]: return (True, None) # return an int between 0-100 to specify how far along this migration is - def progress(self, migration_instance: AsyncMigrationType) -> int: + def progress(self, migration_instance: "AsyncMigration") -> int: return int(100 * migration_instance.current_operation_index / len(self.operations)) + + # returns the async migration instance for this migration. Only works during the migration + def migration_instance(self) -> "AsyncMigration": + from posthog.models.async_migration import AsyncMigration + + return AsyncMigration.objects.get(name=self.name) + + def get_parameter(self, parameter_name: str): + instance = self.migration_instance() + if parameter_name in instance.parameters: + return instance.parameters[parameter_name] + else: + # Return the default value + return self.parameters[parameter_name][0] diff --git a/posthog/async_migrations/migrations/0005_person_replacing_by_version.py b/posthog/async_migrations/migrations/0005_person_replacing_by_version.py index 897cf9506b4375..1f37b493b058d4 100644 --- a/posthog/async_migrations/migrations/0005_person_replacing_by_version.py +++ b/posthog/async_migrations/migrations/0005_person_replacing_by_version.py @@ -11,13 +11,13 @@ AsyncMigrationDefinition, AsyncMigrationOperation, AsyncMigrationOperationSQL, - AsyncMigrationType, ) from posthog.async_migrations.utils import execute_op_clickhouse, run_optimize_table from posthog.clickhouse.kafka_engine import STORAGE_POLICY from posthog.clickhouse.table_engines import ReplacingMergeTree from posthog.client import sync_execute from posthog.constants import AnalyticsDBMS +from posthog.models.async_migration import AsyncMigration from posthog.models.person.person import Person from posthog.models.person.sql import PERSONS_TABLE_MV_SQL from posthog.redis import get_client @@ -246,7 +246,7 @@ def _persons_insert_query(self, persons: List[Person]) -> Tuple[str, Dict]: params, ) - def progress(self, migration_instance: AsyncMigrationType) -> int: + def progress(self, migration_instance: AsyncMigration) -> int: # We weigh each step before copying persons as equal, and the persons copy as ~50% of progress result = 0.5 * migration_instance.current_operation_index / len(self.operations) diff --git a/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py b/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py index 4a3bd9a2e1f7a6..568a05f5f68d0d 100644 --- a/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py +++ b/posthog/async_migrations/migrations/0006_persons_and_groups_on_events_backfill.py @@ -62,6 +62,16 @@ class Migration(AsyncMigrationDefinition): depends_on = "0005_person_replacing_by_version" + parameters = { + "PERSON_DICT_CACHE_SIZE": (5000000, "ClickHouse cache size (in rows) for persons data.", int), + "PERSON_DISTINCT_ID_DICT_CACHE_SIZE": ( + 5000000, + "ClickHouse cache size (in rows) for person distinct id data.", + int, + ), + "GROUPS_DICT_CACHE_SIZE": (1000000, "ClickHouse cache size (in rows) for groups data.", int), + } + def is_hidden(self) -> bool: return not (get_instance_setting("ASYNC_MIGRATIONS_SHOW_PERSON_ON_EVENTS_MIGRATION") or settings.TEST) @@ -192,57 +202,7 @@ def operations(self): rollback=None, per_shard=True, ), - AsyncMigrationOperationSQL( - sql=f""" - CREATE DICTIONARY IF NOT EXISTS person_dict {{on_cluster_clause}} - ( - team_id Int64, - id UUID, - properties String, - created_at DateTime - ) - PRIMARY KEY team_id, id - SOURCE(CLICKHOUSE(TABLE {TEMPORARY_PERSONS_TABLE_NAME} {self._dictionary_connection_string()})) - LAYOUT(complex_key_cache(size_in_cells 5000000 max_threads_for_updates 6 allow_read_expired_keys 1)) - Lifetime(60000) - """, - rollback=f"DROP DICTIONARY IF EXISTS person_dict {{on_cluster_clause}}", - per_shard=True, - ), - AsyncMigrationOperationSQL( - sql=f""" - CREATE DICTIONARY IF NOT EXISTS person_distinct_id2_dict {{on_cluster_clause}} - ( - team_id Int64, - distinct_id String, - person_id UUID - ) - PRIMARY KEY team_id, distinct_id - SOURCE(CLICKHOUSE(TABLE {TEMPORARY_PDI2_TABLE_NAME} {self._dictionary_connection_string()})) - LAYOUT(complex_key_cache(size_in_cells 50000000 max_threads_for_updates 6 allow_read_expired_keys 1)) - Lifetime(60000) - """, - rollback=f"DROP DICTIONARY IF EXISTS person_distinct_id2_dict {{on_cluster_clause}}", - per_shard=True, - ), - AsyncMigrationOperationSQL( - sql=f""" - CREATE DICTIONARY IF NOT EXISTS groups_dict {{on_cluster_clause}} - ( - team_id Int64, - group_type_index UInt8, - group_key String, - group_properties String, - created_at DateTime - ) - PRIMARY KEY team_id, group_type_index, group_key - SOURCE(CLICKHOUSE(TABLE {TEMPORARY_GROUPS_TABLE_NAME} {self._dictionary_connection_string()})) - LAYOUT(complex_key_cache(size_in_cells 1000000 max_threads_for_updates 6 allow_read_expired_keys 1)) - Lifetime(60000) - """, - rollback=f"DROP DICTIONARY IF EXISTS groups_dict {{on_cluster_clause}}", - per_shard=True, - ), + AsyncMigrationOperation(fn=self._create_dictionaries, rollback_fn=self._clear_temporary_tables), AsyncMigrationOperationSQL( sql=f""" ALTER TABLE {EVENTS_DATA_TABLE()} @@ -308,6 +268,62 @@ def _update_properties_column_compression_codec(self, query_id, codec): sql=f"ALTER TABLE {EVENTS_DATA_TABLE()} ON CLUSTER '{settings.CLICKHOUSE_CLUSTER}' MODIFY COLUMN {column} VARCHAR Codec({codec})", ) + def _create_dictionaries(self, query_id): + execute_op_clickhouse( + f""" + CREATE DICTIONARY IF NOT EXISTS person_dict {{on_cluster_clause}} + ( + team_id Int64, + id UUID, + properties String, + created_at DateTime + ) + PRIMARY KEY team_id, id + SOURCE(CLICKHOUSE(TABLE {TEMPORARY_PERSONS_TABLE_NAME} {self._dictionary_connection_string()})) + LAYOUT(complex_key_cache(size_in_cells %(cache_size)s max_threads_for_updates 6 allow_read_expired_keys 1)) + Lifetime(60000) + """, + {"cache_size": self.get_parameter("PERSON_DICT_CACHE_SIZE")}, + per_shard=True, + query_id=query_id, + ), + execute_op_clickhouse( + f""" + CREATE DICTIONARY IF NOT EXISTS person_distinct_id2_dict {{on_cluster_clause}} + ( + team_id Int64, + distinct_id String, + person_id UUID + ) + PRIMARY KEY team_id, distinct_id + SOURCE(CLICKHOUSE(TABLE {TEMPORARY_PDI2_TABLE_NAME} {self._dictionary_connection_string()})) + LAYOUT(complex_key_cache(size_in_cells %(cache_size)s max_threads_for_updates 6 allow_read_expired_keys 1)) + Lifetime(60000) + """, + {"cache_size": self.get_parameter("PERSON_DISTINCT_ID_DICT_CACHE_SIZE")}, + per_shard=True, + query_id=query_id, + ), + execute_op_clickhouse( + f""" + CREATE DICTIONARY IF NOT EXISTS groups_dict {{on_cluster_clause}} + ( + team_id Int64, + group_type_index UInt8, + group_key String, + group_properties String, + created_at DateTime + ) + PRIMARY KEY team_id, group_type_index, group_key + SOURCE(CLICKHOUSE(TABLE {TEMPORARY_GROUPS_TABLE_NAME} {self._dictionary_connection_string()})) + LAYOUT(complex_key_cache(size_in_cells %(cache_size)s max_threads_for_updates 6 allow_read_expired_keys 1)) + Lifetime(60000) + """, + {"cache_size": self.get_parameter("GROUPS_DICT_CACHE_SIZE")}, + per_shard=True, + query_id=query_id, + ) + def _wait_for_mutation_done(self, query_id): # :KLUDGE: mutations_sync does not work with ON CLUSTER queries, causing race conditions with subsequent steps sleep_until_finished("events table backill", lambda: self._count_running_mutations() > 0) diff --git a/posthog/async_migrations/runner.py b/posthog/async_migrations/runner.py index 2bb9cf35da0fa2..dc226c65c6fe8a 100644 --- a/posthog/async_migrations/runner.py +++ b/posthog/async_migrations/runner.py @@ -170,7 +170,7 @@ def update_migration_progress(migration_instance: AsyncMigration): migration_instance.refresh_from_db() try: - progress = get_async_migration_definition(migration_instance.name).progress(migration_instance) # type: ignore + progress = get_async_migration_definition(migration_instance.name).progress(migration_instance) update_async_migration(migration_instance=migration_instance, progress=progress) except: pass diff --git a/posthog/async_migrations/setup.py b/posthog/async_migrations/setup.py index eb7b175bb42192..ee7aef6d7e3d8e 100644 --- a/posthog/async_migrations/setup.py +++ b/posthog/async_migrations/setup.py @@ -13,7 +13,7 @@ def reload_migration_definitions(): for name, module in all_migrations.items(): - ALL_ASYNC_MIGRATIONS[name] = module.Migration() + ALL_ASYNC_MIGRATIONS[name] = module.Migration(name) ALL_ASYNC_MIGRATIONS: Dict[str, AsyncMigrationDefinition] = {} @@ -109,7 +109,7 @@ def get_async_migration_definition(migration_name: str) -> AsyncMigrationDefinit if TEST: test_migrations = import_submodules(ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH) if migration_name in test_migrations: - return test_migrations[migration_name].Migration() + return test_migrations[migration_name].Migration(migration_name) return ALL_ASYNC_MIGRATIONS[migration_name] diff --git a/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py b/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py index 5a0e1b1e522014..322cea57d03457 100644 --- a/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py +++ b/posthog/async_migrations/test/test_0006_persons_and_groups_on_events_backfill.py @@ -289,7 +289,6 @@ def test_no_extra_tables(self): def test_rollback(self): migration = get_async_migration_definition(MIGRATION_NAME) - self.assertEqual(len(migration.operations), 18) migration.operations[-1].fn = lambda _: 0 / 0 # type: ignore migration_successful = run_migration() diff --git a/posthog/async_migrations/test/test_definition.py b/posthog/async_migrations/test/test_definition.py index 2e8438f66050e6..8ed0f7155ea7f6 100644 --- a/posthog/async_migrations/test/test_definition.py +++ b/posthog/async_migrations/test/test_definition.py @@ -2,18 +2,23 @@ from infi.clickhouse_orm.utils import import_submodules from posthog.async_migrations.definition import AsyncMigrationDefinition, AsyncMigrationOperation -from posthog.async_migrations.setup import ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH +from posthog.async_migrations.setup import ( + ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH, + get_async_migration_definition, + setup_async_migrations, +) +from posthog.models.async_migration import AsyncMigration from posthog.test.base import BaseTest from posthog.version_requirement import ServiceVersionRequirement +@pytest.mark.ee class TestAsyncMigrationDefinition(BaseTest): - @pytest.mark.ee def test_get_async_migration_definition(self): from posthog.async_migrations.examples.example import example_fn, example_rollback_fn modules = import_submodules(ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH) - example_migration = modules["example"].Migration() + example_migration = modules["example"].Migration("example") self.assertTrue(isinstance(example_migration, AsyncMigrationDefinition)) self.assertTrue(isinstance(example_migration.operations[0], AsyncMigrationOperation)) @@ -23,3 +28,21 @@ def test_get_async_migration_definition(self): self.assertEqual(example_migration.operations[-1].fn, example_fn) self.assertEqual(example_migration.operations[-1].rollback_fn, example_rollback_fn) self.assertTrue(isinstance(example_migration.service_version_requirements[0], ServiceVersionRequirement)) + + def test_get_migration_instance_and_parameters(self): + setup_async_migrations(ignore_posthog_version=True) + + MIGRATION_NAME = "0006_persons_and_groups_on_events_backfill" + + definition = get_async_migration_definition(MIGRATION_NAME) + instance = AsyncMigration.objects.get(name=MIGRATION_NAME) + + self.assertEqual(definition.migration_instance(), instance) + + self.assertEqual( + definition.get_parameter("PERSON_DICT_CACHE_SIZE"), definition.parameters["PERSON_DICT_CACHE_SIZE"][0] + ) + + instance.parameters = {"PERSON_DICT_CACHE_SIZE": 123} + instance.save() + self.assertEqual(definition.get_parameter("PERSON_DICT_CACHE_SIZE"), 123) diff --git a/posthog/async_migrations/test/test_runner.py b/posthog/async_migrations/test/test_runner.py index 7211677b515d02..a205f9be049aaa 100644 --- a/posthog/async_migrations/test/test_runner.py +++ b/posthog/async_migrations/test/test_runner.py @@ -17,7 +17,7 @@ class TestRunner(AsyncMigrationBaseTest): def setUp(self): - self.migration = Migration() + self.migration = Migration("TEST_MIGRATION") self.TEST_MIGRATION_DESCRIPTION = self.migration.description create_async_migration(name="test_migration", description=self.TEST_MIGRATION_DESCRIPTION) return super().setUp() diff --git a/posthog/migrations/0253_add_async_migration_parameters.py b/posthog/migrations/0253_add_async_migration_parameters.py new file mode 100644 index 00000000000000..d89d55c34ab02f --- /dev/null +++ b/posthog/migrations/0253_add_async_migration_parameters.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.14 on 2022-07-13 10:38 + +from django.db import migrations, models + + +# :KLUDGE: Work around test_migrations_are_safe +class AddFiendNullSafe(migrations.AddField): + def describe(self): + return super().describe() + " -- not-null-ignore" + + +class Migration(migrations.Migration): + + dependencies = [ + ("posthog", "0252_reset_insight_refreshing_status"), + ] + + operations = [ + AddFiendNullSafe(model_name="asyncmigration", name="parameters", field=models.JSONField(default=dict),), + ] diff --git a/posthog/models/async_migration.py b/posthog/models/async_migration.py index c0242198c8133e..98e966b4c4da7f 100644 --- a/posthog/models/async_migration.py +++ b/posthog/models/async_migration.py @@ -47,6 +47,8 @@ class Meta: posthog_min_version: models.CharField = models.CharField(max_length=20, null=True, blank=True) posthog_max_version: models.CharField = models.CharField(max_length=20, null=True, blank=True) + parameters: models.JSONField = models.JSONField(default=dict) + def get_all_completed_async_migrations(): return AsyncMigration.objects.filter(status=MigrationStatus.CompletedSuccessfully) diff --git a/posthog/tasks/test/test_async_migrations.py b/posthog/tasks/test/test_async_migrations.py index 47235894a61117..bb7cfce0797e76 100644 --- a/posthog/tasks/test/test_async_migrations.py +++ b/posthog/tasks/test/test_async_migrations.py @@ -13,7 +13,7 @@ from posthog.tasks.async_migrations import check_async_migration_health from posthog.test.base import BaseTest -TEST_MIGRATION_DESCRIPTION = Migration().description +TEST_MIGRATION_DESCRIPTION = Migration("TEST_MIGRATION").description MOCK_CELERY_TASK_ID = "some_task_id" From 91811d262f642772a2f499b3874b4ed90ab2ae6d Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Wed, 27 Jul 2022 11:57:17 +0100 Subject: [PATCH 183/213] fix(feature-flags): Enable overrides to work for not ingested persons (#11002) --- posthog/api/test/test_decide.py | 4 +- posthog/models/feature_flag.py | 24 +++- .../test/__snapshots__/test_feature_flag.ambr | 0 posthog/test/test_feature_flag.py | 108 +++++++++++++++++- 4 files changed, 130 insertions(+), 6 deletions(-) delete mode 100644 posthog/test/__snapshots__/test_feature_flag.ambr diff --git a/posthog/api/test/test_decide.py b/posthog/api/test/test_decide.py index dda09e0ae19e18..238a0eecbe83a5 100644 --- a/posthog/api/test/test_decide.py +++ b/posthog/api/test/test_decide.py @@ -261,12 +261,12 @@ def test_feature_flags_v2_with_property_overrides(self): with override_instance_config("GEOIP_PROPERTY_OVERRIDES_TEAMS", f"{self.team.pk}"): - with self.assertNumQueries(4): + with self.assertNumQueries(3): response = self._post_decide(api_version=2, ip=australia_ip) self.assertTrue(response.json()["featureFlags"]["beta-feature"]) self.assertTrue("multivariate-flag" not in response.json()["featureFlags"]) - with self.assertNumQueries(4): + with self.assertNumQueries(3): response = self._post_decide(api_version=2, distinct_id="other_id", ip=australia_ip) self.assertTrue(response.json()["featureFlags"]["beta-feature"]) self.assertTrue("multivariate-flag" not in response.json()["featureFlags"]) diff --git a/posthog/models/feature_flag.py b/posthog/models/feature_flag.py index fd0921610939da..6f91e5845f2850 100644 --- a/posthog/models/feature_flag.py +++ b/posthog/models/feature_flag.py @@ -18,8 +18,9 @@ from posthog.models.group import Group from posthog.models.group_type_mapping import GroupTypeMapping from posthog.models.property import GroupTypeIndex, GroupTypeName +from posthog.models.property.property import Property from posthog.models.signals import mutable_receiver -from posthog.queries.base import properties_to_Q +from posthog.queries.base import match_property, properties_to_Q from .filters import Filter from .person import Person, PersonDistinctId @@ -216,7 +217,18 @@ def get_matching_variant(self, feature_flag: FeatureFlag) -> Optional[str]: def is_condition_match(self, feature_flag: FeatureFlag, condition: Dict, condition_index: int): rollout_percentage = condition.get("rollout_percentage") if len(condition.get("properties", [])) > 0: - if not self._condition_matches(feature_flag, condition_index): + properties = Filter(data=condition).property_groups.flat + if self.can_compute_locally(properties): + # :TRICKY: If overrides are enough to determine if a condition is a match, + # we can skip checking the query. + # This ensures match even if the person hasn't been ingested yet. + condition_match = all( + match_property(property, self.property_value_overrides) for property in properties + ) + else: + condition_match = self._condition_matches(feature_flag, condition_index) + + if not condition_match: return False elif rollout_percentage is None: return True @@ -323,6 +335,14 @@ def get_hash(self, feature_flag: FeatureFlag, salt="") -> float: hash_val = int(hashlib.sha1(hash_key.encode("utf-8")).hexdigest()[:15], 16) return hash_val / __LONG_SCALE__ + def can_compute_locally(self, properties: List[Property]) -> bool: + for property in properties: + if property.key not in self.property_value_overrides: + return False + if property.operator == "is_not_set": + return False + return True + def hash_key_overrides(team_id: int, person_id: int) -> Dict[str, str]: feature_flag_to_key_overrides = {} diff --git a/posthog/test/__snapshots__/test_feature_flag.ambr b/posthog/test/__snapshots__/test_feature_flag.ambr deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/posthog/test/test_feature_flag.py b/posthog/test/test_feature_flag.py index d4a7fd5afbb014..d6e6468963ddc9 100644 --- a/posthog/test/test_feature_flag.py +++ b/posthog/test/test_feature_flag.py @@ -204,19 +204,22 @@ def test_multi_property_filters_with_override_properties(self): ] } ) - with self.assertNumQueries(2): + with self.assertNumQueries(1): self.assertEqual( FeatureFlagMatcher([feature_flag], "example_id", property_value_overrides={}).get_match(feature_flag), FeatureFlagMatch(), ) + # can be computed locally self.assertIsNone( FeatureFlagMatcher([feature_flag], "example_id", property_value_overrides={"email": "bzz"}).get_match( feature_flag ) ) - with self.assertNumQueries(2): + with self.assertNumQueries(1): self.assertIsNone(FeatureFlagMatcher([feature_flag], "random_id").get_match(feature_flag)) + + # can be computed locally self.assertEqual( FeatureFlagMatcher( [feature_flag], "random_id", property_value_overrides={"email": "example@example.com"} @@ -224,6 +227,107 @@ def test_multi_property_filters_with_override_properties(self): FeatureFlagMatch(), ) + def test_override_properties_where_person_doesnt_exist_yet(self): + feature_flag = self.create_feature_flag( + filters={ + "groups": [ + {"properties": [{"key": "email", "value": "tim@posthog.com", "type": "person"}]}, + {"properties": [{"key": "email", "value": "example@example.com"}]}, + ] + } + ) + with self.assertNumQueries(0): + self.assertEqual( + FeatureFlagMatcher( + [feature_flag], "example_id", property_value_overrides={"email": "tim@posthog.com"} + ).get_match(feature_flag), + FeatureFlagMatch(), + ) + self.assertIsNone( + FeatureFlagMatcher([feature_flag], "example_id", property_value_overrides={"email": "bzz"}).get_match( + feature_flag + ) + ) + + with self.assertNumQueries(1): + self.assertIsNone(FeatureFlagMatcher([feature_flag], "random_id").get_match(feature_flag)) + + with self.assertNumQueries(0): + self.assertEqual( + FeatureFlagMatcher( + [feature_flag], "random_id", property_value_overrides={"email": "example@example.com"} + ).get_match(feature_flag), + FeatureFlagMatch(), + ) + + def test_override_properties_where_person_doesnt_exist_yet_multiple_conditions(self): + feature_flag = self.create_feature_flag( + filters={ + "groups": [ + { + "properties": [ + {"key": "email", "value": "tim@posthog.com"}, + {"key": "another_prop", "value": "slow"}, + ], + "rollout_percentage": 50, + }, + ] + } + ) + with self.assertNumQueries(2): + # None in both because all conditions don't match + # and user doesn't exist yet + self.assertIsNone( + FeatureFlagMatcher( + [feature_flag], "example_id", property_value_overrides={"email": "tim@posthog.com"} + ).get_match(feature_flag), + FeatureFlagMatch(), + ) + self.assertIsNone( + FeatureFlagMatcher([feature_flag], "example_id", property_value_overrides={"email": "bzz"}).get_match( + feature_flag + ) + ) + + with self.assertNumQueries(1): + self.assertIsNone(FeatureFlagMatcher([feature_flag], "random_id").get_match(feature_flag)) + + with self.assertNumQueries(0): + # Both of these match properties, but second one is outside rollout %. + self.assertEqual( + FeatureFlagMatcher( + [feature_flag], + "random_id_without_rollout", + property_value_overrides={"email": "tim@posthog.com", "another_prop": "slow", "blah": "blah"}, + ).get_match(feature_flag), + FeatureFlagMatch(), + ) + self.assertIsNone( + FeatureFlagMatcher( + [feature_flag], + "random_id_within_rollout", + property_value_overrides={"email": "tim@posthog.com", "another_prop": "slow", "blah": "blah"}, + ).get_match(feature_flag), + ) + + with self.assertNumQueries(0): + # These don't match properties + self.assertIsNone( + FeatureFlagMatcher( + [feature_flag], + "random_id_without_rollout", + property_value_overrides={"email": "tim@posthog.com", "another_prop": "slow2", "blah": "blah"}, + ).get_match(feature_flag), + FeatureFlagMatch(), + ) + self.assertIsNone( + FeatureFlagMatcher( + [feature_flag], + "random_id_without_rollout", + property_value_overrides={"email": "tim2@posthog.com", "another_prop": "slow", "blah": "blah"}, + ).get_match(feature_flag), + ) + def test_multi_property_filters_with_override_properties_with_is_not_set(self): Person.objects.create( team=self.team, distinct_ids=["example_id"], properties={"email": "tim@posthog.com"}, From 3f914c0c16a2e6f2da624e00492fd5ef00406609 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 27 Jul 2022 12:02:09 +0100 Subject: [PATCH 184/213] feat: change insight cache rate with instance setting (#11001) * feat: change insight cache rate with instance setting * fix test --- .../instance/SystemStatus/systemStatusLogic.ts | 1 + posthog/settings/dynamic_settings.py | 6 ++++++ posthog/tasks/test/test_update_cache.py | 14 ++++++-------- posthog/tasks/update_cache.py | 10 ++++------ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts index f970b04b0835e2..76d5ffc75c1aa8 100644 --- a/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts +++ b/frontend/src/scenes/instance/SystemStatus/systemStatusLogic.ts @@ -54,6 +54,7 @@ const EDITABLE_INSTANCE_SETTINGS = [ 'SLACK_APP_CLIENT_ID', 'SLACK_APP_CLIENT_SECRET', 'SLACK_APP_SIGNING_SECRET', + 'PARALLEL_DASHBOARD_ITEM_CACHE', ] export const systemStatusLogic = kea({ diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.py index 9c9577e17ca650..18c54905d52544 100644 --- a/posthog/settings/dynamic_settings.py +++ b/posthog/settings/dynamic_settings.py @@ -139,6 +139,11 @@ "Used to validate Slack events for example when unfurling links", str, ), + "PARALLEL_DASHBOARD_ITEM_CACHE": ( + get_from_env("PARALLEL_DASHBOARD_ITEM_CACHE", default=5), + "user to determine how many insight cache updates to run at a time", + int, + ), } SETTINGS_ALLOWING_API_OVERRIDE = ( @@ -166,6 +171,7 @@ "SLACK_APP_CLIENT_ID", "SLACK_APP_CLIENT_SECRET", "SLACK_APP_SIGNING_SECRET", + "PARALLEL_DASHBOARD_ITEM_CACHE", ) # SECRET_SETTINGS can only be updated but will never be exposed through the API (we do store them plain text in the DB) diff --git a/posthog/tasks/test/test_update_cache.py b/posthog/tasks/test/test_update_cache.py index 92070a59ac9403..777afc57639a90 100644 --- a/posthog/tasks/test/test_update_cache.py +++ b/posthog/tasks/test/test_update_cache.py @@ -13,15 +13,11 @@ from posthog.models.filters.retention_filter import RetentionFilter from posthog.models.filters.stickiness_filter import StickinessFilter from posthog.models.filters.utils import get_filter +from posthog.models.instance_setting import set_instance_setting from posthog.models.sharing_configuration import SharingConfiguration from posthog.models.team.team import Team from posthog.queries.util import get_earliest_timestamp -from posthog.tasks.update_cache import ( - PARALLEL_INSIGHT_CACHE, - synchronously_update_insight_cache, - update_cache_item, - update_cached_items, -) +from posthog.tasks.update_cache import synchronously_update_insight_cache, update_cache_item, update_cached_items from posthog.test.base import APIBaseTest from posthog.types import FilterType from posthog.utils import generate_cache_key, get_safe_cache @@ -705,6 +701,8 @@ def test_broken_exception_insights(self, dashboard_item_update_task_params: Magi @patch("posthog.celery.update_cache_item_task.s") @freeze_time("2022-01-03T00:00:00.000Z") def test_refresh_insight_cache(self, patch_update_cache_item: MagicMock, _patch_apply_async: MagicMock) -> None: + parallel_insight_cache = 5 + set_instance_setting(key="PARALLEL_INSIGHT_CACHE", value=parallel_insight_cache) filter_dict: Dict[str, Any] = { "events": [{"id": "$pageview"}], "properties": [{"key": "$browser", "value": "Mac OS X"}], @@ -723,7 +721,7 @@ def test_refresh_insight_cache(self, patch_update_cache_item: MagicMock, _patch_ filters=filter_dict, last_refresh=datetime(2022, 1, 1).replace(tzinfo=pytz.utc), ) - for _ in range(PARALLEL_INSIGHT_CACHE - 1) + for _ in range(parallel_insight_cache - 1) ] # Valid insights outside of the PARALLEL_INSIGHT_CACHE count with later refresh date to ensure order @@ -740,7 +738,7 @@ def test_refresh_insight_cache(self, patch_update_cache_item: MagicMock, _patch_ tasks, queue_length = update_cached_items() assert tasks == 5 - assert queue_length == PARALLEL_INSIGHT_CACHE + 5 + assert queue_length == parallel_insight_cache + 5 for call_item in patch_update_cache_item.call_args_list: update_cache_item(*call_item[0]) diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index 5bef367b260fbd..21fd508c19d7c4 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -1,6 +1,5 @@ import datetime import json -import os from typing import Any, Dict, List, Optional, Tuple, Union import structlog @@ -31,6 +30,7 @@ from posthog.models import Dashboard, DashboardTile, Filter, Insight, Team from posthog.models.filters.stickiness_filter import StickinessFilter from posthog.models.filters.utils import get_filter +from posthog.models.instance_setting import get_instance_setting from posthog.queries.funnels import ClickhouseFunnelTimeToConvert, ClickhouseFunnelTrends from posthog.queries.funnels.utils import get_funnel_order_class from posthog.queries.paths import Paths @@ -40,8 +40,6 @@ from posthog.types import FilterType from posthog.utils import generate_cache_key -PARALLEL_INSIGHT_CACHE = int(os.environ.get("PARALLEL_DASHBOARD_ITEM_CACHE", 5)) - logger = structlog.get_logger(__name__) CACHE_TYPE_TO_INSIGHT_CLASS = { @@ -53,6 +51,8 @@ def update_cached_items() -> Tuple[int, int]: + PARALLEL_INSIGHT_CACHE = get_instance_setting("PARALLEL_DASHBOARD_ITEM_CACHE") + tasks: List[Optional[Signature]] = [] dashboard_tiles = ( @@ -117,9 +117,7 @@ def task_for_cache_update_candidate(candidate: Union[DashboardTile, Insight]) -> def gauge_cache_update_candidates(dashboard_tiles: QuerySet, shared_insights: QuerySet) -> None: statsd.gauge("update_cache_queue.never_refreshed", dashboard_tiles.filter(last_refresh=None).count()) - oldest_previously_refreshed_tiles: List[DashboardTile] = list( - dashboard_tiles.exclude(last_refresh=None)[0:PARALLEL_INSIGHT_CACHE] - ) + oldest_previously_refreshed_tiles: List[DashboardTile] = list(dashboard_tiles.exclude(last_refresh=None)[0:10]) ages = [] for candidate_tile in oldest_previously_refreshed_tiles: dashboard_cache_age = (datetime.datetime.now(timezone.utc) - candidate_tile.last_refresh).total_seconds() From dbe592c0e917f2711fee4fee1e8ddce074efe70b Mon Sep 17 00:00:00 2001 From: Ben White Date: Wed, 27 Jul 2022 13:07:33 +0200 Subject: [PATCH 185/213] feat: Updated to alpha version of posthog-js (#10983) --- frontend/src/loadPostHogJS.tsx | 5 ++--- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/frontend/src/loadPostHogJS.tsx b/frontend/src/loadPostHogJS.tsx index 01e9f20fa930fd..d4721c985aa9b7 100644 --- a/frontend/src/loadPostHogJS.tsx +++ b/frontend/src/loadPostHogJS.tsx @@ -1,8 +1,8 @@ -import posthog from 'posthog-js' +import posthog, { PostHogConfig } from 'posthog-js' import * as Sentry from '@sentry/react' import { Integration } from '@sentry/types' -const configWithSentry = (config: posthog.Config): posthog.Config => { +const configWithSentry = (config: Partial): Partial => { if ((window as any).SENTRY_DSN) { config.on_xhr_error = (failedRequest: XMLHttpRequest) => { const status = failedRequest.status @@ -22,7 +22,6 @@ export function loadPostHogJS(): void { window.JS_POSTHOG_API_KEY, configWithSentry({ api_host: window.JS_POSTHOG_HOST, - // @ts-expect-error _capture_metrics: true, rageclick: true, debug: window.JS_POSTHOG_SELF_CAPTURE, diff --git a/package.json b/package.json index dbfc3335926e7c..e7c829336450bf 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "kea-window-values": "^3.0.0", "md5": "^2.3.0", "monaco-editor": "^0.23.0", - "posthog-js": "1.26.0", + "posthog-js": "1.27.0-alpha4", "posthog-js-lite": "^0.0.3", "prop-types": "^15.7.2", "query-selector-shadow-dom": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 8a67d60eee595f..e6ba4519ae4906 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14105,10 +14105,10 @@ posthog-js-lite@^0.0.3: resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.3.tgz#87e373706227a849c4e7c6b0cb2066a64ad5b6ed" integrity sha512-wEOs8DEjlFBwgd7l19grosaF7mTlliZ9G9pL0Qji189FDg2ukY5IegUxTyTs7gsTGt6WK9W47BF5yXA5+bwvZg== -posthog-js@1.26.0: - version "1.26.0" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.26.0.tgz#8fd2becfbdf8f165244043d109f140ea0d02a99b" - integrity sha512-Fjc5REUJxrVTQ0WzfChn+CW/UrparZGwINPOtO9FoB25U2IrXnvnTUpkYhSPucZPWUwUMRdXBCo9838COn464Q== +posthog-js@1.27.0-alpha4: + version "1.27.0-alpha4" + resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.27.0-alpha4.tgz#9a5047ba8bd1a78589212c16290e4c651ece0504" + integrity sha512-C1vjAPtRXQk3KxGToDUyRw5aqB7/lsuRkIip76EdK7NAFaXpxXrplK+RT1I/m+TNVoAehK7EGnS/h046kL4TaA== dependencies: "@sentry/types" "^7.2.0" fflate "^0.4.1" From d00d587b1ca7e782295b0f7271dd5bb1c8ff44bb Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Wed, 27 Jul 2022 14:26:19 +0300 Subject: [PATCH 186/213] chore(plugin-server): Improve kafka producer wrapper (#10968) * chore(plugin-server): include extra information on kafka producer errors We're failing to send batches of messages to kafka on a semi-regular basis due to message sizes. It's unclear why this is the case as we try to limit each message batch size. This PR adds information on these failed batches to sentry error messages. Example error: https://sentry.io/organizations/posthog2/issues/3291755686/?project=6423401&query=is%3Aunresolved+level%3Aerror * refactor(plugin-server): Remove Buffer.from from kafka messages This allows us to be much more accurate estimating message sizes, hopefully eliminating a class of errors * estimateMessageSize * Track histogram with message sizes * Flush immediately for too large messages * fud --- plugin-server/src/main/utils.ts | 2 +- plugin-server/src/utils/db/db.ts | 56 ++++++++----------- .../src/utils/db/kafka-producer-wrapper.ts | 31 +++++----- plugin-server/src/utils/db/utils.ts | 20 +++---- .../src/worker/ingestion/process-event.ts | 22 ++++---- plugin-server/src/worker/ingestion/utils.ts | 2 +- .../tests/main/kafka-producer-wrapper.test.ts | 42 +++++++++----- 7 files changed, 86 insertions(+), 89 deletions(-) diff --git a/plugin-server/src/main/utils.ts b/plugin-server/src/main/utils.ts index e8f6e60de8d7b9..0c7e5e3b336c7a 100644 --- a/plugin-server/src/main/utils.ts +++ b/plugin-server/src/main/utils.ts @@ -61,7 +61,7 @@ export async function kafkaHealthcheck( messages: [ { partition: 0, - value: Buffer.from('healthcheck'), + value: 'healthcheck', }, ], }) diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index 300ce2a5d9c816..420e4a1ddde803 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -1139,14 +1139,12 @@ export class DB { topic: KAFKA_PERSON_DISTINCT_ID, messages: [ { - value: Buffer.from( - JSON.stringify({ - ...personDistinctIdCreated, - version, - person_id: person.uuid, - is_deleted: 0, - }) - ), + value: JSON.stringify({ + ...personDistinctIdCreated, + version, + person_id: person.uuid, + is_deleted: 0, + }), }, ], }, @@ -1157,13 +1155,11 @@ export class DB { topic: KAFKA_PERSON_UNIQUE_ID, messages: [ { - value: Buffer.from( - JSON.stringify({ - ...personDistinctIdCreated, - person_id: person.uuid, - is_deleted: 0, - }) - ), + value: JSON.stringify({ + ...personDistinctIdCreated, + person_id: person.uuid, + is_deleted: 0, + }), }, ], }) @@ -1219,9 +1215,7 @@ export class DB { topic: KAFKA_PERSON_DISTINCT_ID, messages: [ { - value: Buffer.from( - JSON.stringify({ ...usefulColumns, version, person_id: target.uuid, is_deleted: 0 }) - ), + value: JSON.stringify({ ...usefulColumns, version, person_id: target.uuid, is_deleted: 0 }), }, ], }) @@ -1231,14 +1225,10 @@ export class DB { topic: KAFKA_PERSON_UNIQUE_ID, messages: [ { - value: Buffer.from( - JSON.stringify({ ...usefulColumns, person_id: target.uuid, is_deleted: 0 }) - ), + value: JSON.stringify({ ...usefulColumns, person_id: target.uuid, is_deleted: 0 }), }, { - value: Buffer.from( - JSON.stringify({ ...usefulColumns, person_id: source.uuid, is_deleted: 1 }) - ), + value: JSON.stringify({ ...usefulColumns, person_id: source.uuid, is_deleted: 1 }), }, ], }) @@ -1937,16 +1927,14 @@ export class DB { topic: KAFKA_GROUPS, messages: [ { - value: Buffer.from( - JSON.stringify({ - group_type_index: groupTypeIndex, - group_key: groupKey, - team_id: teamId, - group_properties: JSON.stringify(properties), - created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouseSecondPrecision), - version, - }) - ), + value: JSON.stringify({ + group_type_index: groupTypeIndex, + group_key: groupKey, + team_id: teamId, + group_properties: JSON.stringify(properties), + created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouseSecondPrecision), + version, + }), }, ], }) diff --git a/plugin-server/src/utils/db/kafka-producer-wrapper.ts b/plugin-server/src/utils/db/kafka-producer-wrapper.ts index 795284ec9d2a62..0a1ff5d8fa0809 100644 --- a/plugin-server/src/utils/db/kafka-producer-wrapper.ts +++ b/plugin-server/src/utils/db/kafka-producer-wrapper.ts @@ -48,9 +48,9 @@ export class KafkaProducerWrapper { } async queueMessage(kafkaMessage: ProducerRecord): Promise { - const messageSize = this.getMessageSize(kafkaMessage) + const messageSize = this.estimateMessageSize(kafkaMessage) - if (this.currentBatchSize + messageSize > this.maxBatchSize) { + if (this.currentBatch.length > 0 && this.currentBatchSize + messageSize > this.maxBatchSize) { // :TRICKY: We want to first flush then immediately add the message to the queue. Awaiting and then pushing would result in a race condition. await this.flush(kafkaMessage) } else { @@ -58,7 +58,11 @@ export class KafkaProducerWrapper { this.currentBatchSize += messageSize const timeSinceLastFlush = Date.now() - this.lastFlushTime - if (timeSinceLastFlush > this.flushFrequencyMs || this.currentBatch.length >= this.maxQueueSize) { + if ( + this.currentBatchSize > this.maxBatchSize || + timeSinceLastFlush > this.flushFrequencyMs || + this.currentBatch.length >= this.maxQueueSize + ) { await this.flush() } } @@ -73,7 +77,7 @@ export class KafkaProducerWrapper { async queueSingleJsonMessage(topic: string, key: Message['key'], object: Record): Promise { await this.queueMessage({ topic, - messages: [{ key, value: Buffer.from(JSON.stringify(object)) }], + messages: [{ key, value: JSON.stringify(object) }], }) } @@ -87,9 +91,11 @@ export class KafkaProducerWrapper { const batchSize = this.currentBatchSize this.lastFlushTime = Date.now() this.currentBatch = append ? [append] : [] - this.currentBatchSize = append ? this.getMessageSize(append) : 0 + this.currentBatchSize = append ? this.estimateMessageSize(append) : 0 + this.statsd?.histogram('query.kafka_send.size', batchSize) const timeout = timeoutGuard('Kafka message sending delayed. Waiting over 30 sec to send messages.') + try { await this.producer.sendBatch({ topicMessages: messages, @@ -118,17 +124,8 @@ export class KafkaProducerWrapper { await this.producer.disconnect() } - private getMessageSize(kafkaMessage: ProducerRecord): number { - return kafkaMessage.messages - .map((message) => { - if (message.value === null) { - return 4 - } - if (!Buffer.isBuffer(message.value)) { - message.value = Buffer.from(message.value) - } - return message.value.length - }) - .reduce((a, b) => a + b) + private estimateMessageSize(kafkaMessage: ProducerRecord): number { + // :TRICKY: This length respects unicode + return Buffer.from(JSON.stringify(kafkaMessage)).length } } diff --git a/plugin-server/src/utils/db/utils.ts b/plugin-server/src/utils/db/utils.ts index 3f8d92f75f2495..b088790d8b0bb8 100644 --- a/plugin-server/src/utils/db/utils.ts +++ b/plugin-server/src/utils/db/utils.ts @@ -91,17 +91,15 @@ export function generateKafkaPersonUpdateMessage( topic: KAFKA_PERSON, messages: [ { - value: Buffer.from( - JSON.stringify({ - id, - created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouseSecondPrecision), - properties: JSON.stringify(properties), - team_id: teamId, - is_identified: isIdentified, - is_deleted: isDeleted, - ...(version !== null ? { version } : {}), - }) - ), + value: JSON.stringify({ + id, + created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouseSecondPrecision), + properties: JSON.stringify(properties), + team_id: teamId, + is_identified: isIdentified, + is_deleted: isDeleted, + ...(version !== null ? { version } : {}), + }), }, ], } diff --git a/plugin-server/src/worker/ingestion/process-event.ts b/plugin-server/src/worker/ingestion/process-event.ts index 1d1dffb76684cc..ba78dd245feae5 100644 --- a/plugin-server/src/worker/ingestion/process-event.ts +++ b/plugin-server/src/worker/ingestion/process-event.ts @@ -243,18 +243,16 @@ export class EventsProcessor { // proto ingestion is deprecated and we won't support new additions to the schema const message = useExternalSchemas ? (EventProto.encodeDelimited(EventProto.create(eventPayload)).finish() as Buffer) - : Buffer.from( - JSON.stringify({ - ...eventPayload, - person_id: personInfo?.uuid, - person_properties: eventPersonProperties, - person_created_at: personInfo - ? castTimestampOrNow(personInfo?.created_at, TimestampFormat.ClickHouseSecondPrecision) - : null, - ...groupsProperties, - ...groupsCreatedAt, - }) - ) + : JSON.stringify({ + ...eventPayload, + person_id: personInfo?.uuid, + person_properties: eventPersonProperties, + person_created_at: personInfo + ? castTimestampOrNow(personInfo?.created_at, TimestampFormat.ClickHouseSecondPrecision) + : null, + ...groupsProperties, + ...groupsCreatedAt, + }) await this.kafkaProducer.queueMessage({ topic: useExternalSchemas ? KAFKA_EVENTS : this.pluginsServer.CLICKHOUSE_JSON_EVENTS_KAFKA_TOPIC, diff --git a/plugin-server/src/worker/ingestion/utils.ts b/plugin-server/src/worker/ingestion/utils.ts index 9451c17baa0404..5da87f8966c6ff 100644 --- a/plugin-server/src/worker/ingestion/utils.ts +++ b/plugin-server/src/worker/ingestion/utils.ts @@ -48,7 +48,7 @@ export function generateEventDeadLetterQueueMessage( topic: KAFKA_EVENTS_DEAD_LETTER_QUEUE, messages: [ { - value: Buffer.from(JSON.stringify(deadLetterQueueEvent)), + value: JSON.stringify(deadLetterQueueEvent), }, ], } diff --git a/plugin-server/tests/main/kafka-producer-wrapper.test.ts b/plugin-server/tests/main/kafka-producer-wrapper.test.ts index a0480fc03915be..1a06f5974ab22b 100644 --- a/plugin-server/tests/main/kafka-producer-wrapper.test.ts +++ b/plugin-server/tests/main/kafka-producer-wrapper.test.ts @@ -26,15 +26,15 @@ describe('KafkaProducerWrapper', () => { it('respects MAX_QUEUE_SIZE', async () => { await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(10) }], + messages: [{ value: '1'.repeat(10) }], }) await producer.queueMessage({ topic: 'b', - messages: [{ value: Buffer.alloc(30) }], + messages: [{ value: '1'.repeat(30) }], }) await producer.queueMessage({ topic: 'b', - messages: [{ value: Buffer.alloc(30) }], + messages: [{ value: '1'.repeat(30) }], }) expect(flushSpy).not.toHaveBeenCalled() @@ -42,7 +42,7 @@ describe('KafkaProducerWrapper', () => { await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(30) }], + messages: [{ value: '1'.repeat(30) }], }) expect(flushSpy).toHaveBeenCalled() @@ -56,39 +56,55 @@ describe('KafkaProducerWrapper', () => { it('respects KAFKA_MAX_MESSAGE_BATCH_SIZE', async () => { await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(450) }], + messages: [{ value: '1'.repeat(400) }], }) await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(40) }], + messages: [{ value: '1'.repeat(20) }], }) expect(flushSpy).not.toHaveBeenCalled() expect(producer.currentBatch.length).toEqual(2) await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(40) }], + messages: [{ value: '1'.repeat(40) }], }) expect(flushSpy).toHaveBeenCalled() expect(producer.currentBatch.length).toEqual(1) - expect(producer.currentBatchSize).toEqual(40) + expect(producer.currentBatchSize).toBeGreaterThan(40) + expect(producer.currentBatchSize).toBeLessThan(100) expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ topicMessages: [expect.anything(), expect.anything()], }) }) + it('flushes immediately when message exceeds KAFKA_MAX_MESSAGE_BATCH_SIZE', async () => { + await producer.queueMessage({ + topic: 'a', + messages: [{ value: '1'.repeat(10000) }], + }) + + expect(flushSpy).toHaveBeenCalled() + + expect(producer.currentBatch.length).toEqual(0) + expect(producer.currentBatchSize).toEqual(0) + expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ + topicMessages: [expect.anything()], + }) + }) + it('respects KAFKA_FLUSH_FREQUENCY_MS', async () => { await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(10) }], + messages: [{ value: '1'.repeat(10) }], }) jest.spyOn(global.Date, 'now').mockImplementation(() => new Date('2020-02-27 11:00:20').getTime()) await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(10) }], + messages: [{ value: '1'.repeat(10) }], }) expect(flushSpy).not.toHaveBeenCalled() @@ -97,7 +113,7 @@ describe('KafkaProducerWrapper', () => { jest.spyOn(global.Date, 'now').mockImplementation(() => new Date('2020-02-27 11:00:26').getTime()) await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(10) }], + messages: [{ value: '1'.repeat(10) }], }) expect(flushSpy).toHaveBeenCalled() @@ -114,7 +130,7 @@ describe('KafkaProducerWrapper', () => { it('flushes messages in memory', async () => { await producer.queueMessage({ topic: 'a', - messages: [{ value: Buffer.alloc(10) }], + messages: [{ value: '1'.repeat(10) }], }) jest.spyOn(global.Date, 'now').mockImplementation(() => new Date('2020-02-27 11:00:15').getTime()) @@ -125,7 +141,7 @@ describe('KafkaProducerWrapper', () => { topicMessages: [ { topic: 'a', - messages: [{ value: Buffer.alloc(10) }], + messages: [{ value: '1'.repeat(10) }], }, ], }) From 62ec001461f9fb03dea91f6c8bce7924cb2529af Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 27 Jul 2022 12:38:45 +0100 Subject: [PATCH 187/213] chore: add assertion to test (#11004) * chore: add assertion to test * reorganise assertions --- frontend/src/scenes/persons/personsLogic.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/scenes/persons/personsLogic.test.ts b/frontend/src/scenes/persons/personsLogic.test.ts index 9ef69df38042df..37021253203b21 100644 --- a/frontend/src/scenes/persons/personsLogic.test.ts +++ b/frontend/src/scenes/persons/personsLogic.test.ts @@ -4,6 +4,7 @@ import { personsLogic } from './personsLogic' import { router } from 'kea-router' import { PropertyOperator } from '~/types' import { useMocks } from '~/mocks/jest' +import api from 'lib/api' describe('personsLogic', () => { let logic: ReturnType @@ -109,8 +110,11 @@ describe('personsLogic', () => { }) it('loads a person where id includes +', async () => { + jest.spyOn(api, 'get') await expectLogic(logic, () => { logic.actions.loadPerson('+') + // has encoded from + in the action to %2B in the API call + expect(api.get).toHaveBeenCalledWith('api/person/?distinct_id=%2B') }) .toDispatchActions(['loadPerson', 'loadPersonSuccess']) .toMatchValues({ From 0549ecdd75565100e4f88385486f41e5e60305c4 Mon Sep 17 00:00:00 2001 From: timgl Date: Wed, 27 Jul 2022 13:39:20 +0200 Subject: [PATCH 188/213] chore(billing): Clarify when you'll be charged (#11005) --- frontend/src/scenes/billing/BillingLocked.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/scenes/billing/BillingLocked.tsx b/frontend/src/scenes/billing/BillingLocked.tsx index bd446011b8e56c..e68528a4d51243 100644 --- a/frontend/src/scenes/billing/BillingLocked.tsx +++ b/frontend/src/scenes/billing/BillingLocked.tsx @@ -26,6 +26,9 @@ export function BillingLocked(): JSX.Element | null { our website for pricing information. +
+
+ You'll only be charged for events from the moment you put your credit card details in.

Date: Wed, 27 Jul 2022 14:15:55 +0200 Subject: [PATCH 189/213] feat: Removed Slack Subs flag (#11007) --- .../SubscriptionsModal.stories.tsx | 4 +--- .../Subscriptions/views/EditSubscription.tsx | 11 +++------- frontend/src/lib/constants.tsx | 1 - .../Settings/SlackIntegration.stories.tsx | 4 +--- .../src/scenes/project/Settings/index.tsx | 20 ++++++++----------- 5 files changed, 13 insertions(+), 27 deletions(-) diff --git a/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx b/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx index cd2a9bdc8500d1..823fc9046c1ca7 100644 --- a/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx +++ b/frontend/src/lib/components/Subscriptions/SubscriptionsModal.stories.tsx @@ -5,8 +5,7 @@ import { AvailableFeature, InsightShortId, Realm } from '~/types' import preflightJson from '~/mocks/fixtures/_preflight.json' import { useAvailableFeatures } from '~/mocks/features' import { uuid } from 'lib/utils' -import { useFeatureFlags, useStorybookMocks } from '~/mocks/browser' -import { FEATURE_FLAGS } from 'lib/constants' +import { useStorybookMocks } from '~/mocks/browser' import { LemonButton } from '../LemonButton' import { createMockSubscription, mockIntegration, mockSlackChannels } from '~/test/mocks' @@ -24,7 +23,6 @@ const Template = ( const [modalOpen, setModalOpen] = useState(false) useAvailableFeatures(featureAvailable ? [AvailableFeature.SUBSCRIPTIONS] : []) - useFeatureFlags([FEATURE_FLAGS.SUBSCRIPTIONS_SLACK]) useStorybookMocks({ get: { diff --git a/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx b/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx index 50b9a3c4de2f90..8d8854b78f300d 100644 --- a/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx +++ b/frontend/src/lib/components/Subscriptions/views/EditSubscription.tsx @@ -30,8 +30,6 @@ import { } from 'lib/components/LemonSelectMultiple/LemonSelectMultiple' import { usersLemonSelectOptions } from 'lib/components/UserSelectItem' import { integrationsLogic } from 'scenes/project/Settings/integrationsLogic' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import { FEATURE_FLAGS } from 'lib/constants' import { urls } from 'scenes/urls' import { Skeleton } from 'antd' @@ -66,7 +64,6 @@ export function EditSubscription({ const { deleteSubscription } = useActions(subscriptionslogic) const { slackChannels, slackChannelsLoading, slackIntegration, addToSlackButtonUrl } = useValues(integrationsLogic) const { loadSlackChannels } = useActions(integrationsLogic) - const { featureFlags } = useValues(featureFlagLogic) const emailDisabled = !preflight?.email_service_available const slackDisabled = !slackIntegration @@ -167,11 +164,9 @@ export function EditSubscription({ - {featureFlags[FEATURE_FLAGS.SUBSCRIPTIONS_SLACK] && ( - - - - )} + + + {subscription.target_type === 'email' ? ( <> diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index c63c67662933fb..3bca5d87ce0563 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -112,7 +112,6 @@ export const FEATURE_FLAGS = { ONBOARDING_1_5: 'onboarding-1_5', // owner: @liyiy BREAKDOWN_ATTRIBUTION: 'breakdown-attribution', // owner: @neilkakkar SIMPLIFY_ACTIONS: 'simplify-actions', // owner: @alexkim205, - SUBSCRIPTIONS_SLACK: 'subscriptions-slack', // owner: @benjackwhite SESSION_ANALYSIS: 'session-analysis', // owner: @rcmarron TOOLBAR_LAUNCH_SIDE_ACTION: 'toolbar-launch-side-action', // owner: @pauldambra, FEATURE_FLAG_EXPERIENCE_CONTINUITY: 'feature-flag-experience-continuity', // owner: @neilkakkar diff --git a/frontend/src/scenes/project/Settings/SlackIntegration.stories.tsx b/frontend/src/scenes/project/Settings/SlackIntegration.stories.tsx index 081379a4cb4c70..314514c952fc67 100644 --- a/frontend/src/scenes/project/Settings/SlackIntegration.stories.tsx +++ b/frontend/src/scenes/project/Settings/SlackIntegration.stories.tsx @@ -2,8 +2,7 @@ import React from 'react' import { ComponentMeta } from '@storybook/react' import { AvailableFeature } from '~/types' import { useAvailableFeatures } from '~/mocks/features' -import { useFeatureFlags, useStorybookMocks } from '~/mocks/browser' -import { FEATURE_FLAGS } from 'lib/constants' +import { useStorybookMocks } from '~/mocks/browser' import { mockIntegration } from '~/test/mocks' import { SlackIntegration } from './SlackIntegration' @@ -17,7 +16,6 @@ const Template = (args: { instanceConfigured?: boolean; integrated?: boolean }): const { instanceConfigured = true, integrated = false } = args useAvailableFeatures([AvailableFeature.SUBSCRIPTIONS]) - useFeatureFlags([FEATURE_FLAGS.SUBSCRIPTIONS_SLACK]) useStorybookMocks({ get: { diff --git a/frontend/src/scenes/project/Settings/index.tsx b/frontend/src/scenes/project/Settings/index.tsx index 86ac0bad360be2..6ce6442f80ff41 100644 --- a/frontend/src/scenes/project/Settings/index.tsx +++ b/frontend/src/scenes/project/Settings/index.tsx @@ -15,7 +15,7 @@ import { PageHeader } from 'lib/components/PageHeader' import { Link } from 'lib/components/Link' import { JSBookmarklet } from 'lib/components/JSBookmarklet' import { RestrictedArea, RestrictionScope } from 'lib/components/RestrictedArea' -import { FEATURE_FLAGS, OrganizationMembershipLevel } from 'lib/constants' +import { OrganizationMembershipLevel } from 'lib/constants' import { TestAccountFiltersConfig } from './TestAccountFiltersConfig' import { TimezoneConfig } from './TimezoneConfig' import { DataAttributes } from 'scenes/project/Settings/DataAttributes' @@ -35,7 +35,6 @@ import { IconInfo, IconRefresh } from 'lib/components/icons' import { PersonDisplayNameProperties } from './PersonDisplayNameProperties' import { Tooltip } from 'lib/components/Tooltip' import { SlackIntegration } from './SlackIntegration' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' export const scene: SceneExport = { component: ProjectSettings, @@ -86,7 +85,6 @@ export function ProjectSettings(): JSX.Element { const { location } = useValues(router) const { user, hasAvailableFeature } = useValues(userLogic) const hasAdvancedPaths = user?.organization?.available_features?.includes(AvailableFeature.PATHS_ADVANCED) - const { featureFlags } = useValues(featureFlagLogic) useAnchor(location.hash) @@ -283,15 +281,13 @@ export function ProjectSettings(): JSX.Element {

- {featureFlags[FEATURE_FLAGS.SUBSCRIPTIONS_SLACK] && ( - <> -

- Slack integration -

- - - - )} + <> +

+ Slack integration +

+ + +

Data capture configuration

From e70a5a0d7b56dce6e462f4af9b81376167984408 Mon Sep 17 00:00:00 2001 From: Ben White Date: Wed, 27 Jul 2022 15:26:20 +0200 Subject: [PATCH 190/213] fix: Revert posthog-js alpha (#11009) --- frontend/src/loadPostHogJS.tsx | 5 +++-- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/frontend/src/loadPostHogJS.tsx b/frontend/src/loadPostHogJS.tsx index d4721c985aa9b7..01e9f20fa930fd 100644 --- a/frontend/src/loadPostHogJS.tsx +++ b/frontend/src/loadPostHogJS.tsx @@ -1,8 +1,8 @@ -import posthog, { PostHogConfig } from 'posthog-js' +import posthog from 'posthog-js' import * as Sentry from '@sentry/react' import { Integration } from '@sentry/types' -const configWithSentry = (config: Partial): Partial => { +const configWithSentry = (config: posthog.Config): posthog.Config => { if ((window as any).SENTRY_DSN) { config.on_xhr_error = (failedRequest: XMLHttpRequest) => { const status = failedRequest.status @@ -22,6 +22,7 @@ export function loadPostHogJS(): void { window.JS_POSTHOG_API_KEY, configWithSentry({ api_host: window.JS_POSTHOG_HOST, + // @ts-expect-error _capture_metrics: true, rageclick: true, debug: window.JS_POSTHOG_SELF_CAPTURE, diff --git a/package.json b/package.json index e7c829336450bf..dbfc3335926e7c 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "kea-window-values": "^3.0.0", "md5": "^2.3.0", "monaco-editor": "^0.23.0", - "posthog-js": "1.27.0-alpha4", + "posthog-js": "1.26.0", "posthog-js-lite": "^0.0.3", "prop-types": "^15.7.2", "query-selector-shadow-dom": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index e6ba4519ae4906..8a67d60eee595f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14105,10 +14105,10 @@ posthog-js-lite@^0.0.3: resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.3.tgz#87e373706227a849c4e7c6b0cb2066a64ad5b6ed" integrity sha512-wEOs8DEjlFBwgd7l19grosaF7mTlliZ9G9pL0Qji189FDg2ukY5IegUxTyTs7gsTGt6WK9W47BF5yXA5+bwvZg== -posthog-js@1.27.0-alpha4: - version "1.27.0-alpha4" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.27.0-alpha4.tgz#9a5047ba8bd1a78589212c16290e4c651ece0504" - integrity sha512-C1vjAPtRXQk3KxGToDUyRw5aqB7/lsuRkIip76EdK7NAFaXpxXrplK+RT1I/m+TNVoAehK7EGnS/h046kL4TaA== +posthog-js@1.26.0: + version "1.26.0" + resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.26.0.tgz#8fd2becfbdf8f165244043d109f140ea0d02a99b" + integrity sha512-Fjc5REUJxrVTQ0WzfChn+CW/UrparZGwINPOtO9FoB25U2IrXnvnTUpkYhSPucZPWUwUMRdXBCo9838COn464Q== dependencies: "@sentry/types" "^7.2.0" fflate "^0.4.1" From cbb5ea25020c975f0ffd0ed1659b118752e51bfe Mon Sep 17 00:00:00 2001 From: Alex Gyujin Kim Date: Wed, 27 Jul 2022 09:39:24 -0400 Subject: [PATCH 191/213] refactor(data-m): toolbar rename actions to events (#10989) --- frontend/src/toolbar/actions/ActionsList.tsx | 5 +++-- frontend/src/toolbar/actions/ActionsTab.tsx | 10 ++++++++-- frontend/src/toolbar/actions/EditAction.tsx | 8 ++++++-- frontend/src/toolbar/button/DraggableButton.tsx | 6 +++--- frontend/src/toolbar/button/ToolbarButton.tsx | 10 +++++++++- frontend/src/toolbar/elements/ElementInfo.tsx | 10 +++++++--- frontend/src/toolbar/flags/featureFlagsLogic.ts | 6 ++++++ 7 files changed, 42 insertions(+), 13 deletions(-) diff --git a/frontend/src/toolbar/actions/ActionsList.tsx b/frontend/src/toolbar/actions/ActionsList.tsx index 963b5a33ca8494..e05d0afc9d8bf8 100644 --- a/frontend/src/toolbar/actions/ActionsList.tsx +++ b/frontend/src/toolbar/actions/ActionsList.tsx @@ -6,11 +6,12 @@ import { PlusOutlined } from '@ant-design/icons' import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic' import { ActionsListView } from '~/toolbar/actions/ActionsListView' import { Spinner } from 'lib/components/Spinner/Spinner' +import { featureFlagsLogic } from '~/toolbar/flags/featureFlagsLogic' export function ActionsList(): JSX.Element { const { allActions, sortedActions, allActionsLoading, searchTerm } = useValues(actionsLogic) const { setSearchTerm } = useActions(actionsLogic) - + const { shouldSimplifyActions } = useValues(featureFlagsLogic) const { newAction } = useActions(actionsTabLogic) return ( @@ -28,7 +29,7 @@ export function ActionsList(): JSX.Element {
{allActions.length === 0 && allActionsLoading ? ( diff --git a/frontend/src/toolbar/actions/ActionsTab.tsx b/frontend/src/toolbar/actions/ActionsTab.tsx index 185bba52883341..dc94871a5992b1 100644 --- a/frontend/src/toolbar/actions/ActionsTab.tsx +++ b/frontend/src/toolbar/actions/ActionsTab.tsx @@ -9,10 +9,12 @@ import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic' import { EditAction } from '~/toolbar/actions/EditAction' import { ExportOutlined } from '@ant-design/icons' import { urls } from 'scenes/urls' +import { featureFlagsLogic } from '~/toolbar/flags/featureFlagsLogic' export function ActionsTab(): JSX.Element { const { selectedAction } = useValues(actionsTabLogic) const { apiURL } = useValues(toolbarLogic) + const { shouldSimplifyActions } = useValues(featureFlagsLogic) return (
@@ -23,8 +25,12 @@ export function ActionsTab(): JSX.Element { <> diff --git a/frontend/src/toolbar/actions/EditAction.tsx b/frontend/src/toolbar/actions/EditAction.tsx index dbf9c7e7ec87f8..ca5e6c677a5fb7 100644 --- a/frontend/src/toolbar/actions/EditAction.tsx +++ b/frontend/src/toolbar/actions/EditAction.tsx @@ -10,6 +10,7 @@ import { CloseOutlined, DeleteOutlined, } from '@ant-design/icons' +import { featureFlagsLogic } from '~/toolbar/flags/featureFlagsLogic' export function EditAction(): JSX.Element { const [form] = Form.useForm() @@ -17,6 +18,7 @@ export function EditAction(): JSX.Element { const { initialValuesForForm, selectedActionId, inspectingElement, editingFields } = useValues(actionsTabLogic) const { selectAction, inspectForElementWithIndex, setEditingFields, setForm, saveAction, deleteAction } = useActions(actionsTabLogic) + const { shouldSimplifyActions } = useValues(featureFlagsLogic) const { getFieldValue } = form @@ -41,7 +43,8 @@ export function EditAction(): JSX.Element { Cancel

- {selectedActionId === 'new' ? 'New Action' : 'Edit Action'} + {selectedActionId === 'new' ? 'New ' : 'Edit '} + {shouldSimplifyActions ? 'Event' : 'Action'}

) : null}
diff --git a/frontend/src/toolbar/button/DraggableButton.tsx b/frontend/src/toolbar/button/DraggableButton.tsx index 1f0f20aa3b8c2e..de78119ca08392 100644 --- a/frontend/src/toolbar/button/DraggableButton.tsx +++ b/frontend/src/toolbar/button/DraggableButton.tsx @@ -29,7 +29,7 @@ export function DraggableButton(): JSX.Element { hideFlags, saveFlagsPosition, } = useActions(toolbarButtonLogic) - const { countFlagsOverridden } = useValues(featureFlagsLogic) + const { countFlagsOverridden, shouldSimplifyActions } = useValues(featureFlagsLogic) return ( <> @@ -63,8 +63,8 @@ export function DraggableButton(): JSX.Element { (side === 'right' && r < 0 ? 360 : 0)} - label={buttonActionsVisible && (!allActionsLoading || actionCount > 0) ? null : 'Actions'} + label={ + buttonActionsVisible && (!allActionsLoading || actionCount > 0) + ? null + : shouldSimplifyActions + ? 'Events' + : 'Actions' + } labelPosition={side === 'left' ? 'right' : 'left'} labelStyle={{ opacity: actionsExtensionPercentage > 0.8 ? (actionsExtensionPercentage - 0.8) / 0.2 : 0, diff --git a/frontend/src/toolbar/elements/ElementInfo.tsx b/frontend/src/toolbar/elements/ElementInfo.tsx index 35b52e4804245e..682063ec310565 100644 --- a/frontend/src/toolbar/elements/ElementInfo.tsx +++ b/frontend/src/toolbar/elements/ElementInfo.tsx @@ -6,12 +6,14 @@ import { heatmapLogic } from '~/toolbar/elements/heatmapLogic' import { Button, Statistic, Row, Col } from 'antd' import { elementsLogic } from '~/toolbar/elements/elementsLogic' import { ActionsListView } from '~/toolbar/actions/ActionsListView' +import { featureFlagsLogic } from '~/toolbar/flags/featureFlagsLogic' export function ElementInfo(): JSX.Element | null { const { clickCount } = useValues(heatmapLogic) const { hoverElementMeta, selectedElementMeta } = useValues(elementsLogic) const { createAction } = useActions(elementsLogic) + const { shouldSimplifyActions } = useValues(featureFlagsLogic) const activeMeta = hoverElementMeta || selectedElementMeta @@ -52,16 +54,18 @@ export function ElementInfo(): JSX.Element | null { ) : null}
-

Actions ({activeMeta.actions.length})

+

+ {shouldSimplifyActions ? 'Events' : 'Actions'} ({activeMeta.actions.length}) +

{activeMeta.actions.length === 0 ? ( -

No actions include this element

+

No {shouldSimplifyActions ? 'events' : 'actions'} include this element

) : ( a.action)} /> )}
diff --git a/frontend/src/toolbar/flags/featureFlagsLogic.ts b/frontend/src/toolbar/flags/featureFlagsLogic.ts index c1ca91e7a5904b..abfab85d2a23dd 100644 --- a/frontend/src/toolbar/flags/featureFlagsLogic.ts +++ b/frontend/src/toolbar/flags/featureFlagsLogic.ts @@ -7,6 +7,7 @@ import Fuse from 'fuse.js' import { PostHog } from 'posthog-js' import { posthog } from '~/toolbar/posthog' import { encodeParams } from 'kea-router' +import { FEATURE_FLAGS } from 'lib/constants' export const featureFlagsLogic = kea({ path: ['toolbar', 'flags', 'featureFlagsLogic'], @@ -125,6 +126,11 @@ export const featureFlagsLogic = kea({ }, ], countFlagsOverridden: [(s) => [s.localOverrides], (localOverrides) => Object.keys(localOverrides).length], + // Remove once `simplify-actions` FF is released + shouldSimplifyActions: [ + (s) => [s.userFlagsWithOverrideInfo], + (flags) => flags.find((f) => f.feature_flag.name === FEATURE_FLAGS.SIMPLIFY_ACTIONS)?.currentValue || false, + ], }, events: ({ actions }) => ({ afterMount: async () => { From 72aef6c9e4210a277c2b699a17a2de6ab6e96288 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Wed, 27 Jul 2022 15:08:46 +0100 Subject: [PATCH 192/213] feat(feature-flags): endpoint and test for local sdk evaluation (#10957) --- posthog/api/__init__.py | 3 +- posthog/api/feature_flag.py | 30 + posthog/api/test/test_feature_flag.py | 129 ++ posthog/queries/trends/total_volume.py | 1 + posthog/test/test_feature_flag.py | 2062 ++++++++++++++++++++++++ 5 files changed, 2224 insertions(+), 1 deletion(-) diff --git a/posthog/api/__init__.py b/posthog/api/__init__.py index 98ba9c751f9c70..1da44ca0b87226 100644 --- a/posthog/api/__init__.py +++ b/posthog/api/__init__.py @@ -41,11 +41,12 @@ def api_not_found(request): # Legacy endpoints shared (to be removed eventually) router.register(r"annotation", annotation.LegacyAnnotationsViewSet) # Should be completely unused now -router.register(r"feature_flag", feature_flag.LegacyFeatureFlagViewSet) # Should be completely unused now router.register(r"dashboard", dashboard.LegacyDashboardsViewSet) # Should be completely unused now router.register(r"dashboard_item", dashboard.LegacyInsightViewSet) # To be deleted - unified into insight viewset router.register(r"plugin_config", plugin.LegacyPluginConfigViewSet) +router.register(r"feature_flag", feature_flag.LegacyFeatureFlagViewSet) # Used for library side feature flag evaluation + # Nested endpoints shared projects_router = router.register(r"projects", team.TeamViewSet) project_plugins_configs_router = projects_router.register( diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index f634a92db169a5..eaf14e20559710 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -17,6 +17,7 @@ from posthog.models.activity_logging.activity_page import activity_page_response from posthog.models.cohort import Cohort from posthog.models.feature_flag import FeatureFlagMatcher +from posthog.models.group_type_mapping import GroupTypeMapping from posthog.models.property import Property from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission @@ -226,6 +227,35 @@ def my_flags(self, request: request.Request, **kwargs): ) return Response(flags) + @action(methods=["GET"], detail=False) + def local_evaluation(self, request: request.Request, **kwargs): + + feature_flags = ( + FeatureFlag.objects.filter(team=self.team, active=True, deleted=False) + .prefetch_related("experiment_set") + .select_related("created_by") + .order_by("-created_at") + ) + + parsed_flags = [] + for feature_flag in feature_flags: + filters = feature_flag.get_filters() + feature_flag.filters = filters + parsed_flags.append(feature_flag) + + # TODO: Handle cohorts the same way as feature evaluation would, by simplifying cohort properties to + # person properties + + return Response( + { + "flags": [FeatureFlagSerializer(feature_flag).data for feature_flag in parsed_flags], + "group_type_mapping": { + str(row.group_type_index): row.group_type + for row in GroupTypeMapping.objects.filter(team_id=self.team_id) + }, + } + ) + @action(methods=["GET"], url_path="activity", detail=False) def all_activity(self, request: request.Request, **kwargs): limit = int(request.query_params.get("limit", "10")) diff --git a/posthog/api/test/test_feature_flag.py b/posthog/api/test/test_feature_flag.py index c03ff2381c6c35..71642866e451b4 100644 --- a/posthog/api/test/test_feature_flag.py +++ b/posthog/api/test/test_feature_flag.py @@ -8,6 +8,7 @@ from posthog.models import FeatureFlag, GroupTypeMapping, User from posthog.models.cohort import Cohort +from posthog.models.personal_api_key import PersonalAPIKey from posthog.test.base import APIBaseTest from posthog.test.db_context_capturing import capture_db_queries @@ -942,6 +943,134 @@ def test_my_flags_groups(self, mock_capture): self.assertEqual(groups_flag["feature_flag"]["key"], "groups-flag") self.assertEqual(groups_flag["value"], True) + @patch("posthog.api.feature_flag.report_user_action") + def test_local_evaluation(self, mock_capture): + FeatureFlag.objects.all().delete() + GroupTypeMapping.objects.create(team=self.team, group_type="organization", group_type_index=0) + GroupTypeMapping.objects.create(team=self.team, group_type="company", group_type_index=1) + + self.client.post( + f"/api/projects/{self.team.id}/feature_flags/", + { + "name": "Alpha feature", + "key": "alpha-feature", + "filters": { + "groups": [{"rollout_percentage": 20}], + "multivariate": { + "variants": [ + {"key": "first-variant", "name": "First Variant", "rollout_percentage": 50}, + {"key": "second-variant", "name": "Second Variant", "rollout_percentage": 25}, + {"key": "third-variant", "name": "Third Variant", "rollout_percentage": 25}, + ], + }, + }, + }, + format="json", + ) + + self.client.post( + f"/api/projects/{self.team.id}/feature_flags/", + { + "name": "Group feature", + "key": "group-feature", + "filters": {"aggregation_group_type_index": 0, "groups": [{"rollout_percentage": 21}],}, + }, + format="json", + ) + + # old style feature flags + FeatureFlag.objects.create( + name="Beta feature", + key="beta-feature", + team=self.team, + rollout_percentage=51, + filters={"properties": [{"key": "beta-property", "value": "beta-value"}]}, + created_by=self.user, + ) + + key = PersonalAPIKey(label="Test", user=self.user) + key.save() + personal_api_key = key.value + + self.client.logout() + # `local_evaluation` is called by logged out clients! + + # missing API key + response = self.client.get(f"/api/feature_flag/local_evaluation?token={self.team.api_token}") + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + response = self.client.get(f"/api/feature_flag/local_evaluation") + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + response = self.client.get( + f"/api/feature_flag/local_evaluation?token={self.team.api_token}", + HTTP_AUTHORIZATION=f"Bearer {personal_api_key}", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_data = response.json() + self.assertTrue("flags" in response_data and "group_type_mapping" in response_data) + self.assertEqual(len(response_data["flags"]), 3) + + sorted_flags = sorted(response_data["flags"], key=lambda x: x["key"]) + + self.assertDictContainsSubset( + { + "name": "Alpha feature", + "key": "alpha-feature", + "filters": { + "groups": [{"rollout_percentage": 20}], + "multivariate": { + "variants": [ + {"key": "first-variant", "name": "First Variant", "rollout_percentage": 50}, + {"key": "second-variant", "name": "Second Variant", "rollout_percentage": 25}, + {"key": "third-variant", "name": "Third Variant", "rollout_percentage": 25}, + ] + }, + }, + "deleted": False, + "active": True, + "is_simple_flag": True, + "rollout_percentage": 20, + "ensure_experience_continuity": False, + "experiment_set": [], + }, + sorted_flags[0], + ) + self.assertDictContainsSubset( + { + "name": "Beta feature", + "key": "beta-feature", + "filters": { + "groups": [ + {"properties": [{"key": "beta-property", "value": "beta-value"}], "rollout_percentage": 51} + ] + }, + "deleted": False, + "active": True, + "is_simple_flag": False, + "rollout_percentage": None, + "ensure_experience_continuity": False, + "experiment_set": [], + }, + sorted_flags[1], + ) + self.assertDictContainsSubset( + { + "name": "Group feature", + "key": "group-feature", + "filters": {"groups": [{"rollout_percentage": 21}], "aggregation_group_type_index": 0,}, + "deleted": False, + "active": True, + "is_simple_flag": False, + "rollout_percentage": None, + "ensure_experience_continuity": False, + "experiment_set": [], + }, + sorted_flags[2], + ) + + self.assertEqual(response_data["group_type_mapping"], {"0": "organization", "1": "company",}) + def test_validation_person_properties(self): person_request = self._create_flag_with_properties( "person-flag", [{"key": "email", "type": "person", "value": "@posthog.com", "operator": "icontains",},] diff --git a/posthog/queries/trends/total_volume.py b/posthog/queries/trends/total_volume.py index a9d47e298dc935..766e1c811020e7 100644 --- a/posthog/queries/trends/total_volume.py +++ b/posthog/queries/trends/total_volume.py @@ -73,6 +73,7 @@ def _total_volume_query(self, entity: Entity, filter: Filter, team: Team) -> Tup **trend_event_query.active_user_params, ) elif filter.display == TRENDS_CUMULATIVE and entity.math == "dau": + # TODO: for groups aggregation as well cumulative_sql = CUMULATIVE_SQL.format(event_query=event_query) content_sql = VOLUME_SQL.format( event_query=cumulative_sql, start_of_week_fix=start_of_week_fix(filter), **content_sql_params, diff --git a/posthog/test/test_feature_flag.py b/posthog/test/test_feature_flag.py index d6e6468963ddc9..2a2e9b1d3acb80 100644 --- a/posthog/test/test_feature_flag.py +++ b/posthog/test/test_feature_flag.py @@ -1,3 +1,5 @@ +from typing import cast + from django.db import connection from posthog.models import Cohort, FeatureFlag, GroupTypeMapping, Person @@ -620,3 +622,2063 @@ def test_entire_flow_with_hash_key_override(self): # get feature flags for 'other_id', with an override for 'example_id' flags = get_active_feature_flags(self.team.pk, "other_id", {}, "example_id") self.assertEqual(flags, {"beta-feature": True, "multivariate-flag": "first-variant", "default-flag": True,}) + + +class TestFeatureFlagMatcherConsistency(BaseTest): + # These tests are common between all libraries doing local evaluation of feature flags. + # This ensures there are no mismatches between implementations. + + def test_simple_flag_consistency(self): + feature_flag = FeatureFlag.objects.create( + team=self.team, + name="Simple flag", + key="simple-flag", + created_by=self.user, + filters={"groups": [{"properties": [], "rollout_percentage": 45}],}, + ) + + results = [ + False, + True, + True, + False, + True, + False, + False, + True, + False, + True, + False, + True, + True, + False, + True, + False, + False, + False, + True, + True, + False, + True, + False, + False, + True, + False, + True, + True, + False, + False, + False, + True, + True, + True, + True, + False, + False, + False, + False, + False, + False, + True, + True, + False, + True, + True, + False, + False, + False, + True, + True, + False, + False, + False, + False, + True, + False, + True, + False, + True, + False, + True, + True, + False, + True, + False, + True, + False, + True, + True, + False, + False, + True, + False, + False, + True, + False, + True, + False, + False, + True, + False, + False, + False, + True, + True, + False, + True, + True, + False, + True, + True, + True, + True, + True, + False, + True, + True, + False, + False, + True, + True, + True, + True, + False, + False, + True, + False, + True, + True, + True, + False, + False, + False, + False, + False, + True, + False, + False, + True, + True, + True, + False, + False, + True, + False, + True, + False, + False, + True, + False, + False, + False, + False, + False, + False, + False, + False, + True, + True, + False, + False, + True, + False, + False, + True, + True, + False, + False, + True, + False, + True, + False, + True, + True, + True, + False, + False, + False, + True, + False, + False, + False, + False, + True, + True, + False, + True, + True, + False, + True, + False, + True, + True, + False, + True, + False, + True, + True, + True, + False, + True, + False, + False, + True, + True, + False, + True, + False, + True, + True, + False, + False, + True, + True, + True, + True, + False, + True, + True, + False, + False, + True, + False, + True, + False, + False, + True, + True, + False, + True, + False, + True, + False, + False, + False, + False, + False, + False, + False, + True, + False, + True, + True, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, + False, + True, + False, + True, + True, + False, + False, + False, + False, + True, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + True, + True, + False, + True, + False, + True, + True, + False, + True, + False, + True, + False, + False, + False, + True, + True, + True, + True, + False, + False, + False, + False, + False, + True, + True, + True, + False, + False, + True, + True, + False, + False, + False, + False, + False, + True, + False, + True, + True, + True, + True, + False, + True, + True, + True, + False, + False, + True, + False, + True, + False, + False, + True, + True, + True, + False, + True, + False, + False, + False, + True, + True, + False, + True, + False, + True, + False, + True, + True, + True, + True, + True, + False, + False, + True, + False, + True, + False, + True, + True, + True, + False, + True, + False, + True, + True, + False, + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + True, + False, + True, + False, + False, + True, + True, + False, + False, + False, + True, + False, + True, + True, + True, + True, + False, + False, + False, + False, + True, + True, + False, + False, + True, + True, + False, + True, + True, + True, + True, + False, + True, + True, + True, + False, + False, + True, + True, + False, + False, + True, + False, + False, + True, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + True, + True, + False, + False, + True, + False, + False, + True, + False, + True, + False, + False, + True, + False, + False, + False, + False, + False, + False, + True, + False, + False, + False, + False, + False, + False, + False, + False, + False, + True, + True, + True, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, + False, + False, + False, + False, + False, + False, + True, + False, + False, + False, + False, + False, + False, + False, + False, + True, + False, + True, + False, + True, + True, + True, + False, + False, + False, + True, + True, + True, + False, + True, + False, + True, + True, + False, + False, + False, + True, + False, + False, + False, + False, + True, + False, + True, + False, + True, + True, + False, + True, + False, + False, + False, + True, + False, + False, + True, + True, + False, + True, + False, + False, + False, + False, + False, + False, + True, + True, + False, + False, + True, + False, + False, + True, + True, + True, + False, + False, + False, + True, + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + False, + True, + True, + False, + True, + False, + True, + False, + True, + False, + False, + True, + False, + False, + True, + False, + True, + False, + True, + False, + True, + False, + False, + True, + True, + True, + True, + False, + True, + False, + False, + False, + False, + False, + True, + False, + False, + True, + False, + False, + True, + True, + False, + False, + False, + False, + True, + True, + True, + False, + False, + True, + False, + False, + True, + True, + True, + True, + False, + False, + False, + True, + False, + False, + False, + True, + False, + False, + True, + True, + True, + True, + False, + False, + True, + True, + False, + True, + False, + True, + False, + False, + True, + True, + False, + True, + True, + True, + True, + False, + False, + True, + False, + False, + True, + True, + False, + True, + False, + True, + False, + False, + True, + False, + False, + False, + False, + True, + True, + True, + False, + True, + False, + False, + True, + False, + False, + True, + False, + False, + False, + False, + True, + False, + True, + False, + True, + True, + False, + False, + True, + False, + True, + True, + True, + False, + False, + False, + False, + True, + True, + False, + True, + False, + False, + False, + True, + False, + False, + False, + False, + True, + True, + True, + False, + False, + False, + True, + True, + True, + True, + False, + True, + True, + False, + True, + True, + True, + False, + True, + False, + False, + True, + False, + True, + True, + True, + True, + False, + True, + False, + True, + False, + True, + False, + False, + True, + True, + False, + False, + True, + False, + True, + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + True, + True, + True, + False, + False, + False, + True, + False, + True, + True, + False, + False, + False, + False, + False, + True, + False, + True, + False, + False, + True, + True, + False, + True, + True, + True, + True, + False, + False, + True, + False, + False, + True, + False, + True, + False, + True, + True, + False, + False, + False, + True, + False, + True, + True, + False, + False, + False, + True, + False, + True, + False, + True, + True, + False, + True, + False, + False, + True, + False, + False, + False, + True, + True, + True, + False, + False, + False, + False, + False, + True, + False, + False, + True, + True, + True, + True, + True, + False, + False, + False, + False, + False, + False, + False, + False, + True, + True, + True, + False, + False, + True, + True, + False, + True, + True, + False, + True, + False, + True, + False, + False, + False, + True, + False, + False, + True, + False, + False, + True, + True, + True, + True, + False, + False, + True, + False, + True, + True, + False, + False, + True, + False, + False, + True, + True, + False, + True, + False, + False, + True, + True, + True, + False, + False, + False, + False, + False, + True, + False, + True, + False, + False, + False, + False, + False, + True, + True, + False, + True, + True, + True, + False, + False, + False, + False, + True, + True, + True, + True, + False, + True, + True, + False, + True, + False, + True, + False, + True, + False, + False, + False, + False, + True, + True, + True, + True, + False, + False, + True, + False, + True, + True, + False, + False, + False, + False, + False, + False, + True, + False, + True, + False, + True, + True, + False, + False, + True, + True, + True, + True, + False, + False, + True, + False, + True, + True, + False, + False, + True, + True, + True, + False, + True, + False, + False, + True, + True, + False, + False, + False, + True, + False, + False, + True, + False, + False, + False, + True, + True, + True, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + False, + True, + False, + False, + True, + False, + True, + True, + ] + + for i in range(1000): + distinctID = f"distinct_id_{i}" + + feature_flag_match = FeatureFlagMatcher([feature_flag], distinctID).get_match(feature_flag) + + if results[i]: + self.assertEqual(feature_flag_match, FeatureFlagMatch()) + else: + self.assertIsNone(feature_flag_match) + + def test_multivariate_flag_consistency(self): + feature_flag = FeatureFlag.objects.create( + team=self.team, + name="Multivariate flag", + key="multivariate-flag", + created_by=self.user, + filters={ + "groups": [{"properties": [], "rollout_percentage": 55}], + "multivariate": { + "variants": [ + {"key": "first-variant", "name": "First Variant", "rollout_percentage": 50}, + {"key": "second-variant", "name": "Second Variant", "rollout_percentage": 20}, + {"key": "third-variant", "name": "Third Variant", "rollout_percentage": 20}, + {"key": "fourth-variant", "name": "Fourth Variant", "rollout_percentage": 5}, + {"key": "fifth-variant", "name": "Fifth Variant", "rollout_percentage": 5}, + ], + }, + }, + ) + + results = [ + "second-variant", + "second-variant", + "first-variant", + False, + False, + "second-variant", + "first-variant", + False, + False, + False, + "first-variant", + "third-variant", + False, + "first-variant", + "second-variant", + "first-variant", + False, + False, + "fourth-variant", + "first-variant", + False, + "third-variant", + False, + False, + False, + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "third-variant", + False, + "third-variant", + "second-variant", + "first-variant", + False, + "third-variant", + False, + False, + "first-variant", + "second-variant", + False, + "first-variant", + "first-variant", + "second-variant", + False, + "first-variant", + False, + False, + "first-variant", + "first-variant", + "first-variant", + "second-variant", + "first-variant", + False, + "second-variant", + "second-variant", + "third-variant", + "second-variant", + "first-variant", + False, + "first-variant", + "second-variant", + "fourth-variant", + False, + "first-variant", + "first-variant", + "first-variant", + False, + "first-variant", + "second-variant", + False, + "third-variant", + False, + False, + False, + False, + False, + False, + "first-variant", + "fifth-variant", + False, + "second-variant", + "first-variant", + "second-variant", + False, + "third-variant", + "third-variant", + False, + False, + False, + False, + "third-variant", + False, + False, + "first-variant", + "first-variant", + False, + "third-variant", + "third-variant", + False, + "third-variant", + "second-variant", + "third-variant", + False, + False, + "second-variant", + "first-variant", + False, + False, + "first-variant", + False, + False, + False, + False, + "first-variant", + "first-variant", + "first-variant", + False, + False, + False, + "first-variant", + "first-variant", + False, + "first-variant", + "first-variant", + False, + False, + False, + False, + False, + False, + False, + False, + False, + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "second-variant", + "first-variant", + "first-variant", + "first-variant", + "second-variant", + False, + "second-variant", + "first-variant", + "second-variant", + "first-variant", + False, + "second-variant", + "second-variant", + False, + "first-variant", + False, + False, + False, + "third-variant", + "first-variant", + False, + False, + "first-variant", + False, + False, + False, + False, + "first-variant", + False, + False, + False, + False, + False, + False, + False, + "first-variant", + "first-variant", + "third-variant", + "first-variant", + "first-variant", + False, + False, + "first-variant", + False, + False, + "fifth-variant", + "second-variant", + False, + "second-variant", + False, + "first-variant", + "third-variant", + "first-variant", + "fifth-variant", + "third-variant", + False, + False, + "fourth-variant", + False, + False, + False, + False, + "third-variant", + False, + False, + "third-variant", + False, + "first-variant", + "second-variant", + "second-variant", + "second-variant", + False, + "first-variant", + "third-variant", + "first-variant", + "first-variant", + False, + False, + False, + False, + False, + "first-variant", + "first-variant", + "first-variant", + "second-variant", + False, + False, + False, + "second-variant", + False, + False, + "first-variant", + False, + "first-variant", + False, + False, + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "third-variant", + "first-variant", + "third-variant", + "first-variant", + "first-variant", + "second-variant", + "third-variant", + "third-variant", + False, + "second-variant", + "first-variant", + False, + "second-variant", + "first-variant", + False, + "first-variant", + False, + False, + "first-variant", + "fifth-variant", + "first-variant", + False, + False, + False, + False, + "first-variant", + "first-variant", + "second-variant", + False, + "second-variant", + "third-variant", + "third-variant", + False, + "first-variant", + "third-variant", + False, + False, + "first-variant", + False, + "third-variant", + "first-variant", + False, + "third-variant", + "first-variant", + "first-variant", + False, + "first-variant", + "second-variant", + "second-variant", + "first-variant", + False, + False, + False, + "second-variant", + False, + False, + "first-variant", + "first-variant", + False, + "third-variant", + False, + "first-variant", + False, + "third-variant", + False, + "third-variant", + "second-variant", + "first-variant", + False, + False, + "first-variant", + "third-variant", + "first-variant", + "second-variant", + "fifth-variant", + False, + False, + "first-variant", + False, + False, + False, + "third-variant", + False, + "second-variant", + "first-variant", + False, + False, + False, + False, + "third-variant", + False, + False, + "third-variant", + False, + False, + "first-variant", + "third-variant", + False, + False, + "first-variant", + False, + False, + "fourth-variant", + "fourth-variant", + "third-variant", + "second-variant", + "first-variant", + "third-variant", + "fifth-variant", + False, + "first-variant", + "fifth-variant", + False, + "first-variant", + "first-variant", + "first-variant", + False, + False, + False, + "second-variant", + "fifth-variant", + "second-variant", + "first-variant", + "first-variant", + "second-variant", + False, + False, + "third-variant", + False, + "second-variant", + "fifth-variant", + False, + "third-variant", + "first-variant", + False, + False, + "fourth-variant", + False, + False, + "second-variant", + False, + False, + "first-variant", + "fourth-variant", + "first-variant", + "second-variant", + False, + False, + False, + "first-variant", + "third-variant", + "third-variant", + False, + "first-variant", + "first-variant", + "first-variant", + False, + "first-variant", + False, + "first-variant", + "third-variant", + "third-variant", + False, + False, + "first-variant", + False, + False, + "second-variant", + "second-variant", + "first-variant", + "first-variant", + "first-variant", + False, + "fifth-variant", + "first-variant", + False, + False, + False, + "second-variant", + "third-variant", + "first-variant", + "fourth-variant", + "first-variant", + "third-variant", + False, + "first-variant", + "first-variant", + False, + "third-variant", + "first-variant", + "first-variant", + "third-variant", + False, + "fourth-variant", + "fifth-variant", + "first-variant", + "first-variant", + False, + False, + False, + "first-variant", + "first-variant", + "first-variant", + False, + "first-variant", + "first-variant", + "second-variant", + "first-variant", + False, + "first-variant", + "second-variant", + "first-variant", + False, + "first-variant", + "second-variant", + False, + "first-variant", + "first-variant", + False, + "first-variant", + False, + "first-variant", + False, + "first-variant", + False, + False, + False, + "third-variant", + "third-variant", + "first-variant", + False, + False, + "second-variant", + "third-variant", + "first-variant", + "first-variant", + False, + False, + False, + "second-variant", + "first-variant", + False, + "first-variant", + "third-variant", + False, + "first-variant", + False, + False, + False, + "first-variant", + "third-variant", + "third-variant", + False, + False, + False, + False, + "third-variant", + "fourth-variant", + "fourth-variant", + "first-variant", + "second-variant", + False, + "first-variant", + False, + "second-variant", + "first-variant", + "third-variant", + False, + "third-variant", + False, + "first-variant", + "first-variant", + "third-variant", + False, + False, + False, + "fourth-variant", + "second-variant", + "first-variant", + False, + False, + "first-variant", + "fourth-variant", + False, + "first-variant", + "third-variant", + "first-variant", + False, + False, + "third-variant", + False, + "first-variant", + False, + "first-variant", + "first-variant", + "third-variant", + "second-variant", + "fourth-variant", + False, + "first-variant", + False, + False, + False, + False, + "second-variant", + "first-variant", + "second-variant", + False, + "first-variant", + False, + "first-variant", + "first-variant", + False, + "first-variant", + "first-variant", + "second-variant", + "third-variant", + "first-variant", + "first-variant", + "first-variant", + False, + False, + False, + "third-variant", + False, + "first-variant", + "first-variant", + "first-variant", + "third-variant", + "first-variant", + "first-variant", + "second-variant", + "first-variant", + "fifth-variant", + "fourth-variant", + "first-variant", + "second-variant", + False, + "fourth-variant", + False, + False, + False, + "fourth-variant", + False, + False, + "third-variant", + False, + False, + False, + "first-variant", + "third-variant", + "third-variant", + "second-variant", + "first-variant", + "second-variant", + "first-variant", + False, + "first-variant", + False, + False, + False, + False, + False, + "first-variant", + "first-variant", + False, + "second-variant", + False, + False, + "first-variant", + False, + "second-variant", + "first-variant", + "first-variant", + "first-variant", + "third-variant", + "second-variant", + False, + False, + "fifth-variant", + "third-variant", + False, + False, + "first-variant", + False, + False, + False, + "first-variant", + "second-variant", + "third-variant", + "third-variant", + False, + False, + "first-variant", + False, + "third-variant", + "first-variant", + False, + False, + False, + False, + "fourth-variant", + "first-variant", + False, + False, + False, + "third-variant", + False, + False, + "second-variant", + "first-variant", + False, + False, + "second-variant", + "third-variant", + "first-variant", + "first-variant", + False, + "first-variant", + "first-variant", + False, + False, + "second-variant", + "third-variant", + "second-variant", + "third-variant", + False, + False, + "first-variant", + False, + False, + "first-variant", + False, + "second-variant", + False, + False, + False, + False, + "first-variant", + False, + "third-variant", + False, + "first-variant", + False, + False, + "second-variant", + "third-variant", + "second-variant", + "fourth-variant", + "first-variant", + "first-variant", + "first-variant", + False, + "first-variant", + False, + "second-variant", + False, + False, + False, + False, + False, + "first-variant", + False, + False, + False, + False, + False, + "first-variant", + False, + "second-variant", + False, + False, + False, + False, + "second-variant", + False, + "first-variant", + False, + "third-variant", + False, + False, + "first-variant", + "third-variant", + False, + "third-variant", + False, + False, + "second-variant", + False, + "first-variant", + "second-variant", + "first-variant", + False, + False, + False, + False, + False, + "second-variant", + False, + False, + "first-variant", + "third-variant", + False, + "first-variant", + False, + False, + False, + False, + False, + "first-variant", + "second-variant", + False, + False, + False, + "first-variant", + "first-variant", + "fifth-variant", + False, + False, + False, + "first-variant", + False, + "third-variant", + False, + False, + "second-variant", + False, + False, + False, + False, + False, + "fourth-variant", + "second-variant", + "first-variant", + "second-variant", + False, + "second-variant", + False, + "second-variant", + False, + "first-variant", + False, + "first-variant", + "first-variant", + False, + "second-variant", + False, + "first-variant", + False, + "fifth-variant", + False, + "first-variant", + "first-variant", + False, + False, + False, + "first-variant", + False, + "first-variant", + "third-variant", + False, + False, + "first-variant", + "first-variant", + False, + False, + "fifth-variant", + False, + False, + "third-variant", + False, + "third-variant", + "first-variant", + "first-variant", + "third-variant", + "third-variant", + False, + "first-variant", + False, + False, + False, + False, + False, + "first-variant", + False, + False, + False, + False, + "second-variant", + "first-variant", + "second-variant", + "first-variant", + False, + "fifth-variant", + "first-variant", + False, + False, + "fourth-variant", + "first-variant", + "first-variant", + False, + False, + "fourth-variant", + "first-variant", + False, + "second-variant", + "third-variant", + "third-variant", + "first-variant", + "first-variant", + False, + False, + False, + "first-variant", + "first-variant", + "first-variant", + False, + "third-variant", + "third-variant", + "third-variant", + False, + False, + "first-variant", + "first-variant", + False, + "second-variant", + False, + False, + "second-variant", + False, + "third-variant", + "first-variant", + "second-variant", + "fifth-variant", + "first-variant", + "first-variant", + False, + "first-variant", + "fifth-variant", + False, + False, + False, + "third-variant", + "first-variant", + "first-variant", + "second-variant", + "fourth-variant", + "first-variant", + "second-variant", + "first-variant", + False, + False, + False, + "second-variant", + "third-variant", + False, + False, + "first-variant", + False, + False, + False, + False, + False, + False, + "first-variant", + "first-variant", + False, + "third-variant", + False, + "first-variant", + False, + "third-variant", + "third-variant", + "first-variant", + "first-variant", + False, + "second-variant", + False, + "second-variant", + "first-variant", + False, + False, + False, + "second-variant", + False, + "third-variant", + False, + "first-variant", + "fifth-variant", + "first-variant", + "first-variant", + False, + False, + "first-variant", + False, + False, + False, + "first-variant", + "fourth-variant", + "first-variant", + "first-variant", + "first-variant", + "fifth-variant", + False, + False, + False, + "second-variant", + False, + False, + False, + "first-variant", + "first-variant", + False, + False, + "first-variant", + "first-variant", + "second-variant", + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "first-variant", + "third-variant", + "first-variant", + False, + "second-variant", + False, + False, + "third-variant", + "second-variant", + "third-variant", + False, + "first-variant", + "third-variant", + "second-variant", + "first-variant", + "third-variant", + False, + False, + "first-variant", + "first-variant", + False, + False, + False, + "first-variant", + "third-variant", + "second-variant", + "first-variant", + "first-variant", + "first-variant", + False, + "third-variant", + "second-variant", + "third-variant", + False, + False, + "third-variant", + "first-variant", + False, + "first-variant", + ] + + for i in range(1000): + distinctID = f"distinct_id_{i}" + + feature_flag_match = FeatureFlagMatcher([feature_flag], distinctID).get_match(feature_flag) + + if results[i]: + self.assertEqual(feature_flag_match, FeatureFlagMatch(variant=cast(str, results[i]))) + else: + self.assertIsNone(feature_flag_match) From 411f47bfe6f3c1de12290dc823d5478ca82b6d75 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Wed, 27 Jul 2022 15:55:42 +0100 Subject: [PATCH 193/213] feat: only query once when caching insights (#11010) * feat: only query once when caching insights * refactor to simplify * why wonder if there are no matches --- posthog/tasks/test/test_update_cache.py | 20 ++--- posthog/tasks/update_cache.py | 98 +++++++++++++------------ 2 files changed, 62 insertions(+), 56 deletions(-) diff --git a/posthog/tasks/test/test_update_cache.py b/posthog/tasks/test/test_update_cache.py index 777afc57639a90..3ca75c968db60a 100644 --- a/posthog/tasks/test/test_update_cache.py +++ b/posthog/tasks/test/test_update_cache.py @@ -363,7 +363,7 @@ def test_update_cache_item_calls_right_funnel_class_clickhouse( CacheType.FUNNEL, {"filter": filter.toJSON(), "team_id": self.team.pk,}, ) - self.assertEqual(funnel_mock.call_count, 2) # once for Insight check, once for dashboard tile check + self.assertEqual(funnel_mock.call_count, 1) # trends funnel filter = base_filter.with_data({"funnel_viz_type": "trends"}) @@ -375,7 +375,7 @@ def test_update_cache_item_calls_right_funnel_class_clickhouse( CacheType.FUNNEL, {"filter": filter.toJSON(), "team_id": self.team.pk,}, ) - self.assertEqual(funnel_trends_mock.call_count, 2) + self.assertEqual(funnel_trends_mock.call_count, 1) # time to convert funnel filter = base_filter.with_data({"funnel_viz_type": "time_to_convert", "funnel_order_type": "strict"}) @@ -387,7 +387,7 @@ def test_update_cache_item_calls_right_funnel_class_clickhouse( CacheType.FUNNEL, {"filter": filter.toJSON(), "team_id": self.team.pk,}, ) - self.assertEqual(funnel_time_to_convert_mock.call_count, 2) + self.assertEqual(funnel_time_to_convert_mock.call_count, 1) # strict funnel filter = base_filter.with_data({"funnel_order_type": "strict"}) @@ -399,7 +399,7 @@ def test_update_cache_item_calls_right_funnel_class_clickhouse( CacheType.FUNNEL, {"filter": filter.toJSON(), "team_id": self.team.pk,}, ) - self.assertEqual(funnel_strict_mock.call_count, 2) + self.assertEqual(funnel_strict_mock.call_count, 1) # unordered funnel filter = base_filter.with_data({"funnel_order_type": "unordered"}) @@ -411,7 +411,7 @@ def test_update_cache_item_calls_right_funnel_class_clickhouse( CacheType.FUNNEL, {"filter": filter.toJSON(), "team_id": self.team.pk,}, ) - self.assertEqual(funnel_unordered_mock.call_count, 2) + self.assertEqual(funnel_unordered_mock.call_count, 1) def _test_refresh_dashboard_cache_types( self, filter: FilterType, cache_type: CacheType, patch_update_cache_item: MagicMock, @@ -496,8 +496,8 @@ def test_stickiness_regression(self, patch_update_cache_item: MagicMock, _patch_ @patch("posthog.tasks.update_cache._calculate_by_filter") def test_errors_refreshing(self, patch_calculate_by_filter: MagicMock) -> None: """ - When there are no filters on the dashboard the tile and insight cache key match - the cache only updates cache counts on the Insight not the dashboard tile + When there are no filters on the dashboard, the tile and insight cache keys match + The cache updates cache counts on both the Insight and the dashboard tile """ with freeze_time("2021-08-25T22:09:14.252Z") as frozen_datetime: dashboard_to_cache = create_shared_dashboard(team=self.team, is_shared=True, last_accessed_at=now()) @@ -520,10 +520,10 @@ def _update_cached_items() -> None: _update_cached_items() self.assertEqual(Insight.objects.get().refresh_attempt, 1) - self.assertEqual(DashboardTile.objects.get().refresh_attempt, None) + self.assertEqual(DashboardTile.objects.get().refresh_attempt, 1) _update_cached_items() self.assertEqual(Insight.objects.get().refresh_attempt, 2) - self.assertEqual(DashboardTile.objects.get().refresh_attempt, None) + self.assertEqual(DashboardTile.objects.get().refresh_attempt, 2) # Magically succeeds, reset counter patch_calculate_by_filter.side_effect = None @@ -543,7 +543,7 @@ def _update_cached_items() -> None: _update_cached_items() _update_cached_items() self.assertEqual(Insight.objects.get().refresh_attempt, 3) - self.assertEqual(DashboardTile.objects.get().refresh_attempt, 0) + self.assertEqual(DashboardTile.objects.get().refresh_attempt, 3) self.assertEqual(patch_calculate_by_filter.call_count, 3) # If a user later comes back and manually refreshes we should reset refresh_attempt diff --git a/posthog/tasks/update_cache.py b/posthog/tasks/update_cache.py index 21fd508c19d7c4..debc39b000b6f9 100644 --- a/posthog/tasks/update_cache.py +++ b/posthog/tasks/update_cache.py @@ -142,40 +142,48 @@ def gauge_cache_update_candidates(dashboard_tiles: QuerySet, shared_insights: Qu @timed("update_cache_item_timer") def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Dict[str, Any]]: + dashboard_id = payload.get("dashboard_id", None) + insight_id = payload.get("insight_id", "unknown") + filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) team = Team.objects.get(pk=team_id) filter = get_filter(data=filter_dict, team=team) - # Doing the filtering like this means we'll update _all_ Insights and DashboardTiles with the same filters hash insights_queryset = Insight.objects.filter(Q(team_id=team_id, filters_hash=key)) + insights_queryset.update(refreshing=True) dashboard_tiles_queryset = DashboardTile.objects.filter(insight__team_id=team_id, filters_hash=key) + dashboard_tiles_queryset.update(refreshing=True) - # at least one must return something, if they both return they will be identical - insight_result = _update_cache_for_queryset(cache_type, filter, key, team, insights_queryset) - tiles_result = _update_cache_for_queryset(cache_type, filter, key, team, dashboard_tiles_queryset) + result = None + try: + if (dashboard_id and dashboard_tiles_queryset.exists()) or insights_queryset.exists(): + result = _update_cache_for_queryset(cache_type, filter, key, team) + except Exception as e: + statsd.incr("update_cache_item_error", tags={"team": team.id}) + _mark_refresh_attempt_for(insights_queryset) + _mark_refresh_attempt_for(dashboard_tiles_queryset) + with push_scope() as scope: + scope.set_tag("cache_key", key) + scope.set_tag("team_id", team.id) + scope.set_tag("insight_id", insight_id) + scope.set_tag("dashboard_id", dashboard_id) + capture_exception(e) + logger.error("update_cache_item_error", exc=e, exc_info=True, team_id=team.id, cache_key=key) + raise e - if tiles_result is not None: - result = tiles_result - elif insight_result is not None: - result = insight_result + if result: + statsd.incr("update_cache_item_success", tags={"team": team.id}) + insights_queryset.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) + dashboard_tiles_queryset.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) else: - dashboard_id = payload.get("dashboard_id", None) - insight_id = payload.get("insight_id", "unknown") + insights_queryset.update(last_refresh=timezone.now(), refreshing=False) + dashboard_tiles_queryset.update(last_refresh=timezone.now(), refreshing=False) statsd.incr( "update_cache_item_no_results", tags={"team": team_id, "cache_key": key, "insight_id": insight_id, "dashboard_id": dashboard_id,}, ) - # there is strong likelihood these querysets match no insights or dashboard tiles - _mark_refresh_attempt_for(insights_queryset) - _mark_refresh_attempt_for(dashboard_tiles_queryset) - # so mark the item that triggered the update - if insight_id != "unknown": - _mark_refresh_attempt_for( - Insight.objects.filter(id=insight_id) - if not dashboard_id - else DashboardTile.objects.filter(insight_id=insight_id, dashboard_id=dashboard_id) - ) + _mark_refresh_attempt_when_no_results(dashboard_id, dashboard_tiles_queryset, insight_id, insights_queryset) result = [] logger.info( @@ -189,36 +197,34 @@ def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Di return result +def _mark_refresh_attempt_when_no_results( + dashboard_id: Optional[int], + dashboard_tiles_queryset: QuerySet, + insight_id: Union[int, str], + insights_queryset: QuerySet, +) -> None: + if insights_queryset.exists() or dashboard_tiles_queryset.exists(): + _mark_refresh_attempt_for(insights_queryset) + _mark_refresh_attempt_for(dashboard_tiles_queryset) + else: + if insight_id != "unknown": + _mark_refresh_attempt_for( + Insight.objects.filter(id=insight_id) + if not dashboard_id + else DashboardTile.objects.filter(insight_id=insight_id, dashboard_id=dashboard_id) + ) + + def _update_cache_for_queryset( - cache_type: CacheType, filter: Filter, key: str, team: Team, queryset: QuerySet + cache_type: CacheType, filter: Filter, key: str, team: Team ) -> Optional[List[Dict[str, Any]]]: - if not queryset.exists(): - return None - - queryset.update(refreshing=True) - try: - if cache_type == CacheType.FUNNEL: - result = _calculate_funnel(filter, key, team) - else: - result = _calculate_by_filter(filter, key, team, cache_type) - cache.set( - key, {"result": result, "type": cache_type, "last_refresh": timezone.now()}, settings.CACHED_RESULTS_TTL - ) - except Exception as e: - statsd.incr("update_cache_item_error", tags={"team": team.id}) - _mark_refresh_attempt_for(queryset) - with push_scope() as scope: - scope.set_tag("cache_key", key) - scope.set_tag("team_id", team.id) - capture_exception(e) - logger.error("update_cache_item_error", exc=e, exc_info=True, team_id=team.id, cache_key=key) - raise e - if result: - statsd.incr("update_cache_item_success", tags={"team": team.id}) - queryset.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) + if cache_type == CacheType.FUNNEL: + result = _calculate_funnel(filter, key, team) else: - queryset.update(last_refresh=timezone.now(), refreshing=False) + result = _calculate_by_filter(filter, key, team, cache_type) + + cache.set(key, {"result": result, "type": cache_type, "last_refresh": timezone.now()}, settings.CACHED_RESULTS_TTL) return result From 234b8d7759008b10d0b7b27a0c8d1c43fafb5208 Mon Sep 17 00:00:00 2001 From: Ben White Date: Wed, 27 Jul 2022 18:35:32 +0200 Subject: [PATCH 194/213] fix: Removed CSV button exports for dashboard (#11012) --- frontend/src/scenes/dashboard/DashboardHeader.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/frontend/src/scenes/dashboard/DashboardHeader.tsx b/frontend/src/scenes/dashboard/DashboardHeader.tsx index 9913e88cefc736..427cdff4791a03 100644 --- a/frontend/src/scenes/dashboard/DashboardHeader.tsx +++ b/frontend/src/scenes/dashboard/DashboardHeader.tsx @@ -189,12 +189,6 @@ export function DashboardHeader(): JSX.Element | null { path: apiUrl(), }, }, - { - export_format: ExporterFormat.CSV, - export_context: { - path: apiUrl(), - }, - }, ]} /> From fd0318802143e7caf5a4dfc4ef9aab950bdde862 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Wed, 27 Jul 2022 18:44:31 +0100 Subject: [PATCH 195/213] chore(feature-flags): Release experience continuity (#11013) --- .../src/scenes/feature-flags/FeatureFlag.tsx | 71 +++++++++---------- 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/frontend/src/scenes/feature-flags/FeatureFlag.tsx b/frontend/src/scenes/feature-flags/FeatureFlag.tsx index 3e2db9240b31c2..03eecbc1b4d106 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlag.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlag.tsx @@ -26,8 +26,6 @@ import { VerticalForm } from 'lib/forms/VerticalForm' import { LemonTextArea } from 'lib/components/LemonTextArea/LemonTextArea' import { LemonInput } from 'lib/components/LemonInput/LemonInput' import { LemonCheckbox } from 'lib/components/LemonCheckbox' -import { FEATURE_FLAGS } from 'lib/constants' -import { featureFlagLogic as enabledFeatureFlagsToCheckLogic } from 'lib/logic/featureFlagLogic' import { EventBufferNotice } from 'scenes/events/EventBufferNotice' import { AlertMessage } from 'lib/components/AlertMessage' import { urls } from 'scenes/urls' @@ -84,8 +82,6 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element { // :KLUDGE: Match by select only allows Select.Option as children, so render groups option directly rather than as a child const matchByGroupsIntroductionOption = GroupsIntroductionOption({ value: -2 }) - const { featureFlags: enabledFeatureFlagsToCheck } = useValues(enabledFeatureFlagsToCheckLogic) - return (
{featureFlag ? ( @@ -203,42 +199,39 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element { placeholder="Adding a helpful description can ensure others know what this feature is for." /> - - {enabledFeatureFlagsToCheck[FEATURE_FLAGS.FEATURE_FLAG_EXPERIENCE_CONTINUITY] && ( - - {({ value, onChange }) => ( -
- - Persist flag across authentication steps{' '} - Beta -
- } - onChange={() => onChange(!value)} - rowProps={{ fullWidth: true }} - checked={value} - /> -
- If your feature flag is applied prior to an identify or authentication - event, use this to ensure that feature flags are not reset after a - person is identified. This ensures the experience for the anonymous - person is carried forward to the authenticated person. Currently - supported for posthog-js only. -
+ + {({ value, onChange }) => ( +
+ + Persist flag across authentication steps{' '} + Beta +
+ } + onChange={() => onChange(!value)} + rowProps={{ fullWidth: true }} + checked={value} + /> +
+ If your feature flag is applied prior to an identify or authentication + event, use this to ensure that feature flags are not reset after a person is + identified. This ensures the experience for the anonymous person is carried + forward to the authenticated person. Currently supported for posthog-js + only.
- )} - - )} +
+ )} + From a70b4b28c6f0d9b4129e08da984e50523939ba77 Mon Sep 17 00:00:00 2001 From: Harry Waye Date: Wed, 27 Jul 2022 20:37:44 +0100 Subject: [PATCH 196/213] chore(web): add django-prometheus exposed on /_metrics (#11000) * chore(web): add django-prometheus exposed on /_metrics This exposes a number of metrics, see https://github.com/korfuri/django-prometheus/blob/97d5748664afaa0e9bb6e62458428b826c79ccf9/documentation/exports.md for details. It includes histogram of timings by viewname before and after middleware. I'm not particularly interested in these right now, but rather would like to expose Kafka Producer metrics as per https://github.com/PostHog/posthog/pull/10997 * Refactor to use gunicorn server hooks * also add expose to dockerfile * wip --- .devcontainer/Dockerfile | 2 +- bin/docker-server | 9 ++++++++- gunicorn.config.py | 24 ++++++++++++++++++++++++ posthog/settings/web.py | 4 +++- posthog/urls.py | 8 ++++++++ production.Dockerfile | 4 ++++ requirements.in | 1 + requirements.txt | 6 +++++- 8 files changed, 54 insertions(+), 4 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 43c601b5471d5c..74e755916edead 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,7 +7,7 @@ # experience as rich as possible. Perhaps later down the line it might be worth # rolling our own # -FROM mcr.microsoft.com/vscode/devcontainers/python:3.9-bullseye +FROM mcr.microsoft.com/vscode/devcontainers/python:3.8-bullseye # Make sure all exit codes on pipes cause failures SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/bin/docker-server b/bin/docker-server index 70cf3a1ec88af7..4c71deb1422ece 100755 --- a/bin/docker-server +++ b/bin/docker-server @@ -3,6 +3,13 @@ set -e ./bin/migrate-check +# To ensure we are able to expose metrics from multiple processes, we need to +# provide a directory for `prometheus_client` to store a shared registry. +export PROMETHEUS_MULTIPROC_DIR=$(mktemp -d) +trap 'rm -rf "$PROMETHEUS_MULTIPROC_DIR"' EXIT + +export PROMETHEUS_METRICS_EXPORT_PORT=8001 + gunicorn posthog.wsgi \ --config gunicorn.config.py \ --bind 0.0.0.0:8000 \ @@ -13,4 +20,4 @@ gunicorn posthog.wsgi \ --workers=2 \ --threads=4 \ --worker-class=gthread \ - --limit-request-line=8190 + --limit-request-line=8190 $@ diff --git a/gunicorn.config.py b/gunicorn.config.py index dde98015a2011b..86d2c3238d7a16 100644 --- a/gunicorn.config.py +++ b/gunicorn.config.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os + +from prometheus_client import CollectorRegistry, multiprocess, start_http_server loglevel = "error" keepalive = 120 @@ -29,3 +32,24 @@ def on_starting(server): print("Server running on \x1b[4mhttp://{}:{}\x1b[0m".format(*server.address[0])) print("Questions? Please shoot us an email at \x1b[4mhey@posthog.com\x1b[0m") print("\nTo stop, press CTRL + C") + + +def when_ready(server): + """ + To ease being able to hide the /metrics endpoint when running in production, + we serve the metrics on a separate port, using the + prometheus_client.multiprocess Collector to pull in data from the worker + processes. + """ + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + port = int(os.environ.get("PROMETHEUS_METRICS_EXPORT_PORT", 8001)) + start_http_server(port=port, registry=registry) + + +def worker_exit(server, worker): + """ + Ensure that we mark workers as dead with the prometheus_client such that + any cleanup can happen. + """ + multiprocess.mark_process_dead(worker.pid) diff --git a/posthog/settings/web.py b/posthog/settings/web.py index 2a8cae6f2f2813..af1a62a1692544 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -43,15 +43,16 @@ ] MIDDLEWARE = [ + "django_prometheus.middleware.PrometheusBeforeMiddleware", "posthog.gzip_middleware.ScopedGZipMiddleware", "django_structlog.middlewares.RequestMiddleware", "django_structlog.middlewares.CeleryMiddleware", "django.middleware.security.SecurityMiddleware", "posthog.middleware.ShortCircuitMiddleware", + "posthog.middleware.AllowIPMiddleware", # NOTE: we need healthcheck high up to avoid hitting middlewares that may be # using dependencies that the healthcheck should be checking. It should be # ok below the above middlewares however. - "posthog.middleware.AllowIPMiddleware", "posthog.health.healthcheck_middleware", "google.cloud.sqlcommenter.django.middleware.SqlCommenter", "django.contrib.sessions.middleware.SessionMiddleware", @@ -66,6 +67,7 @@ "axes.middleware.AxesMiddleware", "posthog.middleware.AutoProjectMiddleware", "posthog.middleware.CHQueries", + "django_prometheus.middleware.PrometheusAfterMiddleware", ] if STATSD_HOST is not None: diff --git a/posthog/urls.py b/posthog/urls.py index 522726436e50d3..65e4ed2f6370fb 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -6,6 +6,7 @@ from django.urls import URLPattern, include, path, re_path from django.views.decorators import csrf from django.views.decorators.csrf import csrf_exempt +from django_prometheus.exports import ExportToDjangoView from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView from posthog.api import ( @@ -142,6 +143,13 @@ def opt_slash_path(route: str, view: Callable, name: Optional[str] = None) -> UR path("", include("social_django.urls", namespace="social")), ] +if settings.DEBUG: + # If we have DEBUG=1 set, then let's expose the metrics for debugging. Note + # that in production we expose these metrics on a separate port, to ensure + # external clients cannot see them. See the gunicorn setup for details on + # what we do. + urlpatterns.append(path("_metrics", ExportToDjangoView)) + if settings.TEST: # Used in posthog-js e2e tests diff --git a/production.Dockerfile b/production.Dockerfile index b88112ba197dc2..e760a775b9c58c 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -175,4 +175,8 @@ COPY gunicorn.config.py ./ # Expose container port and run entry point script EXPOSE 8000 + +# Expose the port from which we serve OpenMetrics data +EXPOSE 8001 + CMD ["./bin/docker"] diff --git a/requirements.in b/requirements.in index 0425bdd4bf1aa9..709dfc30397c10 100644 --- a/requirements.in +++ b/requirements.in @@ -22,6 +22,7 @@ django-extensions==3.1.2 django-filter==2.4.0 django-loginas==0.3.9 django-picklefield==3.0.1 +django-prometheus==2.2.0 django-redis==4.12.1 django-statsd==2.5.2 django-structlog==2.1.3 diff --git a/requirements.txt b/requirements.txt index bf23f98740e285..11bc3a0e496e6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -111,6 +111,8 @@ django-loginas==0.3.9 # via -r requirements.in django-picklefield==3.0.1 # via -r requirements.in +django-prometheus==2.2.0 + # via -r requirements.in django-redis==4.12.1 # via -r requirements.in django-rest-hooks @ git+https://github.com/zapier/django-rest-hooks.git@v1.6.0 @@ -162,7 +164,7 @@ idna==2.8 # yarl importlib-metadata==1.6.0 # via -r requirements.in -importlib-resources==5.8.0 +importlib-resources==5.9.0 # via jsonschema infi-clickhouse-orm @ git+https://github.com/PostHog/infi.clickhouse_orm@37722f350f3b449bbcd6564917c436b0d93e796f # via -r requirements.in @@ -229,6 +231,8 @@ pickleshare==0.7.5 # via -r requirements.in posthoganalytics==1.4.9 # via -r requirements.in +prometheus-client==0.14.1 + # via django-prometheus protobuf==3.13.0 # via -r requirements.in psycopg2-binary==2.8.6 From 7c59a4a6fa38e69179b5daf55960c0cc61bf30fa Mon Sep 17 00:00:00 2001 From: Rick Marron Date: Wed, 27 Jul 2022 12:39:40 -0700 Subject: [PATCH 197/213] feat(session-analysis): remove feature flag (#11015) * feat(session-analysis): remove feature flag * duration filters only on trends --- frontend/src/lib/constants.tsx | 1 - .../EditorFilters/TrendsGlobalAndOrFilters.tsx | 7 ++----- .../scenes/insights/EditorFilters/TrendsSteps.tsx | 7 ++----- .../ActionFilter/ActionFilterRow/ActionFilterRow.tsx | 12 ++++-------- .../BreakdownFilter/TaxonomicBreakdownFilter.tsx | 12 ++---------- .../taxonomicBreakdownFilterUtils.test.ts | 7 ------- .../BreakdownFilter/taxonomicBreakdownFilterUtils.ts | 12 ++---------- frontend/src/scenes/trends/mathsLogic.tsx | 12 ++++-------- 8 files changed, 16 insertions(+), 54 deletions(-) diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 3bca5d87ce0563..f08c3cbf831193 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -112,7 +112,6 @@ export const FEATURE_FLAGS = { ONBOARDING_1_5: 'onboarding-1_5', // owner: @liyiy BREAKDOWN_ATTRIBUTION: 'breakdown-attribution', // owner: @neilkakkar SIMPLIFY_ACTIONS: 'simplify-actions', // owner: @alexkim205, - SESSION_ANALYSIS: 'session-analysis', // owner: @rcmarron TOOLBAR_LAUNCH_SIDE_ACTION: 'toolbar-launch-side-action', // owner: @pauldambra, FEATURE_FLAG_EXPERIENCE_CONTINUITY: 'feature-flag-experience-continuity', // owner: @neilkakkar // Re-enable person modal CSV downloads when frontend can support new entity properties diff --git a/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx b/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx index f2313eab351b33..56e50b8011a2b8 100644 --- a/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx +++ b/frontend/src/scenes/insights/EditorFilters/TrendsGlobalAndOrFilters.tsx @@ -2,20 +2,17 @@ import React from 'react' import { convertPropertiesToPropertyGroup } from 'lib/utils' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { PropertyGroupFilters } from 'lib/components/PropertyGroupFilters/PropertyGroupFilters' -import { EditorFilterProps } from '~/types' +import { EditorFilterProps, InsightType } from '~/types' import { useActions, useValues } from 'kea' import { trendsLogic } from 'scenes/trends/trendsLogic' import { groupsModel } from '~/models/groupsModel' import { insightLogic } from 'scenes/insights/insightLogic' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import { FEATURE_FLAGS } from 'lib/constants' export function TrendsGlobalAndOrFilters({ filters, insightProps }: EditorFilterProps): JSX.Element { const { setFilters } = useActions(trendsLogic(insightProps)) const { allEventNames } = useValues(insightLogic) const { groupsTaxonomicTypes } = useValues(groupsModel) - const { featureFlags } = useValues(featureFlagLogic) const taxonomicGroupTypes = [ TaxonomicFilterGroupType.EventProperties, @@ -23,7 +20,7 @@ export function TrendsGlobalAndOrFilters({ filters, insightProps }: EditorFilter ...groupsTaxonomicTypes, TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.Elements, - ...(featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS] ? [TaxonomicFilterGroupType.Sessions] : []), + ...(filters.insight === InsightType.TRENDS ? [TaxonomicFilterGroupType.Sessions] : []), ] return ( diff --git a/frontend/src/scenes/insights/EditorFilters/TrendsSteps.tsx b/frontend/src/scenes/insights/EditorFilters/TrendsSteps.tsx index d1efeaee71eaaf..cfaa25911edd31 100644 --- a/frontend/src/scenes/insights/EditorFilters/TrendsSteps.tsx +++ b/frontend/src/scenes/insights/EditorFilters/TrendsSteps.tsx @@ -7,14 +7,11 @@ import { alphabet } from 'lib/utils' import { MathAvailability } from 'scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import React from 'react' -import { FEATURE_FLAGS } from 'lib/constants' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' export function TrendsSteps({ insightProps }: EditorFilterProps): JSX.Element { const { setFilters } = useActions(trendsLogic(insightProps)) const { filters } = useValues(trendsLogic(insightProps)) const { groupsTaxonomicTypes } = useValues(groupsModel) - const { featureFlags } = useValues(featureFlagLogic) const propertiesTaxonomicGroupTypes = [ TaxonomicFilterGroupType.EventProperties, @@ -22,8 +19,8 @@ export function TrendsSteps({ insightProps }: EditorFilterProps): JSX.Element { ...groupsTaxonomicTypes, TaxonomicFilterGroupType.Cohorts, TaxonomicFilterGroupType.Elements, - ].concat(featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS] ? [TaxonomicFilterGroupType.Sessions] : []) - + ...(filters.insight === InsightType.TRENDS ? [TaxonomicFilterGroupType.Sessions] : []), + ] return ( <> {filters.insight === InsightType.LIFECYCLE && ( diff --git a/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx b/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx index a09b9a0e33d520..4c7715990c5ccc 100644 --- a/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx +++ b/frontend/src/scenes/insights/filters/ActionFilter/ActionFilterRow/ActionFilterRow.tsx @@ -30,8 +30,6 @@ import { IconCopy, IconDelete, IconEdit, IconFilter, IconWithCount } from 'lib/c import { SortableHandle as sortableHandle } from 'react-sortable-hoc' import { SortableDragIcon } from 'lib/components/icons' import { LemonButton } from 'lib/components/LemonButton' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import { FEATURE_FLAGS } from 'lib/constants' const DragHandle = sortableHandle(() => ( @@ -123,7 +121,6 @@ export function ActionFilterRow({ } = useActions(logic) const { actions } = useValues(actionsModel) const { mathDefinitions } = useValues(mathsLogic) - const { featureFlags } = useValues(featureFlagLogic) const propertyFiltersVisible = typeof filter.order === 'number' ? entityFilterVisible[filter.order] : false @@ -320,11 +317,10 @@ export function ActionFilterRow({
onMathPropertySelect(index, currentValue)} eventNames={name ? [name] : []} diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx index bcecfc96be76e8..ac7198d2ee113b 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownFilter.tsx @@ -1,8 +1,6 @@ import { useValues } from 'kea' import { propertyFilterTypeToTaxonomicFilterType } from 'lib/components/PropertyFilters/utils' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' -import { FEATURE_FLAGS } from 'lib/constants' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import React from 'react' import { TaxonomicBreakdownButton } from 'scenes/insights/filters/BreakdownFilter/TaxonomicBreakdownButton' import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' @@ -32,7 +30,6 @@ export function BreakdownFilter({ }: TaxonomicBreakdownFilterProps): JSX.Element { const { breakdown, breakdowns, breakdown_type } = filters const { getPropertyDefinition } = useValues(propertyDefinitionsModel) - const { featureFlags } = useValues(featureFlagLogic) let breakdownType = propertyFilterTypeToTaxonomicFilterType(breakdown_type) if (breakdownType === TaxonomicFilterGroupType.Cohorts) { @@ -86,9 +83,7 @@ export function BreakdownFilter({ ? [] : breakdownArray.map((t, index) => { const key = `${t}-${index}` - const isPropertyHistogramable = featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS] - ? !useMultiBreakdown && !!getPropertyDefinition(t)?.is_numerical - : false + const isPropertyHistogramable = !useMultiBreakdown && !!getPropertyDefinition(t)?.is_numerical return ( ) : null}
diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts index 4c6ea47bb3faa2..a5e71f36b3d300 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.test.ts @@ -26,7 +26,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['a', 'b'], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = 'c' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.EventProperties) @@ -51,7 +50,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['all', 1], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = 2 const group: TaxonomicFilterGroup = taxonomicGroupFor( @@ -79,7 +77,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['country'], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = 'height' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.PersonProperties, undefined) @@ -107,7 +104,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['a', 'b'], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = 'c' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.EventProperties, undefined) @@ -127,7 +123,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['all', 1], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = 2 const group: TaxonomicFilterGroup = taxonomicGroupFor( @@ -149,7 +144,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['country'], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = 'height' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.PersonProperties, undefined) @@ -168,7 +162,6 @@ describe('taxonomic breakdown filter utils', () => { breakdownParts: ['$lib'], setFilters, getPropertyDefinition, - histogramFeatureFlag: false, }) const changedBreakdown = '$lib_version' const group: TaxonomicFilterGroup = taxonomicGroupFor(TaxonomicFilterGroupType.GroupsPrefix, 0) diff --git a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts index f4c34dc42ea15c..a0bb1a826b3879 100644 --- a/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts +++ b/frontend/src/scenes/insights/filters/BreakdownFilter/taxonomicBreakdownFilterUtils.ts @@ -11,22 +11,14 @@ interface FilterChange { breakdownParts: (string | number)[] setFilters: (filters: Partial, mergeFilters?: boolean) => void getPropertyDefinition: (propertyName: string | number) => PropertyDefinition | null - histogramFeatureFlag: boolean } -export function onFilterChange({ - useMultiBreakdown, - breakdownParts, - setFilters, - getPropertyDefinition, - histogramFeatureFlag, -}: FilterChange) { +export function onFilterChange({ useMultiBreakdown, breakdownParts, setFilters, getPropertyDefinition }: FilterChange) { return (changedBreakdown: TaxonomicFilterValue, taxonomicGroup: TaxonomicFilterGroup): void => { const changedBreakdownType = taxonomicFilterTypeToPropertyFilterType(taxonomicGroup.type) as BreakdownType if (changedBreakdownType) { - const isHistogramable = - histogramFeatureFlag && !useMultiBreakdown && !!getPropertyDefinition(changedBreakdown)?.is_numerical + const isHistogramable = !useMultiBreakdown && !!getPropertyDefinition(changedBreakdown)?.is_numerical const newFilters: Partial = { breakdown_type: changedBreakdownType, diff --git a/frontend/src/scenes/trends/mathsLogic.tsx b/frontend/src/scenes/trends/mathsLogic.tsx index ed04a31b1525dc..f0c66dfb229272 100644 --- a/frontend/src/scenes/trends/mathsLogic.tsx +++ b/frontend/src/scenes/trends/mathsLogic.tsx @@ -2,9 +2,8 @@ import React from 'react' import { kea } from 'kea' import { groupsModel } from '~/models/groupsModel' import type { mathsLogicType } from './mathsLogicType' -import { EVENT_MATH_TYPE, FEATURE_FLAGS, PROPERTY_MATH_TYPE } from 'lib/constants' +import { EVENT_MATH_TYPE, PROPERTY_MATH_TYPE } from 'lib/constants' import { BaseMathType, PropertyMathType } from '~/types' -import { featureFlagLogic } from 'lib/logic/featureFlagLogic' export interface MathDefinition { name: string @@ -243,7 +242,7 @@ export function apiValueToMathType(math: string | undefined, groupTypeIndex: num export const mathsLogic = kea({ path: ['scenes', 'trends', 'mathsLogic'], connect: { - values: [groupsModel, ['groupTypes', 'aggregationLabel'], featureFlagLogic, ['featureFlags']], + values: [groupsModel, ['groupTypes', 'aggregationLabel']], }, selectors: { eventMathEntries: [ @@ -255,16 +254,13 @@ export const mathsLogic = kea({ (mathDefinitions) => Object.entries(mathDefinitions).filter(([, item]) => item.type == PROPERTY_MATH_TYPE), ], mathDefinitions: [ - (s) => [s.groupsMathDefinitions, s.featureFlags], - (groupOptions, featureFlags): Record => { + (s) => [s.groupsMathDefinitions], + (groupOptions): Record => { const allMathOptions: Record = { ...BASE_MATH_DEFINITIONS, ...groupOptions, ...PROPERTY_MATH_DEFINITIONS, } - if (!featureFlags[FEATURE_FLAGS.SESSION_ANALYSIS]) { - delete allMathOptions[BaseMathType.UniqueSessions] - } return allMathOptions }, ], From a7bb217c3326f992a8e7b88fe24c819cc427b348 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 28 Jul 2022 10:02:35 +0300 Subject: [PATCH 198/213] chore(plugin-server): track plugin/vm setup times (#11008) * chore(plugin-server): Gather stats on plugin setup We're seeing that async servers are having trouble with health checks. One area I'd like to exclude here is that they're overwhelmed with booting plugins. This change allows us to track how plugin setup is doing: is everything succeeding, how long is it taking, etc. Then if it turns out we're blocked on some slow plugins we can action those. * Track setup plugins timing * Track loadSchedule --- plugin-server/src/worker/plugins/setup.ts | 5 +++++ plugin-server/src/worker/vm/lazy.ts | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/plugin-server/src/worker/plugins/setup.ts b/plugin-server/src/worker/plugins/setup.ts index 0c898f7b450c9a..35eb64fe25ec88 100644 --- a/plugin-server/src/worker/plugins/setup.ts +++ b/plugin-server/src/worker/plugins/setup.ts @@ -12,6 +12,8 @@ export async function setupPlugins(server: Hub): Promise { const pluginVMLoadPromises: Array> = [] const statelessVms = {} as StatelessVmMap + const timer = new Date() + for (const [id, pluginConfig] of pluginConfigs) { const plugin = plugins.get(pluginConfig.plugin_id) const prevConfig = server.pluginConfigs.get(id) @@ -40,6 +42,7 @@ export async function setupPlugins(server: Hub): Promise { } await Promise.all(pluginVMLoadPromises) + server.statsd?.timing('setup_plugins.success', timer) server.plugins = plugins server.pluginConfigs = pluginConfigs @@ -118,6 +121,7 @@ async function loadPluginsFromDB(hub: Hub): Promise { + const timer = new Date() server.pluginSchedule = null // gather runEvery* tasks into a schedule @@ -140,4 +144,5 @@ export async function loadSchedule(server: Hub): Promise { } server.pluginSchedule = pluginSchedule + server.statsd?.timing('load_schedule.success', timer) } diff --git a/plugin-server/src/worker/vm/lazy.ts b/plugin-server/src/worker/vm/lazy.ts index 49969905c48f09..2346357da1d96e 100644 --- a/plugin-server/src/worker/vm/lazy.ts +++ b/plugin-server/src/worker/vm/lazy.ts @@ -195,13 +195,20 @@ export class LazyPluginVM { ? pluginDigest(this.pluginConfig.plugin) : `plugin config ID '${this.pluginConfig.id}'` this.totalInitAttemptsCounter++ + const timer = new Date() try { await vm?.run(`${this.vmResponseVariable}.methods.setupPlugin?.()`) + this.hub.statsd?.increment('plugin.setup.success', { plugin: this.pluginConfig.plugin?.name ?? '?' }) + this.hub.statsd?.timing('plugin.setup.timing', timer, { plugin: this.pluginConfig.plugin?.name ?? '?' }) this.ready = true status.info('🔌', `setupPlugin succeeded for ${logInfo}.`) await this.createLogEntry(`setupPlugin succeeded (instance ID ${this.hub.instanceId}).`) void clearError(this.hub, this.pluginConfig) } catch (error) { + this.hub.statsd?.increment('plugin.setup.fail', { plugin: this.pluginConfig.plugin?.name ?? '?' }) + this.hub.statsd?.timing('plugin.setup.fail_timing', timer, { + plugin: this.pluginConfig.plugin?.name ?? '?', + }) this.clearRetryTimeoutIfExists() if (error instanceof RetryError) { error._attempt = this.totalInitAttemptsCounter From 6bbc409b500b67efbc1d26e5bb24fd244e1520cf Mon Sep 17 00:00:00 2001 From: Ben White Date: Thu, 28 Jul 2022 09:56:34 +0200 Subject: [PATCH 199/213] fix: Exclude liveness probes from IP blocks (#11023) --- posthog/middleware.py | 11 +---------- posthog/settings/web.py | 4 ++-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/posthog/middleware.py b/posthog/middleware.py index 2db74877585a10..3fee79fb4f9de9 100644 --- a/posthog/middleware.py +++ b/posthog/middleware.py @@ -54,16 +54,7 @@ def extract_client_ip(self, request: HttpRequest): def __call__(self, request: HttpRequest): response: HttpResponse = self.get_response(request) - if request.path.split("/")[1] in [ - "decide", - "engage", - "track", - "capture", - "batch", - "e", - "static", - "_health", - ]: + if request.path.split("/")[1] in ["decide", "engage", "track", "capture", "batch", "e", "static", "_health"]: return response ip = self.extract_client_ip(request) if ip and any(ip_address(ip) in ip_network(block, strict=False) for block in self.ip_blocks): diff --git a/posthog/settings/web.py b/posthog/settings/web.py index af1a62a1692544..41050e3d13c7c8 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -48,12 +48,12 @@ "django_structlog.middlewares.RequestMiddleware", "django_structlog.middlewares.CeleryMiddleware", "django.middleware.security.SecurityMiddleware", - "posthog.middleware.ShortCircuitMiddleware", - "posthog.middleware.AllowIPMiddleware", # NOTE: we need healthcheck high up to avoid hitting middlewares that may be # using dependencies that the healthcheck should be checking. It should be # ok below the above middlewares however. "posthog.health.healthcheck_middleware", + "posthog.middleware.ShortCircuitMiddleware", + "posthog.middleware.AllowIPMiddleware", "google.cloud.sqlcommenter.django.middleware.SqlCommenter", "django.contrib.sessions.middleware.SessionMiddleware", "corsheaders.middleware.CorsMiddleware", From 8abb467bab13a955f292a1647b4180669cce25c2 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 28 Jul 2022 11:26:16 +0300 Subject: [PATCH 200/213] chore(plugin-server): tweak kafka.send capture logic (#11024) * Swallow errors for flush loop * Log messages with flush * Capture PLUGIN_SERVER_MODE --- plugin-server/src/init.ts | 5 +++++ plugin-server/src/utils/db/kafka-producer-wrapper.ts | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/plugin-server/src/init.ts b/plugin-server/src/init.ts index 93f193432f7166..96c32807a3e973 100644 --- a/plugin-server/src/init.ts +++ b/plugin-server/src/init.ts @@ -14,6 +14,11 @@ export function initApp(config: PluginsServerConfig): void { Sentry.init({ dsn: config.SENTRY_DSN, normalizeDepth: 8, // Default: 3 + initialScope: { + tags: { + PLUGIN_SERVER_MODE: config.PLUGIN_SERVER_MODE, + }, + }, }) } } diff --git a/plugin-server/src/utils/db/kafka-producer-wrapper.ts b/plugin-server/src/utils/db/kafka-producer-wrapper.ts index 0a1ff5d8fa0809..714d4c1279b1d8 100644 --- a/plugin-server/src/utils/db/kafka-producer-wrapper.ts +++ b/plugin-server/src/utils/db/kafka-producer-wrapper.ts @@ -44,7 +44,12 @@ export class KafkaProducerWrapper { this.maxQueueSize = serverConfig.KAFKA_PRODUCER_MAX_QUEUE_SIZE this.maxBatchSize = serverConfig.KAFKA_MAX_MESSAGE_BATCH_SIZE - this.flushInterval = setInterval(() => this.flush(), this.flushFrequencyMs) + this.flushInterval = setInterval(async () => { + // :TRICKY: Swallow uncaught errors from flush as flush is already doing custom error reporting which would get lost. + try { + await this.flush() + } catch (err) {} + }, this.flushFrequencyMs) } async queueMessage(kafkaMessage: ProducerRecord): Promise { @@ -103,6 +108,7 @@ export class KafkaProducerWrapper { } catch (err) { Sentry.captureException(err, { extra: { + messages: messages, batchCount: messages.length, topics: messages.map((record) => record.topic), messageCounts: messages.map((record) => record.messages.length), From 156fa2353fe4bbb1e3e93366afc464c1877f7e09 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 28 Jul 2022 11:58:33 +0300 Subject: [PATCH 201/213] feat(plugin-server): Use Snappy compression codec for kafka production (#10974) * feat(plugin-server): Use Snappy compression codec for kafka production This helps avoid 'message too large' type errors (see https://github.com/PostHog/posthog/pull/10968) by compressing in-flight messages. I would have preferred to use zstd, but the libraries did not compile cleanly on my machine. * Update tests --- plugin-server/package.json | 1 + plugin-server/src/utils/db/kafka-producer-wrapper.ts | 7 ++++++- .../tests/main/kafka-producer-wrapper.test.ts | 7 ++++++- plugin-server/yarn.lock | 12 ++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/plugin-server/package.json b/plugin-server/package.json index c9df29263c57fa..ce917591c1778b 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -60,6 +60,7 @@ "ioredis": "^4.27.6", "jsonwebtoken": "^8.5.1", "kafkajs": "^2.0.2", + "kafkajs-snappy": "^1.1.0", "lru-cache": "^6.0.0", "luxon": "^1.27.0", "node-fetch": "^2.6.1", diff --git a/plugin-server/src/utils/db/kafka-producer-wrapper.ts b/plugin-server/src/utils/db/kafka-producer-wrapper.ts index 714d4c1279b1d8..4a40a14a0c9f07 100644 --- a/plugin-server/src/utils/db/kafka-producer-wrapper.ts +++ b/plugin-server/src/utils/db/kafka-producer-wrapper.ts @@ -1,11 +1,15 @@ import * as Sentry from '@sentry/node' import { StatsD } from 'hot-shots' -import { Message, Producer, ProducerRecord } from 'kafkajs' +import { CompressionCodecs, CompressionTypes, Message, Producer, ProducerRecord } from 'kafkajs' +// @ts-expect-error no type definitions +import SnappyCodec from 'kafkajs-snappy' import { PluginsServerConfig } from '../../types' import { instrumentQuery } from '../metrics' import { timeoutGuard } from './utils' +CompressionCodecs[CompressionTypes.Snappy] = SnappyCodec + /** This class wraps kafkajs producer, adding batching to optimize performance. * * As messages get queued, we flush the queue in the following cases. @@ -104,6 +108,7 @@ export class KafkaProducerWrapper { try { await this.producer.sendBatch({ topicMessages: messages, + compression: CompressionTypes.Snappy, }) } catch (err) { Sentry.captureException(err, { diff --git a/plugin-server/tests/main/kafka-producer-wrapper.test.ts b/plugin-server/tests/main/kafka-producer-wrapper.test.ts index 1a06f5974ab22b..9c6d606b864e07 100644 --- a/plugin-server/tests/main/kafka-producer-wrapper.test.ts +++ b/plugin-server/tests/main/kafka-producer-wrapper.test.ts @@ -1,4 +1,4 @@ -import { Producer } from 'kafkajs' +import { CompressionTypes, Producer } from 'kafkajs' import { PluginsServerConfig } from '../../src/types' import { KafkaProducerWrapper } from '../../src/utils/db/kafka-producer-wrapper' @@ -49,6 +49,7 @@ describe('KafkaProducerWrapper', () => { expect(producer.currentBatch.length).toEqual(0) expect(producer.currentBatchSize).toEqual(0) expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ + compression: CompressionTypes.Snappy, topicMessages: [expect.anything(), expect.anything(), expect.anything(), expect.anything()], }) }) @@ -76,6 +77,7 @@ describe('KafkaProducerWrapper', () => { expect(producer.currentBatchSize).toBeGreaterThan(40) expect(producer.currentBatchSize).toBeLessThan(100) expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ + compression: CompressionTypes.Snappy, topicMessages: [expect.anything(), expect.anything()], }) }) @@ -91,6 +93,7 @@ describe('KafkaProducerWrapper', () => { expect(producer.currentBatch.length).toEqual(0) expect(producer.currentBatchSize).toEqual(0) expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ + compression: CompressionTypes.Snappy, topicMessages: [expect.anything()], }) }) @@ -121,6 +124,7 @@ describe('KafkaProducerWrapper', () => { expect(producer.currentBatch.length).toEqual(0) expect(producer.lastFlushTime).toEqual(Date.now()) expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ + compression: CompressionTypes.Snappy, topicMessages: [expect.anything(), expect.anything(), expect.anything()], }) }) @@ -138,6 +142,7 @@ describe('KafkaProducerWrapper', () => { await producer.flush() expect(mockKafkaProducer.sendBatch).toHaveBeenCalledWith({ + compression: CompressionTypes.Snappy, topicMessages: [ { topic: 'a', diff --git a/plugin-server/yarn.lock b/plugin-server/yarn.lock index 050d47d7c01edc..92744ffb36718a 100644 --- a/plugin-server/yarn.lock +++ b/plugin-server/yarn.lock @@ -6700,6 +6700,13 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" +kafkajs-snappy@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/kafkajs-snappy/-/kafkajs-snappy-1.1.0.tgz#0bc20fb069147b00b34b64f7fb2394e8b2b9747d" + integrity sha512-M4h2WhlxhXmR64z8nwzfGa1Dhhz78vykRfySRL8tuYQ8e6JSOuclbF2FJ8jeMJP3EZWw3uhjvwHlz7Ucu3UdWA== + dependencies: + snappyjs "^0.6.0" + kafkajs@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.0.2.tgz#cdfc8f57aa4fd69f6d9ca1cce4ee89bbc2a3a1f9" @@ -8495,6 +8502,11 @@ snapdragon@^0.8.1: source-map-resolve "^0.5.0" use "^3.1.0" +snappyjs@^0.6.0: + version "0.6.1" + resolved "https://registry.yarnpkg.com/snappyjs/-/snappyjs-0.6.1.tgz#9bca9ff8c54b133a9cc84a71d22779e97fc51878" + integrity sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg== + snowflake-sdk@^1.6.10: version "1.6.10" resolved "https://registry.yarnpkg.com/snowflake-sdk/-/snowflake-sdk-1.6.10.tgz#c6c4f267edbc50d3c1ef6fcc2651188bb8545dce" From 108210e0031590b8173da79108eb1387a78a2f72 Mon Sep 17 00:00:00 2001 From: PostHog bot <69588470+posthog-bot@users.noreply.github.com> Date: Thu, 28 Jul 2022 12:00:04 +0200 Subject: [PATCH 202/213] chore(deps): Update posthog-js to 1.26.1 (#11025) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index dbfc3335926e7c..ef6bf7e06272ce 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "kea-window-values": "^3.0.0", "md5": "^2.3.0", "monaco-editor": "^0.23.0", - "posthog-js": "1.26.0", + "posthog-js": "1.26.1", "posthog-js-lite": "^0.0.3", "prop-types": "^15.7.2", "query-selector-shadow-dom": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 8a67d60eee595f..e3f05ed0cd2195 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14105,10 +14105,10 @@ posthog-js-lite@^0.0.3: resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.3.tgz#87e373706227a849c4e7c6b0cb2066a64ad5b6ed" integrity sha512-wEOs8DEjlFBwgd7l19grosaF7mTlliZ9G9pL0Qji189FDg2ukY5IegUxTyTs7gsTGt6WK9W47BF5yXA5+bwvZg== -posthog-js@1.26.0: - version "1.26.0" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.26.0.tgz#8fd2becfbdf8f165244043d109f140ea0d02a99b" - integrity sha512-Fjc5REUJxrVTQ0WzfChn+CW/UrparZGwINPOtO9FoB25U2IrXnvnTUpkYhSPucZPWUwUMRdXBCo9838COn464Q== +posthog-js@1.26.1: + version "1.26.1" + resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.26.1.tgz#c041869cb72cfcfd0ea09bd82a25957942ecaa0d" + integrity sha512-Rj4fPEOPHzX4eAVsIFDKm2MPja7n6kAJp5ul/ryLeAsL8tTJykACARk4DuLgJygQNFrcHtAweawwvpyNuP/dTA== dependencies: "@sentry/types" "^7.2.0" fflate "^0.4.1" From 6f1dbc57e8653ff78237767799bd451e944308dd Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 28 Jul 2022 12:07:22 +0100 Subject: [PATCH 203/213] fix: pass user id not user object when checking dashboard permission (#11019) * fix: pass user id not user object when checking dashboard permission * update test to prove fix * fix: adding collaborators (#11021) * let people search users when sharing dashboards * submit UUID not email when adding dashboard collaborator --- .../src/scenes/dashboard/DashboardCollaborators.tsx | 4 ++-- posthog/api/insight.py | 2 +- posthog/api/test/test_insight.py | 13 ++++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/frontend/src/scenes/dashboard/DashboardCollaborators.tsx b/frontend/src/scenes/dashboard/DashboardCollaborators.tsx index 35eabb0d5488b2..d2c3ba1691fed5 100644 --- a/frontend/src/scenes/dashboard/DashboardCollaborators.tsx +++ b/frontend/src/scenes/dashboard/DashboardCollaborators.tsx @@ -72,10 +72,10 @@ export function DashboardCollaboration({ dashboardId }: { dashboardId: Dashboard value={explicitCollaboratorsToBeAdded} loading={explicitCollaboratorsLoading} onChange={(newValues) => setExplicitCollaboratorsToBeAdded(newValues)} - filterOption={false} + filterOption={true} mode="multiple" data-attr="subscribed-emails" - options={usersLemonSelectOptions(addableMembers)} + options={usersLemonSelectOptions(addableMembers, 'uuid')} />
Insight: dashboard: Dashboard for dashboard in Dashboard.objects.filter(id__in=ids_to_add): if ( - dashboard.get_effective_privilege_level(self.context["request"].user) + dashboard.get_effective_privilege_level(self.context["request"].user.id) == Dashboard.PrivilegeLevel.CAN_VIEW ): raise PermissionDenied( diff --git a/posthog/api/test/test_insight.py b/posthog/api/test/test_insight.py index c362bfc49bfef7..90d4eba06f05b9 100644 --- a/posthog/api/test/test_insight.py +++ b/posthog/api/test/test_insight.py @@ -1501,8 +1501,8 @@ def test_cannot_update_an_insight_if_on_restricted_dashboard(self): def test_non_admin_user_cannot_add_an_insight_to_a_restricted_dashboard(self): # create insight and dashboard separately with default user - dashboard_restricted: Dashboard = Dashboard.objects.create( - team=self.team, restriction_level=Dashboard.RestrictionLevel.ONLY_COLLABORATORS_CAN_EDIT + dashboard_restricted_id, _ = self._create_dashboard( + {"restriction_level": Dashboard.RestrictionLevel.ONLY_COLLABORATORS_CAN_EDIT} ) insight_id, response_data = self._create_insight(data={"name": "starts un-restricted dashboard"}) @@ -1514,10 +1514,17 @@ def test_non_admin_user_cannot_add_an_insight_to_a_restricted_dashboard(self): self.client.force_login(user_without_permissions) response = self.client.patch( - f"/api/projects/{self.team.id}/insights/{insight_id}", {"dashboards": [dashboard_restricted.id]}, + f"/api/projects/{self.team.id}/insights/{insight_id}", {"dashboards": [dashboard_restricted_id]}, ) assert response.status_code == status.HTTP_403_FORBIDDEN + self.client.force_login(self.user) + + response = self.client.patch( + f"/api/projects/{self.team.id}/insights/{insight_id}", {"dashboards": [dashboard_restricted_id]}, + ) + assert response.status_code == status.HTTP_200_OK + def test_non_admin_user_with_privilege_can_add_an_insight_to_a_restricted_dashboard(self): # create insight and dashboard separately with default user dashboard_restricted: Dashboard = Dashboard.objects.create( From f0f0cd4e15726dfb0d74d82a53d51bc42cd2bc77 Mon Sep 17 00:00:00 2001 From: Ben White Date: Thu, 28 Jul 2022 13:19:56 +0200 Subject: [PATCH 204/213] feat: Testing alpha releases of JS libs (#11011) * feat: Updated to alpha version of posthog-js * Swap to alpha versions of other libs --- frontend/src/loadPostHogJS.tsx | 5 ++--- .../src/toolbar/flags/featureFlagsLogic.ts | 2 +- frontend/src/toolbar/posthog.ts | 19 +++++++++++++------ frontend/src/toolbar/toolbarLogic.ts | 2 +- package.json | 4 ++-- plugin-server/package.json | 2 +- plugin-server/src/utils/posthog.ts | 10 ++++------ .../worker/ingestion/group-type-manager.ts | 15 +++++++++------ .../src/worker/ingestion/person-state.ts | 4 ++-- .../src/worker/ingestion/team-manager.ts | 17 ++++++++++------- .../tests/main/process-event.test.ts | 11 +++++++---- .../ingestion/group-type-manager.test.ts | 14 +++++++++----- .../worker/ingestion/team-manager.test.ts | 18 ++++++++++-------- plugin-server/yarn.lock | 15 +++++++++++---- yarn.lock | 18 +++++++++--------- 15 files changed, 91 insertions(+), 65 deletions(-) diff --git a/frontend/src/loadPostHogJS.tsx b/frontend/src/loadPostHogJS.tsx index 01e9f20fa930fd..d4721c985aa9b7 100644 --- a/frontend/src/loadPostHogJS.tsx +++ b/frontend/src/loadPostHogJS.tsx @@ -1,8 +1,8 @@ -import posthog from 'posthog-js' +import posthog, { PostHogConfig } from 'posthog-js' import * as Sentry from '@sentry/react' import { Integration } from '@sentry/types' -const configWithSentry = (config: posthog.Config): posthog.Config => { +const configWithSentry = (config: Partial): Partial => { if ((window as any).SENTRY_DSN) { config.on_xhr_error = (failedRequest: XMLHttpRequest) => { const status = failedRequest.status @@ -22,7 +22,6 @@ export function loadPostHogJS(): void { window.JS_POSTHOG_API_KEY, configWithSentry({ api_host: window.JS_POSTHOG_HOST, - // @ts-expect-error _capture_metrics: true, rageclick: true, debug: window.JS_POSTHOG_SELF_CAPTURE, diff --git a/frontend/src/toolbar/flags/featureFlagsLogic.ts b/frontend/src/toolbar/flags/featureFlagsLogic.ts index abfab85d2a23dd..6cf5ca2ce23d76 100644 --- a/frontend/src/toolbar/flags/featureFlagsLogic.ts +++ b/frontend/src/toolbar/flags/featureFlagsLogic.ts @@ -4,7 +4,7 @@ import type { featureFlagsLogicType } from './featureFlagsLogicType' import { toolbarFetch } from '~/toolbar/utils' import { toolbarLogic } from '~/toolbar/toolbarLogic' import Fuse from 'fuse.js' -import { PostHog } from 'posthog-js' +import type { PostHog } from 'posthog-js' import { posthog } from '~/toolbar/posthog' import { encodeParams } from 'kea-router' import { FEATURE_FLAGS } from 'lib/constants' diff --git a/frontend/src/toolbar/posthog.ts b/frontend/src/toolbar/posthog.ts index d68ed8388b719c..01c0f3e8696115 100644 --- a/frontend/src/toolbar/posthog.ts +++ b/frontend/src/toolbar/posthog.ts @@ -1,9 +1,16 @@ -import { browserPostHog } from 'posthog-js-lite/dist/src/targets/browser' +import PostHog from 'posthog-js-lite' -const apiKey = 'sTMFPsFhdP1Ssg' -const apiHost = 'https://app.posthog.com' +const runningOnPosthog = !!window.POSTHOG_APP_CONTEXT +const apiKey = runningOnPosthog ? window.JS_POSTHOG_API_KEY : 'sTMFPsFhdP1Ssg' +const apiHost = runningOnPosthog ? window.JS_POSTHOG_HOST : 'https://app.posthog.com' -export const posthog = browserPostHog(apiKey, { - apiHost: apiHost, - optedIn: false, // must call .optIn() before any events are sent +export const posthog = new PostHog(apiKey, { + host: apiHost, + enable: false, // must call .optIn() before any events are sent + persistence: 'memory', // We don't want to persist anything, all events are in-memory + persistence_name: apiKey + '_toolbar', // We don't need this but it ensures we don't accidentally mess with the standard persistence }) + +if (runningOnPosthog && window.JS_POSTHOG_SELF_CAPTURE) { + posthog.debug() +} diff --git a/frontend/src/toolbar/toolbarLogic.ts b/frontend/src/toolbar/toolbarLogic.ts index 0927bbc58e1b50..ca418adf343085 100644 --- a/frontend/src/toolbar/toolbarLogic.ts +++ b/frontend/src/toolbar/toolbarLogic.ts @@ -5,7 +5,7 @@ import { clearSessionToolbarToken } from '~/toolbar/utils' import { posthog } from '~/toolbar/posthog' import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic' import { toolbarButtonLogic } from '~/toolbar/button/toolbarButtonLogic' -import { PostHog } from 'posthog-js' +import type { PostHog } from 'posthog-js' export const toolbarLogic = kea({ path: ['toolbar', 'toolbarLogic'], diff --git a/package.json b/package.json index ef6bf7e06272ce..76ddeb4a98d453 100644 --- a/package.json +++ b/package.json @@ -90,8 +90,8 @@ "kea-window-values": "^3.0.0", "md5": "^2.3.0", "monaco-editor": "^0.23.0", - "posthog-js": "1.26.1", - "posthog-js-lite": "^0.0.3", + "posthog-js": "1.27.0-alpha5", + "posthog-js-lite": "2.0.0-alpha5", "prop-types": "^15.7.2", "query-selector-shadow-dom": "^1.0.0", "rc-trigger": "^5.2.5", diff --git a/plugin-server/package.json b/plugin-server/package.json index ce917591c1778b..4997ce3235585e 100644 --- a/plugin-server/package.json +++ b/plugin-server/package.json @@ -66,7 +66,7 @@ "node-fetch": "^2.6.1", "node-schedule": "^2.0.0", "pg": "^8.6.0", - "posthog-js-lite": "^0.0.5", + "posthog-node": "2.0.0-alpha6", "pretty-bytes": "^5.6.0", "protobufjs": "6.11.3", "re2": "^1.16.0", diff --git a/plugin-server/src/utils/posthog.ts b/plugin-server/src/utils/posthog.ts index 1aa08d4e3bdfd0..f7aaa93f13bb5e 100644 --- a/plugin-server/src/utils/posthog.ts +++ b/plugin-server/src/utils/posthog.ts @@ -1,11 +1,9 @@ -import nodeFetch from 'node-fetch' -import { nodePostHog } from 'posthog-js-lite/dist/src/targets/node' +import PostHog from 'posthog-node' -export const posthog = nodePostHog('sTMFPsFhdP1Ssg', { - fetch: nodeFetch, - apiHost: 'https://app.posthog.com', +export const posthog = new PostHog('sTMFPsFhdP1Ssg', { + host: 'https://app.posthog.com', }) if (process.env.NODE_ENV === 'test') { - posthog.optOut() + posthog.disable() } diff --git a/plugin-server/src/worker/ingestion/group-type-manager.ts b/plugin-server/src/worker/ingestion/group-type-manager.ts index 8635499ab2ab22..986f2269372f9e 100644 --- a/plugin-server/src/worker/ingestion/group-type-manager.ts +++ b/plugin-server/src/worker/ingestion/group-type-manager.ts @@ -63,12 +63,15 @@ export class GroupTypeManager { return } - posthog.identify('plugin-server') - posthog.capture('group type ingested', { - team: team.uuid, - groupType, - groupTypeIndex, - $groups: { + posthog.capture({ + distinctId: 'plugin-server', + event: 'group type ingested', + properties: { + team: team.uuid, + groupType, + groupTypeIndex, + }, + groups: { project: team.uuid, organization: team.organization_id, instance: this.instanceSiteUrl, diff --git a/plugin-server/src/worker/ingestion/person-state.ts b/plugin-server/src/worker/ingestion/person-state.ts index 03ae732e41522f..155784aacd2167 100644 --- a/plugin-server/src/worker/ingestion/person-state.ts +++ b/plugin-server/src/worker/ingestion/person-state.ts @@ -264,7 +264,7 @@ export class PersonState { timestamp: DateTime, isIdentifyCall: boolean ): Promise { - // No reason to alias person against itself. Done by posthog-js-lite when updating user properties + // No reason to alias person against itself. Done by posthog-node when updating user properties if (distinctId === previousDistinctId) { return } @@ -280,7 +280,7 @@ export class PersonState { retryIfFailed = true, totalMergeAttempts = 0 ): Promise { - // No reason to alias person against itself. Done by posthog-js-lite when updating user properties + // No reason to alias person against itself. Done by posthog-node when updating user properties if (previousDistinctId === distinctId) { return } diff --git a/plugin-server/src/worker/ingestion/team-manager.ts b/plugin-server/src/worker/ingestion/team-manager.ts index 5ac35ed8a566ee..08ce379ceb1dac 100644 --- a/plugin-server/src/worker/ingestion/team-manager.ts +++ b/plugin-server/src/worker/ingestion/team-manager.ts @@ -215,13 +215,16 @@ DO UPDATE SET property_type=$5 WHERE posthog_propertydefinition.property_type IS ) const distinctIds: { distinct_id: string }[] = organizationMembers.rows for (const { distinct_id } of distinctIds) { - posthog.identify(distinct_id) - posthog.capture('first team event ingested', { - team: team.uuid, - sdk: properties.$lib, - realm: properties.realm, - host: properties.$host, - $groups: { + posthog.capture({ + distinctId: distinct_id, + event: 'first team event ingested', + properties: { + team: team.uuid, + sdk: properties.$lib, + realm: properties.realm, + host: properties.$host, + }, + groups: { project: team.uuid, organization: team.organization_id, instance: this.instanceSiteUrl, diff --git a/plugin-server/tests/main/process-event.test.ts b/plugin-server/tests/main/process-event.test.ts index 5a0d7cc917ee37..8f05193587ee36 100644 --- a/plugin-server/tests/main/process-event.test.ts +++ b/plugin-server/tests/main/process-event.test.ts @@ -938,10 +938,13 @@ test('capture first team event', async () => { new UUIDT().toString() ) - expect(posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') - expect(posthog.capture).toHaveBeenCalledWith('first team event ingested', { - team: team.uuid, - $groups: { + expect(posthog.capture).toHaveBeenCalledWith({ + distinctId: 'plugin_test_user_distinct_id_1001', + event: 'first team event ingested', + properties: { + team: team.uuid, + }, + groups: { project: team.uuid, organization: team.organization_id, instance: 'unknown', diff --git a/plugin-server/tests/worker/ingestion/group-type-manager.test.ts b/plugin-server/tests/worker/ingestion/group-type-manager.test.ts index e8364fa63bf3be..7ad59c07450a1b 100644 --- a/plugin-server/tests/worker/ingestion/group-type-manager.test.ts +++ b/plugin-server/tests/worker/ingestion/group-type-manager.test.ts @@ -91,11 +91,15 @@ describe('GroupTypeManager()', () => { expect(hub.db.postgresQuery).toHaveBeenCalledTimes(3) // FETCH + INSERT + Team lookup const team = await hub.db.fetchTeam(2) - expect(posthog.capture).toHaveBeenCalledWith('group type ingested', { - team: team!.uuid, - groupType: 'second', - groupTypeIndex: 1, - $groups: { + expect(posthog.capture).toHaveBeenCalledWith({ + distinctId: 'plugin-server', + event: 'group type ingested', + properties: { + team: team!.uuid, + groupType: 'second', + groupTypeIndex: 1, + }, + groups: { project: team!.uuid, organization: team!.organization_id, instance: 'unknown', diff --git a/plugin-server/tests/worker/ingestion/team-manager.test.ts b/plugin-server/tests/worker/ingestion/team-manager.test.ts index 24f6e79097d4b9..704557b519d20a 100644 --- a/plugin-server/tests/worker/ingestion/team-manager.test.ts +++ b/plugin-server/tests/worker/ingestion/team-manager.test.ts @@ -268,7 +268,6 @@ describe('TeamManager()', () => { it('does not capture event', async () => { await teamManager.updateEventNamesAndProperties(2, 'new-event', { property_name: 'efg', number: 4 }) - expect(posthog.identify).not.toHaveBeenCalled() expect(posthog.capture).not.toHaveBeenCalled() }) @@ -311,13 +310,16 @@ describe('TeamManager()', () => { }) const team = await teamManager.fetchTeam(2) - expect(posthog.identify).toHaveBeenCalledWith('plugin_test_user_distinct_id_1001') - expect(posthog.capture).toHaveBeenCalledWith('first team event ingested', { - team: team!.uuid, - host: 'localhost:8000', - realm: undefined, - sdk: 'python', - $groups: { + expect(posthog.capture).toHaveBeenCalledWith({ + distinctId: 'plugin_test_user_distinct_id_1001', + event: 'first team event ingested', + properties: { + team: team!.uuid, + host: 'localhost:8000', + realm: undefined, + sdk: 'python', + }, + groups: { organization: 'ca30f2ec-e9a4-4001-bf27-3ef194086068', project: team!.uuid, instance: 'unknown', diff --git a/plugin-server/yarn.lock b/plugin-server/yarn.lock index 92744ffb36718a..2cf6449f6b932f 100644 --- a/plugin-server/yarn.lock +++ b/plugin-server/yarn.lock @@ -7811,10 +7811,12 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" -posthog-js-lite@^0.0.5: - version "0.0.5" - resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.5.tgz#984a619190d1c4ef003cb81194d0a6276491d2e8" - integrity sha512-xHVZ9qbBdoqK7G1JVA4k6nheWu50aiEwjJIPQw7Zhi705aX5wr2W8zxdmvtadeEE5qbJNg+/uZ7renOQoqcbJw== +posthog-node@2.0.0-alpha6: + version "2.0.0-alpha6" + resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-2.0.0-alpha6.tgz#9f3dd968583c004e1731f2499312cd3d0be46daa" + integrity sha512-ZPN6wNoRjwlhZPTms3ZmslJKk75mfR9rD6CPAuEtIS6vdMarNRYLDNHFUWE/F9NvgsDlTP0X2SbRf7Lx9qAFHg== + dependencies: + undici "^5.8.0" prelude-ls@^1.2.1: version "1.2.1" @@ -9219,6 +9221,11 @@ unbox-primitive@^1.0.1: has-symbols "^1.0.2" which-boxed-primitive "^1.0.2" +undici@^5.8.0: + version "5.8.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.8.0.tgz#dec9a8ccd90e5a1d81d43c0eab6503146d649a4f" + integrity sha512-1F7Vtcez5w/LwH2G2tGnFIihuWUlc58YidwLiCv+jR2Z50x0tNXpRRw7eOIJ+GvqCqIkg9SB7NWAJ/T9TLfv8Q== + unicode-canonical-property-names-ecmascript@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818" diff --git a/yarn.lock b/yarn.lock index e3f05ed0cd2195..cd05c45b08da0f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14100,15 +14100,15 @@ postcss@^8.4.14: picocolors "^1.0.0" source-map-js "^1.0.2" -posthog-js-lite@^0.0.3: - version "0.0.3" - resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-0.0.3.tgz#87e373706227a849c4e7c6b0cb2066a64ad5b6ed" - integrity sha512-wEOs8DEjlFBwgd7l19grosaF7mTlliZ9G9pL0Qji189FDg2ukY5IegUxTyTs7gsTGt6WK9W47BF5yXA5+bwvZg== - -posthog-js@1.26.1: - version "1.26.1" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.26.1.tgz#c041869cb72cfcfd0ea09bd82a25957942ecaa0d" - integrity sha512-Rj4fPEOPHzX4eAVsIFDKm2MPja7n6kAJp5ul/ryLeAsL8tTJykACARk4DuLgJygQNFrcHtAweawwvpyNuP/dTA== +posthog-js-lite@2.0.0-alpha5: + version "2.0.0-alpha5" + resolved "https://registry.yarnpkg.com/posthog-js-lite/-/posthog-js-lite-2.0.0-alpha5.tgz#60cff1b756ba2723ebb0222ca132fd0de8036210" + integrity sha512-tlkBdypJuvK/s00n4EiQjwYVfuuZv6vt8BF3g1ooIQa2Gz9Vz80p8q3qsPLZ0V5ErGRy6i3Q4fWC9TDzR7GNRQ== + +posthog-js@1.27.0-alpha5: + version "1.27.0-alpha5" + resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.27.0-alpha5.tgz#f440ed01f8d14588ae7b8fcec07d05ae12b66ce6" + integrity sha512-/HV4zjpMzDVa3D+kWzAoiLKxrrtxtVfVoCFmNjRFFwBHTbe391MGIitnHrxNSQEGx4cVA9BhvoqSjNDg/Y+6AA== dependencies: "@sentry/types" "^7.2.0" fflate "^0.4.1" From c9f05fdaf1db00ff3ba72c805966bd69a330f7d2 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Thu, 28 Jul 2022 15:05:00 +0300 Subject: [PATCH 205/213] chore(plugin-server): support tracing in plugin-server (#11029) * Experimental tracing support for plugin server * Add tag to postgresTransaction * Track event pipeline steps as separate spans * Track kafka queueMessage? * Tracing for processEvent, onEvent, onSnapshot * plugin.runTask * Move sentry code * Make tracing rate configurable --- plugin-server/src/config/config.ts | 2 + plugin-server/src/init.ts | 19 +----- plugin-server/src/sentry.ts | 60 +++++++++++++++++ plugin-server/src/types.ts | 1 + plugin-server/src/utils/db/db.ts | 5 +- .../src/utils/db/kafka-producer-wrapper.ts | 44 ++++++++----- plugin-server/src/utils/metrics.ts | 27 +++++--- .../worker/ingestion/event-pipeline/runner.ts | 31 +++++---- .../src/worker/ingestion/person-state.ts | 2 +- .../worker/ingestion/properties-updater.ts | 2 +- plugin-server/src/worker/plugins/run.ts | 60 ++++++++++++----- plugin-server/src/worker/worker.ts | 65 +++++++++++-------- plugin-server/tests/helpers/sql.ts | 7 +- .../worker/ingestion/postgres-parity.test.ts | 2 +- 14 files changed, 222 insertions(+), 105 deletions(-) create mode 100644 plugin-server/src/sentry.ts diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts index 45cc1e4e4efb22..e7a60fbcf07303 100644 --- a/plugin-server/src/config/config.ts +++ b/plugin-server/src/config/config.ts @@ -52,6 +52,7 @@ export function getDefaultConfig(): PluginsServerConfig { TASKS_PER_WORKER: 10, LOG_LEVEL: isTestEnv() ? LogLevel.Warn : LogLevel.Info, SENTRY_DSN: null, + SENTRY_PLUGIN_SERVER_TRACING_SAMPLE_RATE: 0, STATSD_HOST: null, STATSD_PORT: 8125, STATSD_PREFIX: 'plugin-server.', @@ -131,6 +132,7 @@ export function getConfigHelp(): Record { KAFKA_SASL_PASSWORD: 'Kafka SASL password', KAFKAJS_LOG_LEVEL: 'Kafka log level', SENTRY_DSN: 'Sentry ingestion URL', + SENTRY_PLUGIN_SERVER_TRACING_SAMPLE_RATE: 'Rate of tracing in plugin server (between 0 and 1)', STATSD_HOST: 'StatsD host - integration disabled if this is not provided', STATSD_PORT: 'StatsD port', STATSD_PREFIX: 'StatsD prefix', diff --git a/plugin-server/src/init.ts b/plugin-server/src/init.ts index 96c32807a3e973..15268bef134f48 100644 --- a/plugin-server/src/init.ts +++ b/plugin-server/src/init.ts @@ -1,24 +1,9 @@ -import * as Sentry from '@sentry/node' - +import { initSentry } from './sentry' import { PluginsServerConfig } from './types' import { setLogLevel } from './utils/utils' -// Must require as `tsc` strips unused `import` statements and just requiring this seems to init some globals -require('@sentry/tracing') - // Code that runs on app start, in both the main and worker threads export function initApp(config: PluginsServerConfig): void { setLogLevel(config.LOG_LEVEL) - - if (config.SENTRY_DSN) { - Sentry.init({ - dsn: config.SENTRY_DSN, - normalizeDepth: 8, // Default: 3 - initialScope: { - tags: { - PLUGIN_SERVER_MODE: config.PLUGIN_SERVER_MODE, - }, - }, - }) - } + initSentry(config) } diff --git a/plugin-server/src/sentry.ts b/plugin-server/src/sentry.ts new file mode 100644 index 00000000000000..177cf294a7f9dd --- /dev/null +++ b/plugin-server/src/sentry.ts @@ -0,0 +1,60 @@ +import * as Sentry from '@sentry/node' +import * as Tracing from '@sentry/tracing' +import { Span, SpanContext, TransactionContext } from '@sentry/types' +import { AsyncLocalStorage } from 'node:async_hooks' + +import { PluginsServerConfig } from './types' + +// Must require as `tsc` strips unused `import` statements and just requiring this seems to init some globals +require('@sentry/tracing') + +const asyncLocalStorage = new AsyncLocalStorage() + +// Code that runs on app start, in both the main and worker threads +export function initSentry(config: PluginsServerConfig): void { + if (config.SENTRY_DSN) { + Sentry.init({ + dsn: config.SENTRY_DSN, + normalizeDepth: 8, // Default: 3 + initialScope: { + tags: { + PLUGIN_SERVER_MODE: config.PLUGIN_SERVER_MODE, + }, + }, + tracesSampleRate: config.SENTRY_PLUGIN_SERVER_TRACING_SAMPLE_RATE, + }) + } +} + +export function getSpan(): Tracing.Span | undefined { + return asyncLocalStorage.getStore() +} + +export function runInTransaction(transactionContext: TransactionContext, callback: () => Promise): Promise { + const transaction = Sentry.startTransaction(transactionContext) + return asyncLocalStorage.run(transaction, async () => { + try { + const result = await callback() + return result + } finally { + transaction.finish() + } + }) +} + +export function runInSpan(spanContext: SpanContext, callback: (span?: Span) => Promise): Promise { + const parentSpan = getSpan() + if (parentSpan) { + const span = parentSpan.startChild(spanContext) + return asyncLocalStorage.run(span, async () => { + try { + const result = await callback() + return result + } finally { + span.finish() + } + }) + } else { + return callback() + } +} diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index 3b69e9bb2acf19..2eb4008f1e7a51 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -104,6 +104,7 @@ export interface PluginsServerConfig extends Record { PLUGINS_RELOAD_PUBSUB_CHANNEL: string LOG_LEVEL: LogLevel SENTRY_DSN: string | null + SENTRY_PLUGIN_SERVER_TRACING_SAMPLE_RATE: number STATSD_HOST: string | null STATSD_PORT: number STATSD_PREFIX: string diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index 420e4a1ddde803..160b09eeebd148 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -226,6 +226,7 @@ export class DB { } public postgresTransaction( + tag: string, transaction: (client: PoolClient) => Promise ): Promise { return instrumentQuery(this.statsd, 'query.postgres_transation', undefined, async () => { @@ -925,7 +926,7 @@ export class DB { ): Promise { const kafkaMessages: ProducerRecord[] = [] - const person = await this.postgresTransaction(async (client) => { + const person = await this.postgresTransaction('createPerson', async (client) => { const insertResult = await this.postgresQuery( 'INSERT INTO posthog_person (created_at, properties, properties_last_updated_at, properties_last_operation, team_id, is_user_id, is_identified, uuid, version) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) RETURNING *', [ @@ -1963,7 +1964,7 @@ export class DB { jobName: string, jobPayloadJson: Record ): Promise { - await this.postgresTransaction(async (client) => { + await this.postgresTransaction('addOrUpdatePublicJob', async (client) => { let publicJobs: Record = ( await this.postgresQuery( 'SELECT public_jobs FROM posthog_plugin WHERE id = $1 FOR UPDATE', diff --git a/plugin-server/src/utils/db/kafka-producer-wrapper.ts b/plugin-server/src/utils/db/kafka-producer-wrapper.ts index 4a40a14a0c9f07..b7226fa1cdd702 100644 --- a/plugin-server/src/utils/db/kafka-producer-wrapper.ts +++ b/plugin-server/src/utils/db/kafka-producer-wrapper.ts @@ -4,6 +4,7 @@ import { CompressionCodecs, CompressionTypes, Message, Producer, ProducerRecord // @ts-expect-error no type definitions import SnappyCodec from 'kafkajs-snappy' +import { runInSpan } from '../../sentry' import { PluginsServerConfig } from '../../types' import { instrumentQuery } from '../metrics' import { timeoutGuard } from './utils' @@ -56,25 +57,32 @@ export class KafkaProducerWrapper { }, this.flushFrequencyMs) } - async queueMessage(kafkaMessage: ProducerRecord): Promise { - const messageSize = this.estimateMessageSize(kafkaMessage) - - if (this.currentBatch.length > 0 && this.currentBatchSize + messageSize > this.maxBatchSize) { - // :TRICKY: We want to first flush then immediately add the message to the queue. Awaiting and then pushing would result in a race condition. - await this.flush(kafkaMessage) - } else { - this.currentBatch.push(kafkaMessage) - this.currentBatchSize += messageSize - - const timeSinceLastFlush = Date.now() - this.lastFlushTime - if ( - this.currentBatchSize > this.maxBatchSize || - timeSinceLastFlush > this.flushFrequencyMs || - this.currentBatch.length >= this.maxQueueSize - ) { - await this.flush() + queueMessage(kafkaMessage: ProducerRecord): Promise { + return runInSpan( + { + op: 'kafka.queueMessage', + }, + async () => { + const messageSize = this.estimateMessageSize(kafkaMessage) + + if (this.currentBatch.length > 0 && this.currentBatchSize + messageSize > this.maxBatchSize) { + // :TRICKY: We want to first flush then immediately add the message to the queue. Awaiting and then pushing would result in a race condition. + await this.flush(kafkaMessage) + } else { + this.currentBatch.push(kafkaMessage) + this.currentBatchSize += messageSize + + const timeSinceLastFlush = Date.now() - this.lastFlushTime + if ( + this.currentBatchSize > this.maxBatchSize || + timeSinceLastFlush > this.flushFrequencyMs || + this.currentBatch.length >= this.maxQueueSize + ) { + await this.flush() + } + } } - } + ) } async queueMessages(kafkaMessages: ProducerRecord[]): Promise { diff --git a/plugin-server/src/utils/metrics.ts b/plugin-server/src/utils/metrics.ts index ede8ff57b717da..73178c1156462d 100644 --- a/plugin-server/src/utils/metrics.ts +++ b/plugin-server/src/utils/metrics.ts @@ -1,24 +1,33 @@ import { StatsD, Tags } from 'hot-shots' +import { runInSpan } from '../sentry' import { UUID } from './utils' type StopCallback = () => void -export async function instrumentQuery( +export function instrumentQuery( statsd: StatsD | undefined, metricName: string, tag: string | undefined, runQuery: () => Promise ): Promise { - const tags: Tags | undefined = tag ? { queryTag: tag } : undefined - const timer = new Date() + return runInSpan( + { + op: metricName, + description: tag, + }, + async () => { + const tags: Tags | undefined = tag ? { queryTag: tag } : undefined + const timer = new Date() - statsd?.increment(`${metricName}.total`, tags) - try { - return await runQuery() - } finally { - statsd?.timing(metricName, timer, tags) - } + statsd?.increment(`${metricName}.total`, tags) + try { + return await runQuery() + } finally { + statsd?.timing(metricName, timer, tags) + } + } + ) } export function captureEventLoopMetrics(statsd: StatsD, instanceId: UUID): StopCallback { diff --git a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts index 7301a1e2cf81be..8c1e34f7960085 100644 --- a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts +++ b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts @@ -1,6 +1,7 @@ import { PluginEvent, ProcessedPluginEvent } from '@posthog/plugin-scaffold' import * as Sentry from '@sentry/node' +import { runInSpan } from '../../../sentry' import { Hub, IngestionEvent } from '../../../types' import { timeoutGuard } from '../../../utils/db/utils' import { status } from '../../../utils/status' @@ -132,17 +133,25 @@ export class EventPipelineRunner { name: Step, ...args: ArgsType ): Promise { - const timeout = timeoutGuard('Event pipeline step stalled. Timeout warning after 30 sec!', { - step: name, - event: JSON.stringify(this.originalEvent), - }) - try { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error - return EVENT_PIPELINE_STEPS[name](this, ...args) - } finally { - clearTimeout(timeout) - } + return runInSpan( + { + op: 'runStep', + description: name, + }, + () => { + const timeout = timeoutGuard('Event pipeline step stalled. Timeout warning after 30 sec!', { + step: name, + event: JSON.stringify(this.originalEvent), + }) + try { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + return EVENT_PIPELINE_STEPS[name](this, ...args) + } finally { + clearTimeout(timeout) + } + } + ) } nextStep>( diff --git a/plugin-server/src/worker/ingestion/person-state.ts b/plugin-server/src/worker/ingestion/person-state.ts index 155784aacd2167..598957c381b252 100644 --- a/plugin-server/src/worker/ingestion/person-state.ts +++ b/plugin-server/src/worker/ingestion/person-state.ts @@ -423,7 +423,7 @@ export class PersonState { // This is low-probability so likely won't occur on second retry of this block. // In the rare case of the person changing VERY often however, it may happen even a few times, // in which case we'll bail and rethrow the error. - await this.db.postgresTransaction(async (client) => { + await this.db.postgresTransaction('mergePeople', async (client) => { try { const [person, updatePersonMessages] = await this.db.updatePersonDeprecated( mergeInto, diff --git a/plugin-server/src/worker/ingestion/properties-updater.ts b/plugin-server/src/worker/ingestion/properties-updater.ts index dcd836cbd9a0df..e227d755c0de0d 100644 --- a/plugin-server/src/worker/ingestion/properties-updater.ts +++ b/plugin-server/src/worker/ingestion/properties-updater.ts @@ -28,7 +28,7 @@ export async function upsertGroup( timestamp: DateTime ): Promise { try { - const [propertiesUpdate, createdAt, version] = await db.postgresTransaction(async (client) => { + const [propertiesUpdate, createdAt, version] = await db.postgresTransaction('upsertGroup', async (client) => { const group: Group | undefined = await db.fetchGroup(teamId, groupTypeIndex, groupKey, client, { forUpdate: true, }) diff --git a/plugin-server/src/worker/plugins/run.ts b/plugin-server/src/worker/plugins/run.ts index ac604c66837868..f39e7a56fe5dab 100644 --- a/plugin-server/src/worker/plugins/run.ts +++ b/plugin-server/src/worker/plugins/run.ts @@ -1,5 +1,6 @@ import { PluginEvent, ProcessedPluginEvent } from '@posthog/plugin-scaffold' +import { runInSpan } from '../../sentry' import { Hub, PluginConfig, PluginTaskType, VMMethods } from '../../types' import { processError } from '../../utils/db/error' import { IllegalOperationError } from '../../utils/utils' @@ -11,12 +12,18 @@ export async function runOnEvent(hub: Hub, event: ProcessedPluginEvent): Promise await Promise.all( pluginMethodsToRun .filter(([, method]) => !!method) - .map( - async ([pluginConfig, onEvent]) => - await runRetriableFunction('on_event', hub, pluginConfig, { - tryFn: async () => await onEvent!(event), - event, - }) + .map(([pluginConfig, onEvent]) => + runInSpan( + { + op: 'plugin.runOnEvent', + description: pluginConfig.plugin?.name || '?', + }, + () => + runRetriableFunction('on_event', hub, pluginConfig, { + tryFn: async () => await onEvent!(event), + event, + }) + ) ) ) } @@ -27,12 +34,18 @@ export async function runOnSnapshot(hub: Hub, event: ProcessedPluginEvent): Prom await Promise.all( pluginMethodsToRun .filter(([, method]) => !!method) - .map( - async ([pluginConfig, onSnapshot]) => - await runRetriableFunction('on_snapshot', hub, pluginConfig, { - tryFn: async () => await onSnapshot!(event), - event, - }) + .map(([pluginConfig, onSnapshot]) => + runInSpan( + { + op: 'plugin.runOnSnapshot', + description: pluginConfig.plugin?.name || '?', + }, + () => + runRetriableFunction('on_snapshot', hub, pluginConfig, { + tryFn: async () => await onSnapshot!(event), + event, + }) + ) ) ) } @@ -42,7 +55,7 @@ export async function runProcessEvent(hub: Hub, event: PluginEvent): Promise processEvent(returnedEvent!) + )) || null if (returnedEvent && returnedEvent.team_id !== teamId) { returnedEvent.team_id = teamId throw new IllegalOperationError('Plugin tried to change event.team_id') @@ -111,7 +131,17 @@ export async function runPluginTask( `Task "${taskName}" not found for plugin "${pluginConfig?.plugin?.name}" with config id ${pluginConfig}` ) } - response = await (payload ? task?.exec(payload) : task?.exec()) + response = await runInSpan( + { + op: 'plugin.runTask', + description: pluginConfig?.plugin?.name || '?', + data: { + taskName, + taskType, + }, + }, + () => (payload ? task?.exec(payload) : task?.exec()) + ) } catch (error) { await processError(hub, pluginConfig || null, error) diff --git a/plugin-server/src/worker/worker.ts b/plugin-server/src/worker/worker.ts index 2e24c9c96033a0..4d0c1a81bc3269 100644 --- a/plugin-server/src/worker/worker.ts +++ b/plugin-server/src/worker/worker.ts @@ -1,6 +1,7 @@ import * as Sentry from '@sentry/node' import { initApp } from '../init' +import { runInTransaction } from '../sentry' import { Hub, PluginsServerConfig } from '../types' import { processError } from '../utils/db/error' import { createHub } from '../utils/db/hub' @@ -30,35 +31,43 @@ export async function createWorker(config: PluginsServerConfig, threadId: number export const createTaskRunner = (hub: Hub): PiscinaTaskWorker => - async ({ task, args }) => { - const timer = new Date() - let response - - Sentry.setContext('task', { task, args }) - - if (task in workerTasks) { - try { - // must clone the object, as we may get from VM2 something like { ..., properties: Proxy {} } - response = cloneObject(await workerTasks[task](hub, args)) - } catch (e) { - status.info('🔔', e) - Sentry.captureException(e) - response = { error: e.message } + ({ task, args }) => + runInTransaction( + { + op: 'piscina task', + name: task, + data: args, + }, + async () => { + const timer = new Date() + let response + + Sentry.setContext('task', { task, args }) + + if (task in workerTasks) { + try { + // must clone the object, as we may get from VM2 something like { ..., properties: Proxy {} } + response = cloneObject(await workerTasks[task](hub, args)) + } catch (e) { + status.info('🔔', e) + Sentry.captureException(e) + response = { error: e.message } + } + } else { + response = { error: `Worker task "${task}" not found in: ${Object.keys(workerTasks).join(', ')}` } + } + + hub.statsd?.timing(`piscina_task.${task}`, timer) + if (task === 'runPluginJob') { + hub.statsd?.timing('plugin_job', timer, { + type: String(args.job?.type), + pluginConfigId: String(args.job?.pluginConfigId), + pluginConfigTeam: String(args.job?.pluginConfigTeam), + }) + } + return response } - } else { - response = { error: `Worker task "${task}" not found in: ${Object.keys(workerTasks).join(', ')}` } - } - - hub.statsd?.timing(`piscina_task.${task}`, timer) - if (task === 'runPluginJob') { - hub.statsd?.timing('plugin_job', timer, { - type: String(args.job?.type), - pluginConfigId: String(args.job?.pluginConfigId), - pluginConfigTeam: String(args.job?.pluginConfigTeam), - }) - } - return response - } + ) export function processUnhandledRejections(error: Error, server: Hub): void { const pluginConfigId = pluginConfigIdFromStack(error.stack || '', server.pluginConfigSecretLookup) diff --git a/plugin-server/tests/helpers/sql.ts b/plugin-server/tests/helpers/sql.ts index e616e93fdf7591..d0b3f0f9884de0 100644 --- a/plugin-server/tests/helpers/sql.ts +++ b/plugin-server/tests/helpers/sql.ts @@ -277,8 +277,11 @@ export function onQuery(hub: Hub, onQueryCallback: (queryText: string) => any): spyOnQueryFunction(hub.postgres) const postgresTransaction = hub.db.postgresTransaction.bind(hub.db) - hub.db.postgresTransaction = async (transaction: (client: PoolClient) => Promise): Promise => { - return await postgresTransaction(async (client: PoolClient) => { + hub.db.postgresTransaction = async ( + tag: string, + transaction: (client: PoolClient) => Promise + ): Promise => { + return await postgresTransaction(tag, async (client: PoolClient) => { const query = client.query spyOnQueryFunction(client) const response = await transaction(client) diff --git a/plugin-server/tests/worker/ingestion/postgres-parity.test.ts b/plugin-server/tests/worker/ingestion/postgres-parity.test.ts index b27a4a4efba887..026c3592fd7161 100644 --- a/plugin-server/tests/worker/ingestion/postgres-parity.test.ts +++ b/plugin-server/tests/worker/ingestion/postgres-parity.test.ts @@ -416,7 +416,7 @@ describe('postgres parity', () => { // delete person - await hub.db.postgresTransaction(async (client) => { + await hub.db.postgresTransaction('', async (client) => { const deletePersonMessage = await hub.db.deletePerson(person, client) await hub.db!.kafkaProducer!.queueMessage(deletePersonMessage[0]) }) From 37d7ddd475d335f71cfa0063d31d704011f9bd9a Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jul 2022 15:38:05 +0200 Subject: [PATCH 206/213] feat(property-filters): Add hover title for long filters (#11020) --- .../components/InsightCard/InsightDetails.tsx | 20 ++++++--- .../PropertyFilters/PathItemFilters.tsx | 2 +- .../components/PropertyFilterButton.tsx | 41 +++++++++---------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/frontend/src/lib/components/InsightCard/InsightDetails.tsx b/frontend/src/lib/components/InsightCard/InsightDetails.tsx index cbe963e184bfba..f1d7c2ca415e8a 100644 --- a/frontend/src/lib/components/InsightCard/InsightDetails.tsx +++ b/frontend/src/lib/components/InsightCard/InsightDetails.tsx @@ -1,7 +1,7 @@ import { useValues } from 'kea' import { FEATURE_FLAGS } from 'lib/constants' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import { allOperatorsMapping, alphabet, convertPropertyGroupToProperties } from 'lib/utils' +import { allOperatorsMapping, alphabet, convertPropertyGroupToProperties, formatPropertyLabel } from 'lib/utils' import React from 'react' import { LocalFilter, toLocalFilters } from 'scenes/insights/filters/ActionFilter/entityFilterLogic' import { BreakdownFilter } from 'scenes/insights/filters/BreakdownFilter' @@ -15,9 +15,10 @@ import { LemonDivider } from '../LemonDivider' import { Lettermark } from '../Lettermark/Lettermark' import { Link } from '../Link' import { ProfilePicture } from '../ProfilePicture' -import { PropertyFilterText } from '../PropertyFilters/components/PropertyFilterButton' -import { PropertyKeyInfo } from '../PropertyKeyInfo' +import { keyMapping, PropertyKeyInfo } from '../PropertyKeyInfo' import { TZLabel } from '../TimezoneAware' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' +import { cohortsModel } from '~/models/cohortsModel' function CompactPropertyFiltersDisplay({ properties, @@ -26,6 +27,9 @@ function CompactPropertyFiltersDisplay({ properties: PropertyFilter[] embedded?: boolean }): JSX.Element { + const { cohortsById } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) + return ( <> {properties.map((subFilter, subIndex) => ( @@ -37,7 +41,12 @@ function CompactPropertyFiltersDisplay({ <> person belongs to cohort - + {formatPropertyLabel( + subFilter, + cohortsById, + keyMapping, + (s) => formatPropertyValueForDisplay(subFilter.key, s)?.toString() || '?' + )} ) : ( @@ -46,7 +55,8 @@ function CompactPropertyFiltersDisplay({ {subFilter.key && } - {allOperatorsMapping[subFilter.operator || 'exact']} {subFilter.value} + {allOperatorsMapping[subFilter.operator || 'exact']}{' '} + {Array.isArray(subFilter.value) ? subFilter.value.join(' or ') : subFilter.value} )} diff --git a/frontend/src/lib/components/PropertyFilters/PathItemFilters.tsx b/frontend/src/lib/components/PropertyFilters/PathItemFilters.tsx index 1f51f1e0d91ee1..3943084b68b180 100644 --- a/frontend/src/lib/components/PropertyFilters/PathItemFilters.tsx +++ b/frontend/src/lib/components/PropertyFilters/PathItemFilters.tsx @@ -65,7 +65,7 @@ export function PathItemFilters({ remove(index) }} > - {filter.value as string} + {filter.value.toString()} )} diff --git a/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx b/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx index b65d9c550343b8..69efbde74e2f85 100644 --- a/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx +++ b/frontend/src/lib/components/PropertyFilters/components/PropertyFilterButton.tsx @@ -1,37 +1,24 @@ import './PropertyFilterButton.scss' import { Button } from 'antd' -import { useValues } from 'kea' -import { formatPropertyLabel, midEllipsis } from 'lib/utils' import React from 'react' -import { cohortsModel } from '~/models/cohortsModel' import { AnyPropertyFilter } from '~/types' -import { keyMapping } from 'lib/components/PropertyKeyInfo' -import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' import { CloseButton } from 'lib/components/CloseButton' import { IconCohort, IconPerson, UnverifiedEvent } from 'lib/components/icons' import { Tooltip } from 'lib/components/Tooltip' +import { cohortsModel } from '~/models/cohortsModel' +import { useValues } from 'kea' +import { propertyDefinitionsModel } from '~/models/propertyDefinitionsModel' +import { formatPropertyLabel, midEllipsis } from 'lib/utils' +import { keyMapping } from 'lib/components/PropertyKeyInfo' export interface PropertyFilterButtonProps { onClick?: () => void onClose?: () => void - children?: string | JSX.Element + children?: string item: AnyPropertyFilter style?: React.CSSProperties } -export function PropertyFilterText({ item }: PropertyFilterButtonProps): JSX.Element { - const { cohortsById } = useValues(cohortsModel) - const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) - - return ( - <> - {formatPropertyLabel(item, cohortsById, keyMapping, (s) => - midEllipsis(formatPropertyValueForDisplay(item.key, s)?.toString() || '', 32) - )} - - ) -} - function PropertyFilterIcon({ item }: { item: AnyPropertyFilter }): JSX.Element { let iconElement = <> switch (item?.type) { @@ -62,6 +49,18 @@ function PropertyFilterIcon({ item }: { item: AnyPropertyFilter }): JSX.Element export const PropertyFilterButton = React.forwardRef( function PropertyFilterButton({ onClick, onClose, children, item, style }, ref): JSX.Element { + const { cohortsById } = useValues(cohortsModel) + const { formatPropertyValueForDisplay } = useValues(propertyDefinitionsModel) + + const label = + children || + formatPropertyLabel( + item, + cohortsById, + keyMapping, + (s) => formatPropertyValueForDisplay(item.key, s)?.toString() || '?' + ) + return ( {allActions.length === 0 && allActionsLoading ? ( diff --git a/frontend/src/toolbar/actions/ActionsTab.tsx b/frontend/src/toolbar/actions/ActionsTab.tsx index dc94871a5992b1..70f9f4aad4387c 100644 --- a/frontend/src/toolbar/actions/ActionsTab.tsx +++ b/frontend/src/toolbar/actions/ActionsTab.tsx @@ -30,7 +30,8 @@ export function ActionsTab(): JSX.Element { target="_blank" rel="noopener noreferrer" > - View & edit all {shouldSimplifyActions ? 'events' : 'actions'} + View & edit all {shouldSimplifyActions ? 'calculated events' : 'actions'}{' '} +
diff --git a/frontend/src/toolbar/actions/EditAction.tsx b/frontend/src/toolbar/actions/EditAction.tsx index ca5e6c677a5fb7..f509a2617d787d 100644 --- a/frontend/src/toolbar/actions/EditAction.tsx +++ b/frontend/src/toolbar/actions/EditAction.tsx @@ -44,7 +44,7 @@ export function EditAction(): JSX.Element {

{selectedActionId === 'new' ? 'New ' : 'Edit '} - {shouldSimplifyActions ? 'Event' : 'Action'} + {shouldSimplifyActions ? 'Calculated Event' : 'Action'}

{selectedActionId === 'new' ? 'Create ' : 'Save '} - {shouldSimplifyActions ? 'event' : 'action'} + {shouldSimplifyActions ? 'calculated event' : 'action'}
diff --git a/frontend/src/toolbar/button/DraggableButton.tsx b/frontend/src/toolbar/button/DraggableButton.tsx index de78119ca08392..9c535b2a18df38 100644 --- a/frontend/src/toolbar/button/DraggableButton.tsx +++ b/frontend/src/toolbar/button/DraggableButton.tsx @@ -63,8 +63,8 @@ export function DraggableButton(): JSX.Element { 0) ? null : shouldSimplifyActions - ? 'Events' + ? 'Calculated Events' : 'Actions' } labelPosition={side === 'left' ? 'right' : 'left'} diff --git a/frontend/src/toolbar/elements/ElementInfo.tsx b/frontend/src/toolbar/elements/ElementInfo.tsx index 682063ec310565..5c9cf5392ef720 100644 --- a/frontend/src/toolbar/elements/ElementInfo.tsx +++ b/frontend/src/toolbar/elements/ElementInfo.tsx @@ -55,17 +55,17 @@ export function ElementInfo(): JSX.Element | null {

- {shouldSimplifyActions ? 'Events' : 'Actions'} ({activeMeta.actions.length}) + {shouldSimplifyActions ? 'Calculated Events' : 'Actions'} ({activeMeta.actions.length})

{activeMeta.actions.length === 0 ? ( -

No {shouldSimplifyActions ? 'events' : 'actions'} include this element

+

No {shouldSimplifyActions ? 'calculated events' : 'actions'} include this element

) : ( a.action)} /> )}
diff --git a/posthog/settings/feature_flags.py b/posthog/settings/feature_flags.py index 277c3a4db3e76c..61ea2ec74a8caa 100644 --- a/posthog/settings/feature_flags.py +++ b/posthog/settings/feature_flags.py @@ -7,4 +7,5 @@ PERSISTED_FEATURE_FLAGS = get_list(os.getenv("PERSISTED_FEATURE_FLAGS", "")) + [ "invite-teammates-prompt", "insight-legends", + "simplify-actions", ] diff --git a/posthog/templates/authorize_and_redirect.html b/posthog/templates/authorize_and_redirect.html index 494a004630ccf6..afca37cc4c0dea 100644 --- a/posthog/templates/authorize_and_redirect.html +++ b/posthog/templates/authorize_and_redirect.html @@ -8,7 +8,7 @@
  • See aggregated stats
  • See heatmaps
    • -
  • Create new actions
    • +
  • Create new events
Authorize From 8e0752c21dd796990ed9e4e69a7b7f8b11c1811c Mon Sep 17 00:00:00 2001 From: Eric Duong Date: Thu, 28 Jul 2022 12:12:50 -0400 Subject: [PATCH 210/213] fix: Remove excess intervals (#10992) * fix: don't extend interval * Update snapshots * Update snapshots * Update snapshots * Update snapshots * Update snapshots * fix: update snapshot Co-authored-by: EDsCODE Co-authored-by: posthog-bot Co-authored-by: timgl Co-authored-by: timgl --- .../test/__snapshots__/test_event_query.ambr | 24 +++--- .../test/__snapshots__/test_trends.ambr | 78 +++++++++++++------ ...ickhouse_experiment_secondary_results.ambr | 2 +- .../test_clickhouse_experiments.ambr | 6 +- .../__snapshots__/test_clickhouse_trends.ambr | 36 ++++----- .../__snapshots__/test_action_people.ambr | 2 +- .../test/__snapshots__/test_trends.ambr | 72 +++++++++++------ posthog/queries/test/test_trends.py | 49 ++++++++++++ .../test/__snapshots__/test_breakdowns.ambr | 14 ++-- .../test/__snapshots__/test_formula.ambr | 4 +- posthog/queries/util.py | 8 +- 11 files changed, 202 insertions(+), 93 deletions(-) diff --git a/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr b/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr index b64e9a18312129..7418cbf621a047 100644 --- a/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr +++ b/ee/clickhouse/queries/test/__snapshots__/test_event_query.ambr @@ -52,7 +52,7 @@ AND (has(['Jane'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, _timestamp), 'name'), '^"|"$', '')))) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'event_name' - AND timestamp >= toStartOfDay(toDateTime('2021-01-14 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-01-14 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-01-21 23:59:59') ' --- @@ -63,7 +63,7 @@ FROM events e WHERE team_id = 2 AND event = 'viewed' - AND timestamp >= toStartOfDay(toDateTime('2021-05-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-05-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-05-07 00:00:00') ' --- @@ -88,7 +88,7 @@ HAVING max(is_deleted) = 0) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'viewed' - AND timestamp >= toStartOfDay(toDateTime('2021-05-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-05-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-05-07 00:00:00') AND (pdi.person_id IN (SELECT id @@ -107,7 +107,7 @@ FROM events e WHERE team_id = 2 AND event = 'user signed up' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-14 23:59:59') AND ((has(['hi'], "mat_test_prop")) AND (has(['hi'], "mat_test_prop"))) @@ -120,7 +120,7 @@ FROM events e WHERE team_id = 2 AND event = 'event_name' - AND timestamp >= toStartOfDay(toDateTime('2021-01-14 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-01-14 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-01-21 23:59:59') AND (((match(elements_chain, '(^|;)label(\\.|$|;|:)')))) ' @@ -132,7 +132,7 @@ FROM events e WHERE team_id = 2 AND event = 'event_name' - AND timestamp >= toStartOfDay(toDateTime('2021-01-14 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-01-14 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-01-21 23:59:59') AND (0 = 192) ' @@ -158,7 +158,7 @@ HAVING max(is_deleted) = 0) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2021-05-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-05-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-05-07 00:00:00') AND (pdi.person_id IN (SELECT id @@ -224,7 +224,7 @@ FROM events e WHERE team_id = 2 AND event = 'viewed' - AND timestamp >= toStartOfDay(toDateTime('2021-05-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-05-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-05-07 00:00:00') AND (has(['test_val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'some_key'), '^"|"$', ''))) ' @@ -236,7 +236,7 @@ FROM events e WHERE team_id = 2 AND event = 'viewed' - AND timestamp >= toStartOfDay(toDateTime('2021-05-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-05-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-05-07 00:00:00') AND (has(['test_val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'some_key'), '^"|"$', ''))) ' @@ -262,7 +262,7 @@ GROUP BY group_key) groups_1 ON "$group_1" == groups_1.group_key WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND ((has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', ''))) AND (has(['value'], replaceRegexpAll(JSONExtractRaw(group_properties_1, 'another'), '^"|"$', '')))) @@ -297,7 +297,7 @@ GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND ((has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '')))) ' @@ -323,7 +323,7 @@ HAVING max(is_deleted) = 0) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'viewed' - AND timestamp >= toStartOfDay(toDateTime('2021-05-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2021-05-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2021-05-07 00:00:00') AND (pdi.person_id IN (SELECT person_id diff --git a/ee/clickhouse/queries/test/__snapshots__/test_trends.ambr b/ee/clickhouse/queries/test/__snapshots__/test_trends.ambr index 7a20f71f665794..1242932e6d7bc0 100644 --- a/ee/clickhouse/queries/test/__snapshots__/test_trends.ambr +++ b/ee/clickhouse/queries/test/__snapshots__/test_trends.ambr @@ -119,7 +119,7 @@ breakdown_value ORDER BY d.timestamp) WHERE 11111 = 11111 - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') )) GROUP BY day_start, breakdown_value @@ -194,7 +194,7 @@ GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') in (['finance', 'technology']) GROUP BY day_start, @@ -334,7 +334,7 @@ GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') in (['finance']) GROUP BY day_start, @@ -404,7 +404,7 @@ AND ((has(['Firefox'], replaceRegexpAll(JSONExtractRaw(e.properties, '$browser'), '^"|"$', '')) OR has(['Windows'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', ''))) AND (has(['Mac'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) - AND timestamp >= toStartOfDay(toDateTime('2019-12-22 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-22 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') in (['second url']) GROUP BY day_start, @@ -489,7 +489,7 @@ GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND ((has(['finance'], replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '')))) ) GROUP BY date) @@ -533,7 +533,7 @@ AND (has(['person1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, _timestamp), 'name'), '^"|"$', '')))) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY date) group by day_start @@ -576,13 +576,41 @@ AND (has(['person1'], argMax(person."pmat_name", _timestamp)))) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY date) group by day_start order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60 ' --- +# name: TestClickhouseTrends.test_timezone_weekly + ' + + SELECT groupArray(day_start) as date, + groupArray(count) as data + FROM + (SELECT SUM(total) AS count, + day_start + from + (SELECT toUInt16(0) AS total, + toStartOfWeek(toDateTime('2020-01-26 23:59:59') - toIntervalWeek(number), 0, 'US/Pacific') AS day_start + FROM numbers(dateDiff('week', toStartOfWeek(toDateTime('2020-01-12 08:00:00'), 0, 'US/Pacific'), toDateTime('2020-01-26 23:59:59'), 'US/Pacific')) + UNION ALL SELECT toUInt16(0) AS total, + toStartOfWeek(toDateTime('2020-01-12 08:00:00'), 0, 'US/Pacific') + UNION ALL SELECT count(*) as data, + toStartOfWeek(toDateTime(timestamp), 0, 'US/Pacific') as date + FROM + (SELECT e.timestamp as timestamp + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2020-01-12 08:00:00'), 0), 'US/Pacific'), 'UTC') + AND timestamp <= toDateTime('2020-01-26 23:59:59') ) + GROUP BY date) + group by day_start + order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60 + ' +--- # name: TestClickhouseTrends.test_timezones ' @@ -604,7 +632,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) GROUP BY date) group by day_start @@ -640,7 +668,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-22 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-22 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) GROUP BY date) group by day_start @@ -701,7 +729,7 @@ GROUP BY d.timestamp ORDER BY d.timestamp) WHERE 1 = 1 - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) group by day_start order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60 @@ -728,7 +756,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) GROUP BY date) group by day_start @@ -793,7 +821,7 @@ HAVING argMax(is_deleted, version) = 0) as pdi ON events.distinct_id = pdi.distinct_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') in (['Mac']) GROUP BY day_start, @@ -827,7 +855,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfHour(toDateTime('2020-01-03 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfHour(toDateTime('2020-01-03 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-04 07:59:59') ) GROUP BY date) group by day_start @@ -891,7 +919,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfHour(toDateTime('2020-01-05 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfHour(toDateTime('2020-01-05 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 18:02:01') ) GROUP BY date) group by day_start @@ -945,7 +973,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfHour(toDateTime('2020-01-05 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfHour(toDateTime('2020-01-05 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 18:02:01') ) GROUP BY date) group by day_start @@ -1039,7 +1067,7 @@ AND event = 'sign up' AND ((NOT (replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') ILIKE '%@posthog.com%') OR has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-07-01 00:00:00') AND replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') in (['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com']) GROUP BY day_start, @@ -1139,7 +1167,7 @@ WHERE e.team_id = 2 AND event = 'sign up' AND ((has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-07-01 00:00:00') AND replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') in (['test2@posthog.com']) GROUP BY day_start, @@ -1203,7 +1231,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1', '']) ) GROUP BY $session_id, @@ -1263,7 +1291,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1', '']) ) GROUP BY $session_id, @@ -1295,7 +1323,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) events GROUP BY $session_id) ' @@ -1323,7 +1351,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) events GROUP BY $session_id) ' @@ -1363,7 +1391,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY $session_id, date) GROUP BY date) @@ -1406,7 +1434,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY $session_id, date) GROUP BY date) @@ -1492,7 +1520,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1']) ) GROUP BY $session_id, @@ -1586,7 +1614,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1']) ) GROUP BY $session_id, diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr index 3b25769c8eadee..9ba4f374170872 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiment_secondary_results.ambr @@ -51,7 +51,7 @@ WHERE e.team_id = 2 AND event = '$pageview' AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-06 00:00:00') AND replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') in (['control', 'test']) GROUP BY day_start, diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr index 6adf091cc54a90..887ab482be539c 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr @@ -253,7 +253,7 @@ WHERE e.team_id = 2 AND event = '$pageview' AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-06 00:00:00') AND replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') in (['test', 'control']) GROUP BY day_start, @@ -336,7 +336,7 @@ AND event = '$feature_flag_called' AND (has(['control', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag_response'), '^"|"$', '')) AND has(['a-b-test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature_flag'), '^"|"$', ''))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-06 00:00:00') AND replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') in (['control', 'test']) ) GROUP BY person_id, @@ -404,7 +404,7 @@ WHERE e.team_id = 2 AND event = '$pageview1' AND (has(['control', 'test_1', 'test_2', 'test'], replaceRegexpAll(JSONExtractRaw(e.properties, '$feature/a-b-test'), '^"|"$', ''))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-06 00:00:00') AND replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '') in (['control', 'test_1', 'test_2']) GROUP BY day_start, diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr index a37243d30bfba3..55297b5aaba147 100644 --- a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_trends.ambr @@ -7,7 +7,7 @@ FROM events e WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') ) events ' --- @@ -30,7 +30,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') ) GROUP BY actor_id LIMIT 200 @@ -66,7 +66,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') ) GROUP BY date) group by day_start @@ -121,7 +121,7 @@ FROM events e WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') AND (has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', ''))) ) GROUP BY date) @@ -178,7 +178,7 @@ FROM events e WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') ) GROUP BY date) group by day_start @@ -204,7 +204,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-14 23:59:59') ) GROUP BY actor_id LIMIT 200 @@ -277,7 +277,7 @@ HAVING argMax(is_deleted, version) = 0) as pdi ON events.distinct_id = pdi.distinct_id WHERE e.team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') in (['val', 'notval']) ) GROUP BY person_id, @@ -312,7 +312,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-14 23:59:59') AND (has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', ''))) ) GROUP BY actor_id @@ -352,7 +352,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') ) GROUP BY person_id) GROUP BY date) @@ -379,7 +379,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-14 23:59:59') ) GROUP BY actor_id LIMIT 200 @@ -437,7 +437,7 @@ FROM events e WHERE e.team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') in (['val', 'notval']) GROUP BY day_start, @@ -470,7 +470,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-14 23:59:59') AND (has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', ''))) ) GROUP BY actor_id @@ -585,7 +585,7 @@ breakdown_value ORDER BY d.timestamp) WHERE 11111 = 11111 - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') )) GROUP BY day_start, breakdown_value @@ -660,7 +660,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-15 23:59:59') ) GROUP BY date) group by day_start @@ -732,7 +732,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2011-12-31 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2011-12-31 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-14 23:59:59') ) GROUP BY date) group by day_start @@ -768,7 +768,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2012-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2012-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2012-01-16 23:59:59') ) GROUP BY date) group by day_start @@ -797,7 +797,7 @@ FROM events e WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 00:00:00') AND (NOT has([''], "$group_0")) ) GROUP BY date) @@ -846,7 +846,7 @@ FROM events e WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 00:00:00') ) GROUP BY date) group by day_start diff --git a/posthog/api/test/__snapshots__/test_action_people.ambr b/posthog/api/test/__snapshots__/test_action_people.ambr index f830b93c76ec48..3ce0dfc4641837 100644 --- a/posthog/api/test/__snapshots__/test_action_people.ambr +++ b/posthog/api/test/__snapshots__/test_action_people.ambr @@ -25,7 +25,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = '$pageview' - AND timestamp >= toStartOfDay(toDateTime('2020-01-08 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-08 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND (has([''], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', ''))) ) GROUP BY actor_id diff --git a/posthog/queries/test/__snapshots__/test_trends.ambr b/posthog/queries/test/__snapshots__/test_trends.ambr index 2157a0f2117608..bc3cf6cea1fac4 100644 --- a/posthog/queries/test/__snapshots__/test_trends.ambr +++ b/posthog/queries/test/__snapshots__/test_trends.ambr @@ -119,7 +119,7 @@ breakdown_value ORDER BY d.timestamp) WHERE 11111 = 11111 - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') )) GROUP BY day_start, breakdown_value @@ -186,7 +186,7 @@ AND ((has(['Firefox'], replaceRegexpAll(JSONExtractRaw(e.properties, '$browser'), '^"|"$', '')) OR has(['Windows'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', ''))) AND (has(['Mac'], replaceRegexpAll(JSONExtractRaw(e.properties, '$os'), '^"|"$', '')))) - AND timestamp >= toStartOfDay(toDateTime('2019-12-22 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-22 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$current_url'), '^"|"$', '') in (['second url']) GROUP BY day_start, @@ -264,7 +264,7 @@ AND (has(['person1'], replaceRegexpAll(JSONExtractRaw(argMax(person.properties, _timestamp), 'name'), '^"|"$', '')))) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY date) group by day_start @@ -307,13 +307,41 @@ AND (has(['person1'], argMax(person."pmat_name", _timestamp)))) person ON person.id = pdi.person_id WHERE team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY date) group by day_start order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60 ' --- +# name: TestFOSSTrends.test_timezone_weekly + ' + + SELECT groupArray(day_start) as date, + groupArray(count) as data + FROM + (SELECT SUM(total) AS count, + day_start + from + (SELECT toUInt16(0) AS total, + toStartOfWeek(toDateTime('2020-01-26 23:59:59') - toIntervalWeek(number), 0, 'US/Pacific') AS day_start + FROM numbers(dateDiff('week', toStartOfWeek(toDateTime('2020-01-12 08:00:00'), 0, 'US/Pacific'), toDateTime('2020-01-26 23:59:59'), 'US/Pacific')) + UNION ALL SELECT toUInt16(0) AS total, + toStartOfWeek(toDateTime('2020-01-12 08:00:00'), 0, 'US/Pacific') + UNION ALL SELECT count(*) as data, + toStartOfWeek(toDateTime(timestamp), 0, 'US/Pacific') as date + FROM + (SELECT e.timestamp as timestamp + FROM events e + WHERE team_id = 2 + AND event = 'sign up' + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2020-01-12 08:00:00'), 0), 'US/Pacific'), 'UTC') + AND timestamp <= toDateTime('2020-01-26 23:59:59') ) + GROUP BY date) + group by day_start + order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60 + ' +--- # name: TestFOSSTrends.test_timezones ' @@ -335,7 +363,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) GROUP BY date) group by day_start @@ -371,7 +399,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-22 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-22 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) GROUP BY date) group by day_start @@ -432,7 +460,7 @@ GROUP BY d.timestamp ORDER BY d.timestamp) WHERE 1 = 1 - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) group by day_start order by day_start SETTINGS allow_experimental_window_functions = 1) SETTINGS timeout_before_checking_execution_speed = 60 @@ -459,7 +487,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') ) GROUP BY date) group by day_start @@ -524,7 +552,7 @@ HAVING argMax(is_deleted, version) = 0) as pdi ON events.distinct_id = pdi.distinct_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-29 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-29 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$os'), '^"|"$', '') in (['Mac']) GROUP BY day_start, @@ -558,7 +586,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfHour(toDateTime('2020-01-03 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfHour(toDateTime('2020-01-03 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-04 07:59:59') ) GROUP BY date) group by day_start @@ -622,7 +650,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfHour(toDateTime('2020-01-05 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfHour(toDateTime('2020-01-05 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 18:02:01') ) GROUP BY date) group by day_start @@ -676,7 +704,7 @@ FROM events e WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfHour(toDateTime('2020-01-05 08:00:00'), 'US/Pacific') + AND timestamp >= toTimezone(toDateTime(toStartOfHour(toDateTime('2020-01-05 08:00:00')), 'US/Pacific'), 'UTC') AND timestamp <= toDateTime('2020-01-05 18:02:01') ) GROUP BY date) group by day_start @@ -770,7 +798,7 @@ AND event = 'sign up' AND ((NOT (replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') ILIKE '%@posthog.com%') OR has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-07-01 00:00:00') AND replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') in (['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com']) GROUP BY day_start, @@ -870,7 +898,7 @@ WHERE e.team_id = 2 AND event = 'sign up' AND ((has(['val'], replaceRegexpAll(JSONExtractRaw(e.properties, 'key'), '^"|"$', '')))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-01 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-01 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-07-01 00:00:00') AND replaceRegexpAll(JSONExtractRaw(person_props, 'email'), '^"|"$', '') in (['test2@posthog.com']) GROUP BY day_start, @@ -934,7 +962,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1', '']) ) GROUP BY $session_id, @@ -994,7 +1022,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1', '']) ) GROUP BY $session_id, @@ -1026,7 +1054,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) events GROUP BY $session_id) ' @@ -1054,7 +1082,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) events GROUP BY $session_id) ' @@ -1094,7 +1122,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY $session_id, date) GROUP BY date) @@ -1137,7 +1165,7 @@ GROUP BY $session_id) as sessions ON sessions.$session_id = e.$session_id WHERE team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY $session_id, date) GROUP BY date) @@ -1223,7 +1251,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0, 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime('2019-12-28 00:00:00'), 0), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1']) ) GROUP BY $session_id, @@ -1317,7 +1345,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'sign up' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') AND replaceRegexpAll(JSONExtractRaw(properties, '$some_property'), '^"|"$', '') in (['value2', 'value1']) ) GROUP BY $session_id, diff --git a/posthog/queries/test/test_trends.py b/posthog/queries/test/test_trends.py index e0ada62331509a..14047ab15345cd 100644 --- a/posthog/queries/test/test_trends.py +++ b/posthog/queries/test/test_trends.py @@ -4310,6 +4310,55 @@ def test_timezones(self, patch_feature_enabled): ) self.assertEqual(response[0]["data"], [1.0]) + @snapshot_clickhouse_queries + @patch("posthoganalytics.feature_enabled", return_value=True) + def test_timezone_weekly(self, patch_feature_enabled): + self.team.timezone = "US/Pacific" + self.team.save() + _create_person(team_id=self.team.pk, distinct_ids=["blabla"], properties={}) + with freeze_time("2020-01-12T02:01:01Z"): + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$current_url": "first url", "$browser": "Firefox", "$os": "Mac"}, + ) + + with freeze_time("2020-01-12T09:01:01Z"): + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$current_url": "first url", "$browser": "Firefox", "$os": "Mac"}, + ) + + with freeze_time("2020-01-22T01:01:01Z"): + _create_event( + team=self.team, + event="sign up", + distinct_id="blabla", + properties={"$current_url": "second url", "$browser": "Firefox", "$os": "Mac"}, + ) + + #  volume + with freeze_time("2020-01-26T07:00:00Z"): + response = trends().run( + Filter( + data={ + "date_from": "-14d", + "interval": "week", + "events": [{"id": "sign up", "name": "sign up",},], + }, + team=self.team, + ), + self.team, + ) + + self.assertEqual(response[0]["data"], [1.0, 1.0, 0.0]) + self.assertEqual( + response[0]["labels"], ["12-Jan-2020", "19-Jan-2020", "26-Jan-2020"], + ) + def test_same_day(self): _create_person(team_id=self.team.pk, distinct_ids=["blabla"], properties={}) with freeze_time("2020-01-03T01:01:01Z"): diff --git a/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr b/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr index 095ea5fafc1402..6f6bd7885ef3bd 100644 --- a/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_breakdowns.ambr @@ -51,7 +51,7 @@ FROM events e WHERE e.team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) is not null GROUP BY day_start, @@ -117,7 +117,7 @@ FROM events e WHERE e.team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND toFloat64OrNull(toString(replaceRegexpAll(JSONExtractRaw(properties, 'movie_length'), '^"|"$', ''))) is not null GROUP BY day_start, @@ -201,7 +201,7 @@ WHERE e.team_id = 2 AND event = 'watched movie' AND (NOT has(['https://test.com'], replaceRegexpAll(JSONExtractRaw(e.properties, '$current_url'), '^"|"$', ''))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND sessions.session_duration in ([120, 180, 60, 91, 0]) GROUP BY day_start, @@ -258,7 +258,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND toFloat64OrNull(toString(sessions.session_duration)) is not null GROUP BY breakdown_value @@ -338,7 +338,7 @@ WHERE e.team_id = 2 AND event = 'watched movie' AND (NOT has(['https://test.com'], replaceRegexpAll(JSONExtractRaw(e.properties, '$current_url'), '^"|"$', ''))) - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND toFloat64OrNull(toString(sessions.session_duration)) is not null GROUP BY day_start, @@ -420,7 +420,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND sessions.session_duration in ([180, 120, 91, 60, 0]) GROUP BY day_start, @@ -504,7 +504,7 @@ GROUP BY $session_id) sessions ON sessions.$session_id = e.$session_id WHERE e.team_id = 2 AND event = 'watched movie' - AND timestamp >= toStartOfDay(toDateTime('2020-01-02 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2020-01-02 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-12 23:59:59') AND toFloat64OrNull(toString(sessions.session_duration)) is not null GROUP BY day_start, diff --git a/posthog/queries/trends/test/__snapshots__/test_formula.ambr b/posthog/queries/trends/test/__snapshots__/test_formula.ambr index 98c1926a78a7f3..5714a554133958 100644 --- a/posthog/queries/trends/test/__snapshots__/test_formula.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_formula.ambr @@ -23,7 +23,7 @@ FROM events e WHERE team_id = 2 AND event = 'session start' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY date) group by day_start @@ -56,7 +56,7 @@ HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id WHERE team_id = 2 AND event = 'session start' - AND timestamp >= toStartOfDay(toDateTime('2019-12-28 00:00:00'), 'UTC') + AND timestamp >= toTimezone(toDateTime(toStartOfDay(toDateTime('2019-12-28 00:00:00')), 'UTC'), 'UTC') AND timestamp <= toDateTime('2020-01-04 23:59:59') ) GROUP BY date) group by day_start diff --git a/posthog/queries/util.py b/posthog/queries/util.py index d5bc90cb0ebb20..00fa682872f99d 100644 --- a/posthog/queries/util.py +++ b/posthog/queries/util.py @@ -142,10 +142,14 @@ def get_interval_func_ch(period: Optional[str]) -> str: def date_from_clause(interval_annotation: str, round_interval: bool) -> str: if round_interval: + # Truncate function in clickhouse will remove the time granularity and leave only the date + # Specify that this truncated date is the local timezone target + # Convert target to UTC so that stored timestamps can be compared accordingly + # Example: `2022-04-05 07:00:00` -> truncated to `2022-04-05` -> 2022-04-05 00:00:00 PST -> 2022-04-05 07:00:00 UTC if interval_annotation == "toStartOfWeek": - return "AND timestamp >= toStartOfWeek(toDateTime(%(date_from)s), 0, %(timezone)s)" + return "AND timestamp >= toTimezone(toDateTime(toStartOfWeek(toDateTime(%(date_from)s), 0), %(timezone)s), 'UTC')" - return "AND timestamp >= {interval}(toDateTime(%(date_from)s), %(timezone)s)".format( + return "AND timestamp >= toTimezone(toDateTime({interval}(toDateTime(%(date_from)s)), %(timezone)s), 'UTC')".format( interval=interval_annotation ) else: From a3922863aa041fef05be9c33f37bef3f48ccf0ac Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Thu, 28 Jul 2022 21:17:08 +0100 Subject: [PATCH 211/213] fix: allow searching and adding on staff users tab (#11033) --- frontend/src/scenes/instance/SystemStatus/StaffUsersTab.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/scenes/instance/SystemStatus/StaffUsersTab.tsx b/frontend/src/scenes/instance/SystemStatus/StaffUsersTab.tsx index ed2b3b7fb09c84..a7c17a979857cd 100644 --- a/frontend/src/scenes/instance/SystemStatus/StaffUsersTab.tsx +++ b/frontend/src/scenes/instance/SystemStatus/StaffUsersTab.tsx @@ -120,10 +120,10 @@ export function StaffUsersTab(): JSX.Element { loading={allUsersLoading} value={staffUsersToBeAdded} onChange={(newValues) => setStaffUsersToBeAdded(newValues)} - filterOption={false} + filterOption={true} mode="multiple" data-attr="subscribed-emails" - options={usersLemonSelectOptions(nonStaffUsers)} + options={usersLemonSelectOptions(nonStaffUsers, 'uuid')} /> Date: Thu, 28 Jul 2022 22:28:51 +0200 Subject: [PATCH 212/213] fix(tests): Fix circular import when running local tests (#11028) --- ee/clickhouse/queries/test/test_paths.py | 118 +++++++++++------------ 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/ee/clickhouse/queries/test/test_paths.py b/ee/clickhouse/queries/test/test_paths.py index 1341352caf8f1f..17af81a87eb671 100644 --- a/ee/clickhouse/queries/test/test_paths.py +++ b/ee/clickhouse/queries/test/test_paths.py @@ -7,7 +7,6 @@ from django.utils import timezone from freezegun import freeze_time -from ee.clickhouse.queries.paths import ClickhousePaths, ClickhousePathsActors from ee.clickhouse.queries.paths.paths_event_query import PathEventQuery from posthog.constants import ( FUNNEL_PATH_AFTER_STEP, @@ -19,13 +18,14 @@ from posthog.models.group.util import create_group from posthog.models.group_type_mapping import GroupTypeMapping from posthog.models.session_recording_event.util import create_session_recording_event +from posthog.queries.paths import Paths, PathsActors from posthog.queries.test.test_paths import paths_test_factory from posthog.test.base import _create_event, _create_person, snapshot_clickhouse_queries, test_with_materialized_columns ONE_MINUTE = 60_000 # 1 minute in milliseconds -class TestClickhousePaths(paths_test_factory(ClickhousePaths)): # type: ignore +class TestClickhousePaths(paths_test_factory(Paths)): # type: ignore maxDiff = None @@ -60,7 +60,7 @@ def _get_people_at_path(self, filter, path_start=None, path_end=None, funnel_fil person_filter = filter.with_data( {"path_start_key": path_start, "path_end_key": path_end, "path_dropoff_key": path_dropoff} ) - _, serialized_actors = ClickhousePathsActors(person_filter, self.team, funnel_filter).get_actors() + _, serialized_actors = PathsActors(person_filter, self.team, funnel_filter).get_actors() return [row["id"] for row in serialized_actors] def test_step_limit(self): @@ -98,7 +98,7 @@ def test_step_limit(self): with freeze_time("2012-01-7T03:21:34.000Z"): filter = PathFilter(data={"step_limit": 2}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, [{"source": "1_/1", "target": "2_/2", "value": 1, "average_conversion_time": ONE_MINUTE}] @@ -108,7 +108,7 @@ def test_step_limit(self): with freeze_time("2012-01-7T03:21:34.000Z"): filter = PathFilter(data={"step_limit": 3}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -121,7 +121,7 @@ def test_step_limit(self): with freeze_time("2012-01-7T03:21:34.000Z"): filter = PathFilter(data={"step_limit": 4}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -193,7 +193,7 @@ def test_step_conversion_times(self): ), filter = PathFilter(data={"step_limit": 4, "date_from": "2012-01-01", "include_event_types": ["$pageview"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -248,7 +248,7 @@ def test_path_event_ordering(self): filter = PathFilter( data={"date_from": "2021-05-01", "date_to": "2021-05-03", "include_event_types": ["custom_event"]} ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, [ @@ -405,7 +405,7 @@ def test_path_by_grouping(self): "path_groupings": ["between_step_1_*", "between_step_2_*", "step drop*"], } path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertCountEqual( response, [ @@ -545,7 +545,7 @@ def test_path_by_grouping_replacement(self): "date_to": "2021-05-07 00:00:00", } path_filter = PathFilter(data=data) - response_no_flag = ClickhousePaths(team=self.team, filter=path_filter).run() + response_no_flag = Paths(team=self.team, filter=path_filter).run() self.assertNotEqual( response_no_flag, @@ -566,7 +566,7 @@ def test_path_by_grouping_replacement(self): ) data.update({"path_replacements": "true"}) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertEqual( response, @@ -705,7 +705,7 @@ def test_path_by_grouping_replacement_multiple(self): "path_replacements": True, } path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertEqual( response, correct_response, ) @@ -717,7 +717,7 @@ def test_path_by_grouping_replacement_multiple(self): data.update({"local_path_cleaning_filters": [{"alias": "/", "regex": "/\\d+(/|\\?)?"}]}) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertEqual( response, correct_response, ) @@ -733,7 +733,7 @@ def test_path_by_grouping_replacement_multiple(self): } ) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertEqual( response, correct_response, ) @@ -757,7 +757,7 @@ def test_path_by_funnel_after_dropoff(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -811,7 +811,7 @@ def test_path_by_funnel_after_dropoff_with_group_filter(self): path_filter = PathFilter(data=data).with_data( {"properties": [{"key": "industry", "value": "technology", "type": "group", "group_type_index": 0}]} ) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -969,7 +969,7 @@ def test_path_by_funnel_after_step_respects_conversion_window(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -1023,7 +1023,7 @@ def test_path_by_funnel_after_step(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -1112,7 +1112,7 @@ def test_path_by_funnel_after_step_limit(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -1151,7 +1151,7 @@ def test_path_by_funnel_before_dropoff(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -1201,7 +1201,7 @@ def test_path_by_funnel_before_step(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -1257,7 +1257,7 @@ def test_path_by_funnel_between_step(self): } funnel_filter = Filter(data=data) path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() + response = Paths(team=self.team, filter=path_filter, funnel_filter=funnel_filter).run() self.assertEqual( response, [ @@ -1424,7 +1424,7 @@ def test_paths_end(self): "date_to": "2021-05-07 00:00:00", } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter,) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter,) self.assertEqual( response, [ @@ -1447,7 +1447,7 @@ def test_paths_end(self): "date_to": "2021-05-07 00:00:00", } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter,) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter,) self.assertEqual( response, [ @@ -1526,7 +1526,7 @@ def test_event_inclusion_exclusion_filters(self): _ = [*p1, *p2, *p3] filter = PathFilter(data={"step_limit": 4, "date_from": "2012-01-01", "include_event_types": ["$pageview"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1537,7 +1537,7 @@ def test_event_inclusion_exclusion_filters(self): ) filter = filter.with_data({"include_event_types": ["$screen"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1548,7 +1548,7 @@ def test_event_inclusion_exclusion_filters(self): ) filter = filter.with_data({"include_event_types": ["custom_event"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1559,7 +1559,7 @@ def test_event_inclusion_exclusion_filters(self): ) filter = filter.with_data({"include_event_types": [], "include_custom_events": ["/custom1", "/custom2"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1567,7 +1567,7 @@ def test_event_inclusion_exclusion_filters(self): ) filter = filter.with_data({"include_event_types": [], "include_custom_events": ["/custom3", "blah"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, [], @@ -1576,7 +1576,7 @@ def test_event_inclusion_exclusion_filters(self): filter = filter.with_data( {"include_event_types": ["$pageview", "$screen", "custom_event"], "include_custom_events": []} ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1597,7 +1597,7 @@ def test_event_inclusion_exclusion_filters(self): "exclude_events": ["/custom1", "/1", "/2", "/3"], } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, [ @@ -1698,14 +1698,14 @@ def test_event_exclusion_filters_with_wildcards(self): "path_groupings": ["/bar/*/foo"], } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, [{"source": "1_/1", "target": "2_/3", "value": 3, "average_conversion_time": 3 * ONE_MINUTE},], ) filter = filter.with_data({"path_groupings": ["/xxx/invalid/*"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual(len(response), 6) @@ -1761,7 +1761,7 @@ def test_event_inclusion_exclusion_filters_across_single_person(self): _create_event(distinct_id="p1", event="/custom3", team=self.team, timestamp="2012-01-01 03:32:34",), filter = PathFilter(data={"step_limit": 10, "date_from": "2012-01-01"}) # include everything, exclude nothing - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1778,7 +1778,7 @@ def test_event_inclusion_exclusion_filters_across_single_person(self): ) filter = filter.with_data({"include_event_types": ["$pageview", "$screen"]}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1794,7 +1794,7 @@ def test_event_inclusion_exclusion_filters_across_single_person(self): filter = filter.with_data( {"include_event_types": ["$pageview", "$screen"], "include_custom_events": ["/custom2"]} ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1815,7 +1815,7 @@ def test_event_inclusion_exclusion_filters_across_single_person(self): "exclude_events": ["/custom1", "/custom3"], } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1873,7 +1873,7 @@ def test_path_respect_session_limits(self): ), filter = PathFilter(data={"date_from": "2012-01-01"}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -1947,7 +1947,7 @@ def test_path_removes_duplicates(self): ), filter = PathFilter(data={"date_from": "2012-01-01"}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -2069,7 +2069,7 @@ def test_paths_start_and_end(self): }, team=self.team, ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter,) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter,) self.assertEqual( response, [{"source": "1_/5", "target": "2_/about", "value": 2, "average_conversion_time": 60000.0}] ) @@ -2077,7 +2077,7 @@ def test_paths_start_and_end(self): # test aggregation for long paths filter = filter.with_data({"start_point": "/2", "step_limit": 4}) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter,) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter,) self.assertEqual( response, [ @@ -2210,7 +2210,7 @@ def test_path_grouping_across_people(self): "path_groupings": ["/bar/*/foo"], } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -2280,7 +2280,7 @@ def test_path_grouping_with_evil_input(self): "path_groupings": ["(a+)+", "[aaa|aaaa]+", "1.*", ".*", "/3?q=1", "/3*"], } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, [ @@ -2543,7 +2543,7 @@ def test_paths_start_dropping_orphaned_edges(self): "edge_limit": "6", } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter,) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter,) self.assertEqual( response, [ @@ -2567,7 +2567,7 @@ def test_path_min_edge_weight(self): "path_groupings": ["between_step_1_*", "between_step_2_*", "step drop*"], } path_filter = PathFilter(data=data) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertCountEqual( response, [ @@ -2597,7 +2597,7 @@ def test_path_min_edge_weight(self): ) path_filter = path_filter.with_data({"edge_limit": 2}) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertCountEqual( response, [ @@ -2621,7 +2621,7 @@ def test_path_min_edge_weight(self): ) path_filter = path_filter.with_data({"edge_limit": 20, "max_edge_weight": 11, "min_edge_weight": 6}) - response = ClickhousePaths(team=self.team, filter=path_filter).run() + response = Paths(team=self.team, filter=path_filter).run() self.assertCountEqual( response, [ @@ -2732,7 +2732,7 @@ def test_path_groups_filtering(self): "properties": [{"key": "industry", "value": "finance", "type": "group", "group_type_index": 0}], } ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -2745,7 +2745,7 @@ def test_path_groups_filtering(self): filter = filter.with_data( {"properties": [{"key": "industry", "value": "technology", "type": "group", "group_type_index": 0}]} ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -2758,7 +2758,7 @@ def test_path_groups_filtering(self): filter = filter.with_data( {"properties": [{"key": "industry", "value": "technology", "type": "group", "group_type_index": 1}]} ) - response = ClickhousePaths(team=self.team, filter=filter).run(team=self.team, filter=filter) + response = Paths(team=self.team, filter=filter).run(team=self.team, filter=filter) self.assertEqual( response, @@ -2841,7 +2841,7 @@ def test_path_recording(self): "include_recordings": "true", } ) - _, serialized_actors = ClickhousePathsActors(filter, self.team).get_actors() + _, serialized_actors = PathsActors(filter, self.team).get_actors() self.assertCountEqual([p1.uuid, p2.uuid], [actor["id"] for actor in serialized_actors]) matched_recordings = [actor["matched_recordings"] for actor in serialized_actors] @@ -2903,7 +2903,7 @@ def test_path_recording_with_no_window_or_session_id(self): "include_recordings": "true", } ) - _, serialized_actors = ClickhousePathsActors(filter, self.team).get_actors() + _, serialized_actors = PathsActors(filter, self.team).get_actors() self.assertEqual([p1.uuid], [actor["id"] for actor in serialized_actors]) self.assertEqual( [[]], [actor["matched_recordings"] for actor in serialized_actors], @@ -2952,7 +2952,7 @@ def test_path_recording_with_start_and_end(self): "include_recordings": "true", } ) - _, serialized_actors = ClickhousePathsActors(filter, self.team).get_actors() + _, serialized_actors = PathsActors(filter, self.team).get_actors() self.assertEqual([p1.uuid], [actor["id"] for actor in serialized_actors]) self.assertEqual( [ @@ -3014,7 +3014,7 @@ def test_path_recording_for_dropoff(self): "include_recordings": "true", } ) - _, serialized_actors = ClickhousePathsActors(filter, self.team).get_actors() + _, serialized_actors = PathsActors(filter, self.team).get_actors() self.assertEqual([], [actor["id"] for actor in serialized_actors]) self.assertEqual( [], [actor["matched_recordings"] for actor in serialized_actors], @@ -3030,7 +3030,7 @@ def test_path_recording_for_dropoff(self): "include_recordings": "true", } ) - _, serialized_actors = ClickhousePathsActors(filter, self.team).get_actors() + _, serialized_actors = PathsActors(filter, self.team).get_actors() self.assertEqual([p1.uuid], [actor["id"] for actor in serialized_actors]) self.assertEqual( [ @@ -3059,21 +3059,21 @@ class TestClickhousePathsEdgeValidation(TestCase): def test_basic_forest(self): edges = self.BASIC_PATH + self.BASIC_PATH_2 - results = ClickhousePaths(PathFilter(), MagicMock()).validate_results(edges) + results = Paths(PathFilter(), MagicMock()).validate_results(edges) self.assertCountEqual(results, self.BASIC_PATH + self.BASIC_PATH_2) def test_basic_forest_with_dangling_edges(self): edges = self.BASIC_PATH + self.BASIC_PATH_2 + [("2_w", "3_z"), ("3_x", "4_d"), ("2_xxx", "3_yyy")] - results = ClickhousePaths(PathFilter(), MagicMock()).validate_results(edges) + results = Paths(PathFilter(), MagicMock()).validate_results(edges) self.assertCountEqual(results, self.BASIC_PATH + self.BASIC_PATH_2) def test_basic_forest_with_dangling_and_cross_edges(self): edges = self.BASIC_PATH + self.BASIC_PATH_2 + [("2_w", "3_z"), ("3_x", "4_d"), ("2_y", "3_c")] - results = ClickhousePaths(PathFilter(), MagicMock()).validate_results(edges) + results = Paths(PathFilter(), MagicMock()).validate_results(edges) self.assertCountEqual(results, self.BASIC_PATH + self.BASIC_PATH_2 + [("2_y", "3_c")]) @@ -3082,6 +3082,6 @@ def test_no_start_point(self): edges.remove(("1_a", "2_b")) # remove first start point edges = list(edges) # type: ignore - results = ClickhousePaths(PathFilter(), MagicMock()).validate_results(edges) + results = Paths(PathFilter(), MagicMock()).validate_results(edges) self.assertCountEqual(results, self.BASIC_PATH_2) From 08e4489a2685ddcf65785adfe4af38286f3e27c1 Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Thu, 28 Jul 2022 22:59:30 +0200 Subject: [PATCH 213/213] fix(persons): Use person UUIDs in merging instead of serial IDs (#11038) * fix(persons): Use person UUIDs in merging instead of serial IDs * fix tests Co-authored-by: Tim Glaser --- frontend/src/scenes/persons/MergeSplitPerson.tsx | 8 ++++---- frontend/src/scenes/persons/mergeSplitPersonLogic.ts | 8 ++++---- posthog/api/person.py | 2 +- posthog/api/test/test_person.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/src/scenes/persons/MergeSplitPerson.tsx b/frontend/src/scenes/persons/MergeSplitPerson.tsx index a48e5ab748db2d..0a20242e319634 100644 --- a/frontend/src/scenes/persons/MergeSplitPerson.tsx +++ b/frontend/src/scenes/persons/MergeSplitPerson.tsx @@ -75,17 +75,17 @@ function MergePerson(): JSX.Element { setSelectedPersonsToMerge(value.map((x) => parseInt(x, 10)))} + onChange={(value) => setSelectedPersonsToMerge(value)} filterOption={false} onSearch={(value) => setListFilters({ search: value })} mode="multiple" data-attr="subscribed-emails" value={selectedPersonsToMerge.map((x) => x.toString())} options={(persons.results || []) - .filter((p: PersonType) => p.id && p.uuid !== person.uuid) + .filter((p: PersonType) => p.uuid && p.uuid !== person.uuid) .map((p) => ({ - key: `${p.id}`, - label: `${p.name || p.id}`, + key: p.uuid as string, + label: (p.name || p.uuid) as string, }))} disabled={executedLoading} /> diff --git a/frontend/src/scenes/persons/mergeSplitPersonLogic.ts b/frontend/src/scenes/persons/mergeSplitPersonLogic.ts index f3ebfa69eaacfc..c8da575cd02577 100644 --- a/frontend/src/scenes/persons/mergeSplitPersonLogic.ts +++ b/frontend/src/scenes/persons/mergeSplitPersonLogic.ts @@ -16,7 +16,7 @@ export interface SplitPersonLogicProps { person: PersonType } -export type PersonIds = NonNullable[] +export type PersonUuids = NonNullable[] export const mergeSplitPersonLogic = kea({ props: {} as SplitPersonLogicProps, @@ -31,7 +31,7 @@ export const mergeSplitPersonLogic = kea({ }), actions: { setActivity: (activity: ActivityType) => ({ activity }), - setSelectedPersonsToMerge: (persons: PersonIds) => ({ persons }), + setSelectedPersonsToMerge: (persons: PersonUuids) => ({ persons }), setSelectedPersonToAssignSplit: (id: string) => ({ id }), cancel: true, }, @@ -44,7 +44,7 @@ export const mergeSplitPersonLogic = kea({ ], person: [props.person, {}], selectedPersonsToMerge: [ - [] as PersonIds, + [] as PersonUuids, { setSelectedPersonsToMerge: (_, { persons }) => persons, }, @@ -73,7 +73,7 @@ export const mergeSplitPersonLogic = kea({ execute: async () => { if (values.activity === ActivityType.MERGE) { const newPerson = await api.create('api/person/' + values.person.id + '/merge/', { - ids: values.selectedPersonsToMerge, + uuids: values.selectedPersonsToMerge, }) if (newPerson.id) { lemonToast.success('Persons have been merged') diff --git a/posthog/api/person.py b/posthog/api/person.py index f641010f8c1fbc..93f719f852b372 100644 --- a/posthog/api/person.py +++ b/posthog/api/person.py @@ -372,7 +372,7 @@ def _get_person_property_values_for_key(self, key, value): @action(methods=["POST"], detail=True) def merge(self, request: request.Request, pk=None, **kwargs) -> response.Response: - people = Person.objects.filter(team_id=self.team_id, pk__in=request.data.get("ids")) + people = Person.objects.filter(team_id=self.team_id, uuid__in=request.data.get("uuids")) person = self.get_object() person.merge_people([p for p in people]) diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.py index b3c2a5d636d125..47498ba019b7d9 100644 --- a/posthog/api/test/test_person.py +++ b/posthog/api/test/test_person.py @@ -252,7 +252,7 @@ def test_merge_people(self, mock_capture_internal) -> None: ) person2 = _create_person(team=self.team, distinct_ids=["2"], properties={"random_prop": "asdf"}) - response = self.client.post("/api/person/%s/merge/" % person1.pk, {"ids": [person2.pk, person3.pk]},) + response = self.client.post("/api/person/%s/merge/" % person1.uuid, {"uuids": [person2.uuid, person3.uuid]},) mock_capture_internal.assert_has_calls( [ mock.call(