diff --git a/ee/clickhouse/queries/experiments/__init__.py b/ee/clickhouse/queries/experiments/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/ee/clickhouse/queries/experiments/funnel_experiment_result.py b/ee/clickhouse/queries/experiments/funnel_experiment_result.py new file mode 100644 index 0000000000000..12021fe112b39 --- /dev/null +++ b/ee/clickhouse/queries/experiments/funnel_experiment_result.py @@ -0,0 +1,126 @@ +import dataclasses +from datetime import datetime +from typing import List, Optional, Tuple, Type + +from numpy.random import default_rng +from rest_framework.exceptions import ValidationError + +from ee.clickhouse.queries.funnels import ClickhouseFunnel, funnel +from posthog.models.filters.filter import Filter +from posthog.models.team import Team + + +@dataclasses.dataclass +class Variant: + name: str + success_count: int + failure_count: int + + +SIMULATION_COUNT = 100_000 + + +class ClickhouseFunnelExperimentResult: + """ + This class calculates Experiment Results. + It returns two things: + 1. A Funnel Breakdown based on Feature Flag values + 2. Probability that Feature Flag value 1 has better conversion rate then FeatureFlag value 2 + + Currently, it only supports two feature flag values: control and test + + The passed in Filter determines which funnel to create, along with the experiment start & end date values + + Calculating (2) uses sampling from a Beta distribution. If `control` value for the feature flag has 10 successes and 12 conversion failures, + we assume the conversion rate follows a Beta(10, 12) distribution. Same for `test` variant. + + Then, we calculcate how many times a sample from `test` variant is higher than a sample from the `control` variant. This becomes the + probability. + """ + + def __init__( + self, + filter: Filter, + team: Team, + feature_flag: str, + experiment_start_date: datetime, + experiment_end_date: Optional[datetime] = None, + funnel_class: Type[ClickhouseFunnel] = ClickhouseFunnel, + ): + + breakdown_key = f"$feature/{feature_flag}" + + query_filter = filter.with_data( + { + "date_from": experiment_start_date, + "date_to": experiment_end_date, + "breakdown": breakdown_key, + "breakdown_type": "event", + "properties": [ + {"key": breakdown_key, "value": ["control", "test"], "operator": "exact", "type": "event"} + ], + # :TRICKY: We don't use properties set on filters, instead using experiment variant options + } + ) + self.funnel = funnel_class(query_filter, team) + + def get_results(self): + funnel_results = self.funnel.run() + variants = self.get_variants(funnel_results) + + probability = self.calculate_results(variants) + + return {"funnel": funnel_results, "probability": probability} + + def get_variants(self, funnel_results): + variants = [] + for result in funnel_results: + total = sum([step["count"] for step in result]) + success = result[-1]["count"] + failure = total - success + breakdown_value = result[0]["breakdown_value"][0] + + variants.append(Variant(breakdown_value, success, failure)) + + # Default variant names: control and test + return sorted(variants, key=lambda variant: variant.name, reverse=True) + + @staticmethod + def calculate_results( + variants: List[Variant], priors: Tuple[int, int] = (1, 1), simulations_count: int = SIMULATION_COUNT + ): + """ + # Calculates probability that A is better than B + # Only supports 2 variants today + + For each variant, we create a Beta distribution of conversion rates, + where alpha (successes) = success count of variant + prior success + beta (failures) = failure count + variant + prior failures + + The prior is information about the world we already know. For example, a stronger prior for failures implies + you'd need extra evidence of successes to confirm that the variant is indeed better. + + By default, we choose a non-informative prior. That is, both success & failure are equally likely. + + """ + if len(variants) > 2: + raise ValidationError("Can't calculate A/B test results for more than 2 variants") + + if len(variants) < 2: + raise ValidationError("Can't calculate A/B test results for less than 2 variants") + + prior_success, prior_failure = priors + + random_sampler = default_rng() + variant_samples = [] + for variant in variants: + # Get `N=simulations` samples from a Beta distribution with alpha = prior_success + variant_sucess, + # and beta = prior_failure + variant_failure + samples = random_sampler.beta( + variant.success_count + prior_success, variant.failure_count + prior_failure, simulations_count + ) + variant_samples.append(samples) + + probability = sum(sample_a > sample_b for (sample_a, sample_b) in zip(*variant_samples)) / simulations_count + + return probability diff --git a/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py b/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py new file mode 100644 index 0000000000000..6eeba6f10205c --- /dev/null +++ b/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py @@ -0,0 +1,13 @@ +import unittest + +from ee.clickhouse.queries.experiments.funnel_experiment_result import ClickhouseFunnelExperimentResult, Variant + + +class TestFunnelExperimentCalculator(unittest.TestCase): + def test_calculate_results(self): + + variant_a = Variant("A", 100, 10) + variant_b = Variant("B", 100, 18) + + probability = ClickhouseFunnelExperimentResult.calculate_results([variant_a, variant_b]) + self.assertTrue(probability > 0.9) diff --git a/ee/clickhouse/views/experiments.py b/ee/clickhouse/views/experiments.py new file mode 100644 index 0000000000000..7c720567ea0b9 --- /dev/null +++ b/ee/clickhouse/views/experiments.py @@ -0,0 +1,139 @@ +from typing import Any + +from rest_framework import request, serializers, viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import IsAuthenticated +from rest_framework.request import Request +from rest_framework.response import Response + +from ee.clickhouse.queries.experiments.funnel_experiment_result import ClickhouseFunnelExperimentResult +from posthog.api.routing import StructuredViewSetMixin +from posthog.models.experiment import Experiment +from posthog.models.feature_flag import FeatureFlag +from posthog.models.filters.filter import Filter +from posthog.models.team import Team +from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission + + +class ExperimentSerializer(serializers.ModelSerializer): + + feature_flag_key = serializers.CharField(source="get_feature_flag_key") + + class Meta: + model = Experiment + fields = [ + "id", + "name", + "description", + "start_date", + "end_date", + "feature_flag_key", + "parameters", + "filters", + "created_by", + "created_at", + "updated_at", + ] + read_only_fields = [ + "id", + "created_by", + "created_at", + "updated_at", + ] + + def validate_feature_flag_key(self, value): + if FeatureFlag.objects.filter(key=value, team_id=self.context["team_id"], deleted=False).exists(): + raise ValidationError("Feature Flag key already exists. Please select a unique key") + + return value + + def create(self, validated_data: dict, *args: Any, **kwargs: Any) -> Experiment: + request = self.context["request"] + validated_data["created_by"] = request.user + team = Team.objects.get(id=self.context["team_id"]) + + feature_flag_key = validated_data.pop("get_feature_flag_key") + + is_draft = "start_date" in validated_data + + properties = validated_data["filters"].get("properties", []) + filters = { + "groups": [{"properties": properties, "rollout_percentage": None}], + "multivariate": { + "variants": [ + {"key": "control", "name": "Control Group", "rollout_percentage": 50}, + {"key": "test", "name": "Test Variant", "rollout_percentage": 50}, + ] + }, + } + + feature_flag = FeatureFlag.objects.create( + key=feature_flag_key, + name=f'Feature Flag for Experiment {validated_data["name"]}', + team=team, + created_by=request.user, + filters=filters, + active=False if is_draft else True, + ) + + experiment = Experiment.objects.create(team=team, feature_flag=feature_flag, **validated_data) + return experiment + + def update(self, instance: Experiment, validated_data: dict, *args: Any, **kwargs: Any) -> Experiment: + + expected_keys = set(["name", "description", "start_date", "end_date", "parameters"]) + given_keys = set(validated_data.keys()) + + extra_keys = given_keys - expected_keys + + if extra_keys: + raise ValidationError(f"Can't update keys: {', '.join(sorted(extra_keys))} on Experiment") + + has_start_date = "start_date" in validated_data + + feature_flag = instance.feature_flag + + if instance.is_draft and has_start_date: + feature_flag.active = True + feature_flag.save() + return super().update(instance, validated_data) + + elif has_start_date: + raise ValidationError("Can't change experiment start date after experiment has begun") + else: + # Not a draft, doesn't have start date + # Or draft without start date + return super().update(instance, validated_data) + + +class ClickhouseExperimentsViewSet(StructuredViewSetMixin, viewsets.ModelViewSet): + serializer_class = ExperimentSerializer + queryset = Experiment.objects.all() + permission_classes = [IsAuthenticated, ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission] + + def get_queryset(self): + return super().get_queryset() + + # ****************************************** + # /projects/:id/experiments/:experiment_id/results + # + # Returns current results of an experiment, and graphs + # 1. Probability of success + # 2. Funnel breakdown graph to display + # ****************************************** + @action(methods=["GET"], detail=True) + def results(self, request: Request, *args: Any, **kwargs: Any) -> Response: + experiment: Experiment = self.get_object() + + if not experiment.filters: + raise ValidationError("Experiment has no target metric") + + result = ClickhouseFunnelExperimentResult( + Filter(experiment.filters), + self.team, + experiment.feature_flag.key, + experiment.start_date, + experiment.end_date, + ).get_results() + return Response(result) diff --git a/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr new file mode 100644 index 0000000000000..b90c9933c0901 --- /dev/null +++ b/ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr @@ -0,0 +1,106 @@ +# name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results + ' + /* request:api_projects_(?P[^_.]+)_experiments_(?P[^_.]+)_results_?$ (ClickhouseExperimentsViewSet) */ + SELECT groupArray(value) + FROM + (SELECT array(trim(BOTH '"' + FROM JSONExtractRaw(properties, '$feature/a-b-test'))) AS value, + count(*) as count + FROM events e + WHERE team_id = 2 + AND event = '$pageview' + AND timestamp >= '2020-01-01 00:00:00' + AND timestamp <= '2020-01-06 23:59:59' + AND has(['control', 'test'], trim(BOTH '"' + FROM JSONExtractRaw(e.properties, '$feature/a-b-test'))) + GROUP BY value + ORDER BY count DESC + LIMIT 10 + OFFSET 0) + ' +--- +# name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results.1 + ' + /* request:api_projects_(?P[^_.]+)_experiments_(?P[^_.]+)_results_?$ (ClickhouseExperimentsViewSet) */ + SELECT countIf(steps = 1) step_1, + countIf(steps = 2) step_2, + avg(step_1_average_conversion_time_inner) step_1_average_conversion_time, + median(step_1_median_conversion_time_inner) step_1_median_conversion_time, + prop + FROM + (SELECT aggregation_target, + steps, + avg(step_1_conversion_time) step_1_average_conversion_time_inner, + median(step_1_conversion_time) step_1_median_conversion_time_inner, + prop + FROM + (SELECT aggregation_target, + steps, + max(steps) over (PARTITION BY aggregation_target, + prop) as max_steps, + step_1_conversion_time, + prop + FROM + (SELECT *, + if(latest_0 < latest_1 + AND latest_1 <= latest_0 + INTERVAL 14 DAY, 2, 1) AS steps , + if(isNotNull(latest_1) + AND latest_1 <= latest_0 + INTERVAL 14 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, + prop + FROM + (SELECT aggregation_target, + timestamp, + step_0, + latest_0, + step_1, + min(latest_1) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_1 , + if(has([['test'], ['control']], prop), prop, ['Other']) as prop + FROM + (SELECT aggregation_target, + timestamp, + if(event = '$pageview', 1, 0) as step_0, + if(step_0 = 1, timestamp, null) as latest_0, + if(event = '$pageleave', 1, 0) as step_1, + if(step_1 = 1, timestamp, null) as latest_1, + array(trim(BOTH '"' + FROM JSONExtractRaw(properties, '$feature/a-b-test'))) AS prop + FROM + (SELECT e.event as event, + e.team_id as team_id, + e.distinct_id as distinct_id, + e.timestamp as timestamp, + pdi.person_id as aggregation_target, + e.properties as properties + FROM events e + INNER JOIN + (SELECT distinct_id, + argMax(person_id, _timestamp) as person_id + FROM + (SELECT distinct_id, + person_id, + max(_timestamp) as _timestamp + FROM person_distinct_id + WHERE team_id = 2 + GROUP BY person_id, + distinct_id, + team_id + HAVING max(is_deleted) = 0) + GROUP BY distinct_id) AS pdi ON events.distinct_id = pdi.distinct_id + WHERE team_id = 2 + AND event IN ['$pageleave', '$pageview'] + AND timestamp >= '2020-01-01 00:00:00' + AND timestamp <= '2020-01-06 23:59:59' + AND has(['control', 'test'], trim(BOTH '"' + FROM JSONExtractRaw(properties, '$feature/a-b-test'))) ) events + WHERE (step_0 = 1 + OR step_1 = 1) )) + WHERE step_0 = 1 SETTINGS allow_experimental_window_functions = 1 )) + GROUP BY aggregation_target, + steps, + prop + HAVING steps = max_steps SETTINGS allow_experimental_window_functions = 1) + GROUP BY prop SETTINGS allow_experimental_window_functions = 1 + ' +--- diff --git a/ee/clickhouse/views/test/test_clickhouse_experiments.py b/ee/clickhouse/views/test/test_clickhouse_experiments.py new file mode 100644 index 0000000000000..1f1cf162a336a --- /dev/null +++ b/ee/clickhouse/views/test/test_clickhouse_experiments.py @@ -0,0 +1,206 @@ +from datetime import datetime + +from rest_framework import status + +from ee.api.test.base import LicensedTestMixin +from ee.clickhouse.test.test_journeys import journeys_for +from ee.clickhouse.util import ClickhouseTestMixin, snapshot_clickhouse_queries +from posthog.models.experiment import Experiment +from posthog.models.feature_flag import FeatureFlag +from posthog.test.base import APIBaseTest + + +class TestExperimentCRUD(APIBaseTest): + def test_creating_updating_basic_experiment(self): + ff_key = "a-b-tests" + response = self.client.post( + f"/api/projects/{self.team.id}/experiments/", + { + "name": "Test Experiment", + "description": "", + "start_date": "2021-12-01T10:23", + "end_date": None, + "feature_flag_key": ff_key, + "parameters": None, + "filters": { + "events": [{"order": 0, "id": "$pageview"}, {"order": 1, "id": "$pageleave"}], + "properties": [ + {"key": "$geoip_country_name", "type": "person", "value": ["france"], "operator": "exact"} + ], + }, + }, + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.json()["name"], "Test Experiment") + self.assertEqual(response.json()["feature_flag_key"], ff_key) + + created_ff = FeatureFlag.objects.get(key=ff_key) + + self.assertEqual(created_ff.key, ff_key) + self.assertEqual(created_ff.filters["multivariate"]["variants"][0]["key"], "control") + self.assertEqual(created_ff.filters["multivariate"]["variants"][1]["key"], "test") + self.assertEqual(created_ff.filters["groups"][0]["properties"][0]["key"], "$geoip_country_name") + + id = response.json()["id"] + end_date = "2021-12-10T00:00" + + # Now update + response = self.client.patch( + f"/api/projects/{self.team.id}/experiments/{id}", {"description": "Bazinga", "end_date": end_date,}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + experiment = Experiment.objects.get(pk=id) + self.assertEqual(experiment.description, "Bazinga") + self.assertEqual(experiment.end_date.strftime("%Y-%m-%dT%H:%M"), end_date) + + def test_invalid_create(self): + # Draft experiment + ff_key = "a-b-tests" + response = self.client.post( + f"/api/projects/{self.team.id}/experiments/", + { + "name": None, # invalid + "description": "", + "start_date": None, + "end_date": None, + "feature_flag_key": ff_key, + "parameters": {}, + "filters": {}, + }, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.json()["detail"], "This field may not be null.") + + def test_invalid_update(self): + # Draft experiment + ff_key = "a-b-tests" + response = self.client.post( + f"/api/projects/{self.team.id}/experiments/", + { + "name": "Test Experiment", + "description": "", + "start_date": None, + "end_date": None, + "feature_flag_key": ff_key, + "parameters": {}, + "filters": {}, + }, + ) + + id = response.json()["id"] + + # Now update + response = self.client.patch( + f"/api/projects/{self.team.id}/experiments/{id}", + {"description": "Bazinga", "filters": {}, "feature_flag_key": "new_key",}, # invalid # invalid + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.json()["detail"], "Can't update keys: filters, get_feature_flag_key on Experiment") + + def test_cant_reuse_existing_feature_flag(self): + ff_key = "a-b-test" + FeatureFlag.objects.create( + team=self.team, rollout_percentage=50, name="Beta feature", key=ff_key, created_by=self.user, + ) + response = self.client.post( + f"/api/projects/{self.team.id}/experiments/", + { + "name": "Test Experiment", + "description": "", + "start_date": "2021-12-01T10:23", + "end_date": None, + "feature_flag_key": ff_key, + "parameters": None, + "filters": {}, + }, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.json()["detail"], "Feature Flag key already exists. Please select a unique key") + + +class ClickhouseTestFunnelExperimentResults(ClickhouseTestMixin, LicensedTestMixin, APIBaseTest): + @snapshot_clickhouse_queries + def test_experiment_flow_with_event_results(self): + journeys_for( + { + "person1": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {"$feature/a-b-test": "test"},}, + {"event": "$pageleave", "timestamp": "2020-01-04", "properties": {"$feature/a-b-test": "test"},}, + ], + # doesn't have feature set + "person2": [ + {"event": "$pageview", "timestamp": "2020-01-03", "properties": {"$feature/a-b-test": "control"}}, + {"event": "$pageleave", "timestamp": "2020-01-05", "properties": {"$feature/a-b-test": "control"}}, + ], + "person3": [ + {"event": "$pageview", "timestamp": "2020-01-04", "properties": {"$feature/a-b-test": "control"}}, + {"event": "$pageleave", "timestamp": "2020-01-05", "properties": {"$feature/a-b-test": "control"}}, + ], + "person_out_of_control": [ + {"event": "$pageview", "timestamp": "2020-01-03",}, + {"event": "$pageleave", "timestamp": "2020-01-05",}, + ], + "person_out_of_end_date": [ + {"event": "$pageview", "timestamp": "2020-08-03", "properties": {"$feature/a-b-test": "control"}}, + {"event": "$pageleave", "timestamp": "2020-08-05", "properties": {"$feature/a-b-test": "control"}}, + ], + # non-converters with FF + "person4": [ + {"event": "$pageview", "timestamp": "2020-01-03", "properties": {"$feature/a-b-test": "test"},}, + ], + }, + self.team, + ) + + ff_key = "a-b-test" + # generates the FF which should result in the above events^ + response = self.client.post( + f"/api/projects/{self.team.id}/experiments/", + { + "name": "Test Experiment", + "description": "", + "start_date": "2020-01-01T00:00", + "end_date": "2020-01-06T00:00", + "feature_flag_key": ff_key, + "parameters": None, + "filters": { + "events": [{"order": 0, "id": "$pageview"}, {"order": 1, "id": "$pageleave"}], + "properties": [ + {"key": "$geoip_country_name", "type": "person", "value": ["france"], "operator": "exact"} + ], + }, + }, + ) + + id = response.json()["id"] + + response = self.client.get(f"/api/projects/{self.team.id}/experiments/{id}/results") + self.assertEqual(200, response.status_code) + + response_data = response.json() + result = response_data["funnel"] + + self.assertEqual(result[0][0]["name"], "$pageview") + self.assertEqual(result[0][0]["count"], 2) + self.assertEqual("test", result[0][0]["breakdown_value"][0]) + + self.assertEqual(result[0][1]["name"], "$pageleave") + self.assertEqual(result[0][1]["count"], 1) + self.assertEqual("test", result[0][1]["breakdown_value"][0]) + + self.assertEqual(result[1][0]["name"], "$pageview") + self.assertEqual(result[1][0]["count"], 2) + self.assertEqual("control", result[1][0]["breakdown_value"][0]) + + self.assertEqual(result[1][1]["name"], "$pageleave") + self.assertEqual(result[1][1]["count"], 2) + self.assertEqual("control", result[1][1]["breakdown_value"][0]) + + # Variant with True: Beta(2, 3) and empty: Beta(3, 1) distribution + # probability tells the variant has low probability of being better. + self.assertTrue(response_data["probability"] < 0.5) diff --git a/latest_migrations.manifest b/latest_migrations.manifest index ef5f26815f0c2..98e35464dc4de 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -3,7 +3,7 @@ auth: 0012_alter_user_first_name_max_length axes: 0006_remove_accesslog_trusted contenttypes: 0002_remove_content_type_name ee: 0005_project_based_permissioning -posthog: 0189_alter_annotation_scope +posthog: 0190_experiment rest_hooks: 0002_swappable_hook_model sessions: 0001_initial social_django: 0010_uid_db_index diff --git a/posthog/api/__init__.py b/posthog/api/__init__.py index d15a28898a020..b90ad13a94b9a 100644 --- a/posthog/api/__init__.py +++ b/posthog/api/__init__.py @@ -100,6 +100,7 @@ def api_not_found(request): from ee.clickhouse.views.cohort import ClickhouseCohortViewSet, LegacyClickhouseCohortViewSet from ee.clickhouse.views.element import ClickhouseElementViewSet, LegacyClickhouseElementViewSet from ee.clickhouse.views.events import ClickhouseEventsViewSet, LegacyClickhouseEventsViewSet + from ee.clickhouse.views.experiments import ClickhouseExperimentsViewSet from ee.clickhouse.views.groups import ClickhouseGroupsTypesView, ClickhouseGroupsView from ee.clickhouse.views.insights import ClickhouseInsightsViewSet, LegacyClickhouseInsightsViewSet from ee.clickhouse.views.paths import ClickhousePathsViewSet, LegacyClickhousePathsViewSet @@ -124,6 +125,7 @@ def api_not_found(request): projects_router.register(r"paths", ClickhousePathsViewSet, "project_paths", ["team_id"]) projects_router.register(r"elements", ClickhouseElementViewSet, "project_elements", ["team_id"]) projects_router.register(r"cohorts", ClickhouseCohortViewSet, "project_cohorts", ["team_id"]) + projects_router.register(r"experiments", ClickhouseExperimentsViewSet, "project_experiments", ["team_id"]) projects_router.register( r"session_recordings", ClickhouseSessionRecordingViewSet, "project_session_recordings", ["team_id"], ) diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index eefbab5631ff1..79bafc27c8e5e 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -150,7 +150,7 @@ class FeatureFlagViewSet(StructuredViewSetMixin, AnalyticsDestroyModelMixin, vie def get_queryset(self) -> QuerySet: queryset = super().get_queryset() if self.action == "list": - queryset = queryset.filter(deleted=False) + queryset = queryset.filter(deleted=False, experiment__isnull=True) return queryset.order_by("-created_at") @action(methods=["GET"], detail=False) diff --git a/posthog/migrations/0190_experiment.py b/posthog/migrations/0190_experiment.py new file mode 100644 index 0000000000000..cec03a5e882d5 --- /dev/null +++ b/posthog/migrations/0190_experiment.py @@ -0,0 +1,39 @@ +# Generated by Django 3.2.5 on 2021-12-09 10:11 + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("posthog", "0189_alter_annotation_scope"), + ] + + operations = [ + migrations.CreateModel( + name="Experiment", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(max_length=400)), + ("description", models.CharField(blank=True, max_length=400, null=True)), + ("filters", models.JSONField(default=dict)), + ("parameters", models.JSONField(default=dict, null=True)), + ("start_date", models.DateTimeField(null=True)), + ("end_date", models.DateTimeField(null=True)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("updated_at", models.DateTimeField(auto_now=True)), + ( + "created_by", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + ( + "feature_flag", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="posthog.featureflag"), + ), + ("team", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="posthog.team")), + ], + ), + ] diff --git a/posthog/models/__init__.py b/posthog/models/__init__.py index b93e642ed974d..d1f35faadd32c 100644 --- a/posthog/models/__init__.py +++ b/posthog/models/__init__.py @@ -8,6 +8,7 @@ from .entity import Entity from .event import Event from .event_definition import EventDefinition +from .experiment import Experiment from .feature_flag import FeatureFlag from .filters import Filter, RetentionFilter from .group import Group @@ -37,6 +38,7 @@ "Entity", "Event", "EventDefinition", + "Experiment", "FeatureFlag", "Filter", "Group", diff --git a/posthog/models/experiment.py b/posthog/models/experiment.py new file mode 100644 index 0000000000000..3e579bfa0bde9 --- /dev/null +++ b/posthog/models/experiment.py @@ -0,0 +1,28 @@ +from django.db import models +from django.utils import timezone + + +class Experiment(models.Model): + name: models.CharField = models.CharField(max_length=400) + description: models.CharField = models.CharField(max_length=400, null=True, blank=True) + team: models.ForeignKey = models.ForeignKey("Team", on_delete=models.CASCADE) + + # Filters define the target metric of an Experiment + filters: models.JSONField = models.JSONField(default=dict) + + # Parameters include configuration fields for the experiment: What the control & test variant are called, + # and any test significance calculation parameters + parameters: models.JSONField = models.JSONField(default=dict, null=True) + feature_flag: models.ForeignKey = models.ForeignKey("FeatureFlag", blank=False, on_delete=models.CASCADE) + created_by: models.ForeignKey = models.ForeignKey("User", on_delete=models.CASCADE) + start_date: models.DateTimeField = models.DateTimeField(null=True) + end_date: models.DateTimeField = models.DateTimeField(null=True) + created_at: models.DateTimeField = models.DateTimeField(default=timezone.now) + updated_at: models.DateTimeField = models.DateTimeField(auto_now=True) + + def get_feature_flag_key(self): + return self.feature_flag.key + + @property + def is_draft(self): + return not self.start_date diff --git a/requirements.in b/requirements.in index ae637ac49b4df..e84e3797a4c7e 100644 --- a/requirements.in +++ b/requirements.in @@ -51,3 +51,4 @@ social-auth-core==4.1.0 statshog==1.0.6 toronado==0.0.11 whitenoise==5.2.0 +numpy==1.21.4 diff --git a/requirements.txt b/requirements.txt index 591e7987c45fb..f29939bf92669 100644 --- a/requirements.txt +++ b/requirements.txt @@ -132,6 +132,8 @@ lzstring==1.0.4 # via -r requirements.in monotonic==1.5 # via posthoganalytics +numpy==1.21.4 + # via -r requirements.in oauthlib==3.1.0 # via # requests-oauthlib