Feature/257 trending communities n posts rehaul

CHANGELOG.md

@@ -4,10 +4,17 @@ The change log for the API server for Okuna.
 
 ## Table of contents
 
+- [Release 0.0.66](#release-0.0.66)
 - [Release 0.0.63](#release-0.0.63)
 - [Release 0.0.59](#release-0.0.59)
 
 
+## Release 0.0.66
+
+- Introduce activity score in posts and communities, add jobs for the same
+- Refactor trending posts and trending communities to use activity score
+
+
 ## Release 0.0.63
 
 - Improve performance for linked users API

openbook/settings.py

@@ -19,6 +19,7 @@
 from dotenv import load_dotenv, find_dotenv
 from sentry_sdk.integrations.django import DjangoIntegration
 from django_replicated.settings import *
+from decimal import Decimal, getcontext
 
 # Logging config
 from sentry_sdk.integrations.rq import RqIntegration
@@ -177,6 +178,8 @@
 REDIS_RQ_DEFAULT_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 1}
 REDIS_RQ_HIGH_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 2}
 REDIS_RQ_LOW_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {'redis_location': REDIS_LOCATION, 'db': 3}
+REDIS_ACTIVITY_SCORES_JOBS_CACHE_LOCATION = '%(redis_location)s/%(db)d' % {
+    'redis_location': REDIS_LOCATION, 'db': 4}
 
 CACHES = {
     'default': {
@@ -211,6 +214,14 @@
         },
         "KEY_PREFIX": "ob-api-rq-low-job-"
     },
+    'activity-score-jobs': {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_ACTIVITY_SCORES_JOBS_CACHE_LOCATION,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient"
+        },
+        "KEY_PREFIX": "ob-api-actvty-score-job-"
+    },
 }
 
 CACHEOPS_REDIS_DB = int(os.environ.get('CACHEOPS_REDIS_DB', '1'))
@@ -236,6 +247,9 @@
     'low': {
         'USE_REDIS_CACHE': 'rq-low-jobs',
     },
+    'process-activity-score': {
+        'USE_REDIS_CACHE': 'activity-score-jobs',
+    },
 }
 
 if IS_BUILD:
@@ -527,7 +541,26 @@
 
 MIN_UNIQUE_TOP_POST_REACTIONS_COUNT = int(os.environ.get('MIN_UNIQUE_TOP_POST_REACTIONS_COUNT', '5'))
 MIN_UNIQUE_TOP_POST_COMMENTS_COUNT = int(os.environ.get('MIN_UNIQUE_TOP_POST_COMMENTS_COUNT', '5'))
-MIN_UNIQUE_TRENDING_POST_REACTIONS_COUNT = int(os.environ.get('MIN_UNIQUE_TRENDING_POST_REACTIONS_COUNT', '5'))
+
+# for activity score, set decimal precision to 10
+getcontext().prec = 10
+
+MIN_ACTIVITY_SCORE_FOR_POST_TRENDING = Decimal((os.environ.get('MIN_ACTIVITY_SCORE_FOR_POST_TRENDING', 0.002)))
+MIN_ACTIVITY_SCORE_FOR_COMMUNITY_TRENDING = Decimal((os.environ.get('MIN_ACTIVITY_SCORE_FOR_COMMUNITY_TRENDING', 0.002)))
+ACTIVITY_ATOMIC_WEIGHT = Decimal((os.environ.get('ACTIVITY_ATOMIC_WEIGHT', 0.001)))
+
+ACTIVITY_UNIQUE_REACTION_MULTIPLIER = int(os.environ.get('ACTIVITY_UNIQUE_REACTION_MULTIPLIER', 1))
+ACTIVITY_UNIQUE_COMMENT_MULTIPLIER = int(os.environ.get('ACTIVITY_UNIQUE_COMMENT_MULTIPLIER', 1))
+ACTIVITY_COUNT_COMMENTS_MULTIPLIER = int(os.environ.get('ACTIVITY_COUNT_COMMENTS_MULTIPLIER', 1))
+ACTIVITY_UNIQUE_POST_MULTIPLIER = int(os.environ.get('ACTIVITY_UNIQUE_POST_MULTIPLIER', 1))
+ACTIVITY_COUNT_POSTS_MULTIPLIER = int(os.environ.get('ACTIVITY_COUNT_POSTS_MULTIPLIER', 1))
+
+ACTIVITY_UNIQUE_REACTION_WEIGHT = Decimal((ACTIVITY_ATOMIC_WEIGHT * ACTIVITY_UNIQUE_REACTION_MULTIPLIER))
+ACTIVITY_UNIQUE_COMMENT_WEIGHT = Decimal((ACTIVITY_ATOMIC_WEIGHT * ACTIVITY_UNIQUE_COMMENT_MULTIPLIER))
+ACTIVITY_COUNT_COMMENTS_WEIGHT = Decimal((ACTIVITY_ATOMIC_WEIGHT * ACTIVITY_COUNT_COMMENTS_MULTIPLIER))
+ACTIVITY_UNIQUE_POST_WEIGHT = Decimal((ACTIVITY_ATOMIC_WEIGHT * ACTIVITY_UNIQUE_POST_MULTIPLIER))
+ACTIVITY_COUNT_POSTS_WEIGHT = Decimal((ACTIVITY_ATOMIC_WEIGHT * ACTIVITY_COUNT_POSTS_MULTIPLIER))
+ACTIVITY_SCORE_EXPIRY_IN_HOURS = int(os.environ.get('ACTIVITY_SCORE_EXPIRY_IN_HOURS', 12))
 
 # Email Config
 
@@ -550,7 +583,7 @@
     OS_TRANSLATION_STRATEGY_NAME = 'testing'
     MIN_UNIQUE_TOP_POST_REACTIONS_COUNT = 1
     MIN_UNIQUE_TOP_POST_COMMENTS_COUNT = 1
-    MIN_UNIQUE_TRENDING_POST_REACTIONS_COUNT = 1
+    MIN_ACTIVITY_SCORE_FOR_POST_TRENDING = 0.001
 
 if IS_PRODUCTION:
     AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')

openbook/urls.py

@@ -84,7 +84,7 @@
 from openbook_posts.views.post_media.views import PostMedia
 from openbook_posts.views.post_reaction.views import PostReactionItem
 from openbook_posts.views.post_reactions.views import PostReactions, PostReactionsEmojiCount, PostReactionEmojiGroups
-from openbook_posts.views.posts.views import Posts, TrendingPosts, TopPosts, TrendingPostsNew, \
+from openbook_posts.views.posts.views import Posts, TrendingPosts, TopPosts, TrendingPostsLegacy, \
     ProfilePostsExcludedCommunities, SearchProfilePostsExcludedCommunities, TopPostsExcludedCommunities, \
     SearchTopPostsExcludedCommunities, ProfilePostsExcludedCommunity, TopPostsExcludedCommunity
 from openbook_importer.views import ImportItem
@@ -238,7 +238,7 @@
     path('<uuid:post_uuid>/', include(post_patterns)),
     path('', Posts.as_view(), name='posts'),
     path('trending/', TrendingPosts.as_view(), name='trending-posts'),
-    path('trending/new/', TrendingPostsNew.as_view(), name='trending-posts-new'),
+    path('trending/new/', TrendingPostsLegacy.as_view(), name='trending-posts-new'),
     path('emojis/groups/', PostReactionEmojiGroups.as_view(), name='posts-emoji-groups'),
     path('profile/', include(posts_profile_patterns)),
     path('top/', include(posts_top_patterns)),

openbook_auth/models.py

@@ -1987,13 +1987,18 @@ def get_trending_posts(self, max_id=None, min_id=None):
         Post = get_post_model()
         return Post.get_trending_posts_for_user_with_id(user_id=self.pk, max_id=max_id, min_id=min_id)
 
-    def get_trending_posts_old(self):
+    def get_trending_posts_legacy(self):
         Post = get_post_model()
-        return Post.get_trending_posts_old_for_user_with_id(user_id=self.pk)
+        return Post.get_trending_posts_for_user_with_id_legacy(user_id=self.pk)
 
     def get_trending_communities(self, category_name=None):
         Community = get_community_model()
-        return Community.get_trending_communities_for_user_with_id(user_id=self.pk, category_name=category_name)
+        return Community.get_trending_communities_for_user_with_id(user_id=self.pk,
+                                                                   category_name=category_name)
+
+    def get_trending_communities_by_members(self, category_name=None):
+        Community = get_community_model()
+        return Community.get_trending_communities_by_members_for_user_with_id(user_id=self.pk, category_name=category_name)
 
     def search_communities_with_query(self, query, excluded_from_profile_posts):
         Community = get_community_model()

openbook_auth/tests/views/test_authenticated_user.py

@@ -826,8 +826,6 @@ def test_cannot_set_invalid_language(self):
             'language_id': 99999
         }, **headers)
 
-        print(response)
-
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         user.refresh_from_db()
         self.assertTrue(user.language.id, language.id)

openbook_common/tests/models.py

@@ -1,6 +1,6 @@
 from unittest.mock import patch
 
-from rest_framework.test import APITestCase
+from rest_framework.test import APITestCase, APITransactionTestCase
 
 
 class OpenbookAPITestCase(APITestCase):
@@ -10,3 +10,12 @@ def setUp(self):
 
     def tearDown(self):
         self.patcher.stop()
+
+
+class OpenbookAPITransactionTestCase(APITransactionTestCase):
+    def setUp(self):
+        self.patcher = patch('openbook_notifications.helpers._send_notification_to_user')
+        self.mock_foo = self.patcher.start()
+
+    def tearDown(self):
+        self.patcher.stop()

openbook_communities/migrations/0034_trendingcommunity.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.2.5 on 2020-02-03 14:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbook_communities', '0033_auto_20191209_1337'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TrendingCommunity',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(db_index=True, editable=False)),
+                ('community', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='trending_community', to='openbook_communities.Community')),
+            ],
+        ),
+    ]

openbook_communities/migrations/0035_community_activity_score.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.5 on 2020-02-10 12:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbook_communities', '0034_trendingcommunity'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='community',
+            name='activity_score',
+            field=models.FloatField(default=0.0),
+        ),
+    ]

openbook_communities/migrations/0036_delete_trendingcommunity.py

@@ -0,0 +1,16 @@
+# Generated by Django 2.2.5 on 2020-02-22 11:49
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbook_communities', '0035_community_activity_score'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='TrendingCommunity',
+        ),
+    ]

openbook_communities/migrations/0037_auto_20200222_1344.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.5 on 2020-02-22 12:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbook_communities', '0036_delete_trendingcommunity'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='community',
+            index=models.Index(fields=['activity_score'], name='openbook_co_activit_07d4ba_idx'),
+        ),
+    ]

openbook_communities/migrations/0038_auto_20200224_1615.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.5 on 2020-02-24 15:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('openbook_communities', '0037_auto_20200222_1344'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='community',
+            name='activity_score',
+            field=models.DecimalField(decimal_places=10, default=0.0, max_digits=10),
+        ),
+    ]

openbook_communities/models.py

@@ -66,9 +66,13 @@ class Community(models.Model):
         _('is deleted'),
         default=False,
     )
+    activity_score = models.DecimalField(default=0.0, decimal_places=10, max_digits=10)
 
     class Meta:
         verbose_name_plural = 'communities'
+        indexes = [
+            models.Index(fields=['activity_score']),
+        ]
 
     @classmethod
     def is_user_with_username_invited_to_community_with_name(cls, username, community_name):
@@ -147,7 +151,12 @@ def get_new_user_suggested_communities(cls):
     def get_trending_communities_for_user_with_id(cls, user_id, category_name=None):
         trending_communities_query = cls._make_trending_communities_query(category_name=category_name)
         trending_communities_query.add(~Q(banned_users__id=user_id), Q.AND)
-        return cls._get_trending_communities_with_query(query=trending_communities_query)
+
+        trending_communities = cls._get_trending_communities_with_query(query=trending_communities_query)
+        if trending_communities.count() == 0:
+            return cls.get_trending_communities_by_members_for_user_with_id(user_id, category_name=category_name)
+
+        return trending_communities
 
     @classmethod
     def get_trending_communities(cls, category_name=None):
@@ -156,11 +165,37 @@ def get_trending_communities(cls, category_name=None):
 
     @classmethod
     def _get_trending_communities_with_query(cls, query):
+        return cls.objects.filter(query).order_by('-activity_score')
+
+    @classmethod
+    def _make_trending_communities_query(cls, category_name=None):
+        trending_communities_query = Q(type=Community.COMMUNITY_TYPE_PUBLIC, is_deleted=False)
+        trending_communities_query.add(Q(activity_score__gte=settings.MIN_ACTIVITY_SCORE_FOR_COMMUNITY_TRENDING), Q.AND)
+        trending_communities_query.add(~Q(moderated_object__status=ModeratedObject.STATUS_APPROVED), Q.AND)
+
+        if category_name:
+            trending_communities_query.add(Q(categories__name=category_name), Q.AND)
+
+        return trending_communities_query
+
+    @classmethod
+    def get_trending_communities_by_members_for_user_with_id(cls, user_id, category_name=None):
+        trending_communities_query = cls._make_trending_communities_by_members_query(category_name=category_name)
+        trending_communities_query.add(~Q(banned_users__id=user_id), Q.AND)
+        return cls._get_trending_communities_by_members_with_query(query=trending_communities_query)
+
+    @classmethod
+    def get_trending_communities_by_members(cls, category_name=None):
+        trending_communities_query = cls._make_trending_communities_by_members_query(category_name=category_name)
+        return cls._get_trending_communities_by_members_with_query(query=trending_communities_query)
+
+    @classmethod
+    def _get_trending_communities_by_members_with_query(cls, query):
         return cls.objects.annotate(Count('memberships')).filter(query).order_by(
             '-memberships__count', '-created')
 
     @classmethod
-    def _make_trending_communities_query(cls, category_name=None):
+    def _make_trending_communities_by_members_query(cls, category_name=None):
         trending_communities_query = Q(type=cls.COMMUNITY_TYPE_PUBLIC, is_deleted=False)
 
         if category_name: