Merge pull request #65 from NicolasConstant/topic_add-last-sync

reply filtering PR
NicolasConstant · Jan 23, 2021 · 32df020 · 32df020
2 parents fee7810 + ffdd041
commit 32df020
Show file tree

Hide file tree

Showing 11 changed files with 923 additions and 18 deletions.
diff --git a/VARIABLES.md b/VARIABLES.md
@@ -10,4 +10,5 @@ You can configure some of BirdsiteLIVE's settings via environment variables (tho
 ## Instance customization 
 
 * `Instance:Name` (default: BirdsiteLIVE) the name of the instance
-* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
+* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it.
+* `Instance:PublishReplies` (default: false) to enable or disable replies publishing.
diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
@@ -6,5 +6,6 @@ public class InstanceSettings
         public string Domain { get; set; }
         public string AdminEmail { get; set; }
         public bool ResolveMentionsInProfiles { get; set; }
+        public bool PublishReplies { get; set; }
     }
 }
diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs
@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
@@ -21,15 +22,17 @@ public class SendTweetsToInboxTask : ISendTweetsToInboxTask
         private readonly IActivityPubService _activityPubService;
         private readonly IStatusService _statusService;
         private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _settings;
         private readonly ILogger<SendTweetsToInboxTask> _logger;
 
 
         #region Ctor
-        public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, ILogger<SendTweetsToInboxTask> logger)
+        public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToInboxTask> logger)
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
             _followersDal = followersDal;
+            _settings = settings;
             _logger = logger;
         }
         #endregion
@@ -52,8 +55,13 @@ public async Task ExecuteAsync(IEnumerable<ExtractedTweet> tweets, Follower foll
                 {
                     try
                     {
-                        var note = _statusService.GetStatus(user.Acct, tweet);
-                        await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+                        if (!tweet.IsReply ||
+                            tweet.IsReply && tweet.IsThread ||
+                            _settings.PublishReplies)
+                        {
+                            var note = _statusService.GetStatus(user.Acct, tweet);
+                            await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox);
+                        }
                     }
                     catch (ArgumentException e)
                     {

diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs
@@ -2,6 +2,7 @@
 using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
@@ -20,14 +21,16 @@ public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask
         private readonly IStatusService _statusService;
         private readonly IActivityPubService _activityPubService;
         private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _settings;
         private readonly ILogger<SendTweetsToSharedInboxTask> _logger;
 
         #region Ctor
-        public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, ILogger<SendTweetsToSharedInboxTask> logger)
+        public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger<SendTweetsToSharedInboxTask> logger)
         {
             _activityPubService = activityPubService;
             _statusService = statusService;
             _followersDal = followersDal;
+            _settings = settings;
             _logger = logger;
         }
         #endregion
@@ -52,8 +55,13 @@ public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, st
                 {
                     try
                     {
-                        var note = _statusService.GetStatus(user.Acct, tweet);
-                        await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
+                        if (!tweet.IsReply ||
+                            tweet.IsReply && tweet.IsThread ||
+                            _settings.PublishReplies)
+                        {
+                            var note = _statusService.GetStatus(user.Acct, tweet);
+                            await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
+                        }
                     }
                     catch (ArgumentException e)
                     {
@@ -66,7 +74,7 @@ public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, st
                             throw;
                         }
                     }
-                    
+
                     syncStatus = tweet.Id;
                 }
             }

diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs
@@ -24,7 +24,9 @@ public ExtractedTweet Extract(ITweet tweet)
                 InReplyToAccount = tweet.InReplyToScreenName,
                 MessageContent = ExtractMessage(tweet),
                 Media = ExtractMedia(tweet.Media),
-                CreatedAt = tweet.CreatedAt.ToUniversalTime()
+                CreatedAt = tweet.CreatedAt.ToUniversalTime(),
+                IsReply = tweet.InReplyToUserId != null,
+                IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id
             };
             return extractedTweet;
         }

diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs
@@ -11,5 +11,7 @@ public class ExtractedTweet
         public ExtractedMedia[] Media { get; set; }
         public DateTime CreatedAt { get; set; }
         public string InReplyToAccount { get; set; }
+        public bool IsReply { get; set; }
+        public bool IsThread { get; set; }
     }
 }
diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs
@@ -40,7 +40,7 @@ public ExtractedTweet GetTweet(long statusId)
             var tweet = Tweet.GetTweet(statusId);
             _statisticsHandler.CalledTweetApi();
             if (tweet == null) return null; //TODO: test this
-            return _tweetExtractor.Extract(tweet);
+             return _tweetExtractor.Extract(tweet);
         }
 
         public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1)

diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json
@@ -13,7 +13,8 @@
     "Name":  "BirdsiteLIVE",
     "Domain": "domain.name",
     "AdminEmail": "[email protected]",
-    "ResolveMentionsInProfiles": true
+    "ResolveMentionsInProfiles": true,
+    "PublishReplies": false 
   },
   "Db": {
     "Type": "postgres",

diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs
@@ -76,7 +76,7 @@ public async Task GetTwitterUsersAsync_Multi_Test()
             processor.WaitFactor = 2;
             processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
-            await Task.Delay(200);
+            await Task.Delay(300);
 
             #region Validations
             twitterUserDalMock.VerifyAll();