Add issue #411 Option to download/skip Twitter replies

TumblThreeApp · Jan 23, 2023 · 8f1d727 · 8f1d727
1 parent 2a1e3f9
commit 8f1d727
Show file tree

Hide file tree

Showing 10 changed files with 77 additions and 12 deletions.
diff --git a/src/TumblThree/SharedAssemblyInfo.cs b/src/TumblThree/SharedAssemblyInfo.cs
@@ -12,5 +12,5 @@
 
 [assembly: ComVisible(false)]
 [assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)]
-[assembly: AssemblyVersion("2.9.3.0")]
-[assembly: AssemblyFileVersion("2.9.3.0")]
+[assembly: AssemblyVersion("2.10.0.0")]
+[assembly: AssemblyFileVersion("2.10.0.0")]
diff --git a/src/TumblThree/TumblThree.Applications/Crawler/TwitterCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/TwitterCrawler.cs
@@ -54,6 +54,9 @@ public class TwitterCrawler : AbstractCrawler, ICrawler, IDisposable
         private string cursor;
         private string oldestApiPost;
 
+        // instead of new field reuse DownloadAnswer for replies
+        private bool BlogDownloadReplies => Blog.DownloadAnswer;
+
         public TwitterCrawler(IShellService shellService, ICrawlerService crawlerService, IProgress<DownloadProgress> progress, IWebRequestFactory webRequestFactory,
             ISharedCookieService cookieService, IPostQueue<AbstractPost> postQueue, IPostQueue<CrawlerData<Tweet>> jsonQueue, IBlog blog, IDownloader downloader,
             ICrawlerDataDownloader crawlerDataDownloader, PauseToken pt, CancellationToken ct)
@@ -221,9 +224,10 @@ private async Task<string> GetApiUrl(string url, byte type, string cursor, int p
                 case 2:
                     if (!string.IsNullOrEmpty(cursor)) cursor = string.Format("&cursor={0}", cursor.Replace("+", "%2B"));    //HttpUtility.UrlEncode(cursor)
                     var restId = (await GetTwUser()).Data.User.RestId;
+                    var includeReplies = BlogDownloadReplies.ToString().ToLower();
                     url = string.Format("https://api.twitter.com/2/timeline/profile/{0}.json" +
-                        "?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&include_tweet_replies=true&userId={0}&count={1}{2}&ext=mediaStats%2ChighlightedLabel%2CcameraMoment",
-                        restId, pageSize, cursor);
+                        "?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cards_platform=Web-12&include_cards=1&include_ext_alt_text=true&include_quote_count=true&include_reply_count=1&tweet_mode=extended&include_entities=true&include_user_entities=true&include_ext_media_color=true&include_ext_media_availability=true&send_error_codes=true&simple_quoted_tweets=true&include_tweet_replies={3}&userId={0}&count={1}{2}&ext=mediaStats%2ChighlightedLabel%2CcameraMoment",
+                        restId, pageSize, cursor, includeReplies);
                     break;
                 case 3:
                     if (!string.IsNullOrEmpty(cursor)) cursor = string.Format("&cursor={0}", cursor.Replace("+", "%2B"));
@@ -720,6 +724,7 @@ private static List<Media> GetMedia(Tweet post)
         private void AddPhotoUrlToDownloadList(Tweet post)
         {
             if (!Blog.DownloadPhoto) return;
+            if (!BlogDownloadReplies && !string.IsNullOrEmpty(post.InReplyToStatusIdStr)) return;
 
             var media = GetMedia(post);
 
@@ -732,6 +737,7 @@ private void AddPhotoUrlToDownloadList(Tweet post)
         private void AddVideoUrlToDownloadList(Tweet post)
         {
             if (!Blog.DownloadVideo && !Blog.DownloadVideoThumbnail) return;
+            if (!BlogDownloadReplies && !string.IsNullOrEmpty(post.InReplyToStatusIdStr)) return;
 
             var media = GetMedia(post);
 
@@ -755,6 +761,7 @@ private void AddTextUrlToDownloadList(Tweet post)
         {
             if (!Blog.DownloadText) return;
             if (!(post.Entities == null || post.Entities.Media == null || post.Entities.Media.Count == 0)) return;
+            if (!BlogDownloadReplies && !string.IsNullOrEmpty(post.InReplyToStatusIdStr)) return;
 
             AddToDownloadList(new TextPost(post.FullText, post.IdStr));
             AddToJsonQueue(new CrawlerData<Tweet>(Path.ChangeExtension(post.IdStr, ".json"), TweetToSave(post)));
@@ -763,6 +770,7 @@ private void AddTextUrlToDownloadList(Tweet post)
         private void AddGifUrlToDownloadList(Tweet post)
         {
             if (!Blog.DownloadPhoto || Blog.SkipGif) return;
+            if (!BlogDownloadReplies && !string.IsNullOrEmpty(post.InReplyToStatusIdStr)) return;
 
             var media = GetMedia(post);
 

diff --git a/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TimelineTweets.cs b/src/TumblThree/TumblThree.Applications/DataModels/Twitter/TimelineTweets.cs
@@ -24,6 +24,9 @@ public class Tweet
         [JsonProperty("full_text")]
         public string FullText { get; set; }
 
+        [JsonProperty("truncated")]
+        public bool Truncated { get; set; }
+
         [JsonProperty("display_text_range")]
         public List<int> DisplayTextRange { get; set; }
 
@@ -36,12 +39,24 @@ public class Tweet
         [JsonProperty("source")]
         public string Source { get; set; }
 
+        [JsonProperty("in_reply_to_status_id_str")]
+        public string InReplyToStatusIdStr { get; set; }
+
+        [JsonProperty("in_reply_to_user_id_str")]
+        public string InReplyToUserIdStr { get; set; }
+
+        [JsonProperty("in_reply_to_screen_name")]
+        public string InReplyToScreenName { get; set; }
+
         [JsonProperty("user_id_str")]
         public string UserIdStr { get; set; }
 
         [JsonProperty("user")]
         public User User { get; set; }
 
+        [JsonProperty("is_quote_status")]
+        public bool IsQuoteStatus { get; set; }
+
         [JsonProperty("retweeted_status_id_str")]
         public string RetweetedStatusIdStr { get; set; }
 
@@ -57,15 +72,30 @@ public class Tweet
         [JsonProperty("quote_count")]
         public int QuoteCount { get; set; }
 
+        [JsonProperty("conversation_id")]
+        public long ConversationId { get; set; }
+
         [JsonProperty("conversation_id_str")]
         public string ConversationIdStr { get; set; }
 
+        [JsonProperty("favorited")]
+        public bool Favorited { get; set; }
+
+        [JsonProperty("retweeted")]
+        public bool Retweeted { get; set; }
+
+        [JsonProperty("possibly_sensitive")]
+        public bool PossiblySensitive { get; set; }
+
         [JsonProperty("possibly_sensitive_editable")]
         public bool PossiblySensitiveEditable { get; set; }
 
         [JsonProperty("lang")]
         public string Lang { get; set; }
 
+        [JsonProperty("supplemental_language")]
+        public string SupplementalLanguage { get; set; }
+
         public object Clone()
         {
             return MemberwiseClone();

diff --git a/src/TumblThree/TumblThree.Applications/Services/SettingsService.cs b/src/TumblThree/TumblThree.Applications/Services/SettingsService.cs
@@ -18,7 +18,7 @@ public IBlog TransferGlobalSettingsToBlog(IBlog blog)
             blog.DownloadPhoto = shellService.Settings.DownloadImages;
             blog.DownloadVideo = shellService.Settings.DownloadVideos;
             blog.DownloadText = shellService.Settings.DownloadTexts;
-            blog.DownloadAnswer = shellService.Settings.DownloadAnswers;
+            blog.DownloadAnswer = blog.BlogType == Domain.Models.BlogTypes.twitter ? false : shellService.Settings.DownloadAnswers;
             blog.DownloadQuote = shellService.Settings.DownloadQuotes;
             blog.DownloadConversation = shellService.Settings.DownloadConversations;
             blog.DownloadLink = shellService.Settings.DownloadLinks;

diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.Designer.cs b/src/TumblThree/TumblThree.Presentation/Properties/Resources.Designer.cs
diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.de.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.de.resx
@@ -967,4 +967,7 @@ Die zur Authentifizierung verwendeten Cookies werden gespeichert und der Authent
   <data name="ToolTipDownloadLikesTo" xml:space="preserve">
     <value>Lade Likes bis zu diesem Datum</value>
   </data>
+  <data name="DownloadReplies" xml:space="preserve">
+    <value>Lade Antworten</value>
+  </data>
 </root>
diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.es.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.es.resx
@@ -964,4 +964,7 @@ Las cookies utilizadas para la autenticación se guardan y el proceso de autenti
   <data name="ToolTipDownloadLikesTo" xml:space="preserve">
     <value>Rastrear los me gusta a esta fecha</value>
   </data>
+  <data name="DownloadReplies" xml:space="preserve">
+    <value>Descargar respuestas</value>
+  </data>
 </root>
diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.fr.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.fr.resx
@@ -968,4 +968,7 @@ Les cookies utilisés pour l'authentification sont enregistrés et le processus
   <data name="ToolTipDownloadLikesTo" xml:space="preserve">
     <value>Ranger les likes à cette date</value>
   </data>
+  <data name="DownloadReplies" xml:space="preserve">
+    <value>Télécharger les réponses</value>
+  </data>
 </root>
diff --git a/src/TumblThree/TumblThree.Presentation/Properties/Resources.resx b/src/TumblThree/TumblThree.Presentation/Properties/Resources.resx
@@ -1216,4 +1216,7 @@ The cookies for the privacy consent are saved and the process needs to be done o
   <data name="ToolTipDownloadLikesTo" xml:space="preserve">
     <value>Crawl likes to this date</value>
   </data>
+  <data name="DownloadReplies" xml:space="preserve">
+    <value>Download replies</value>
+  </data>
 </root>
diff --git a/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml b/src/TumblThree/TumblThree.Presentation/Views/DetailsViews/DetailsTwitterBlogView.xaml
@@ -203,6 +203,8 @@
                                 <RowDefinition Height="Auto" />
                                 <RowDefinition Height="{StaticResource Layout.RowSpace}" />
                                 <RowDefinition Height="Auto" />
+                                <RowDefinition Height="{StaticResource Layout.RowSpace}"/>
+                                <RowDefinition Height="Auto"/>
                                 <RowDefinition Height="{StaticResource Layout.RowSpace}" />
                                 <RowDefinition Height="Auto" />
                                 <RowDefinition Height="{StaticResource Layout.RowSpace}" />
@@ -547,8 +549,12 @@
                                   IsChecked="{Binding BlogFile.DownloadRebloggedPosts, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="2" Grid.Row="25"
                                   Margin="3,0,0,0" Padding="3,0,0,0" />
 
+                            <CheckBox Content="{x:Static p:Resources.DownloadReplies}"
+                                  IsChecked="{Binding BlogFile.DownloadAnswer, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="2" Grid.Row="27"
+                                  Margin="3,0,0,0" Padding="3,0,0,0" />
+
                             <CheckBox Content="{x:Static p:Resources.DownloadVideoThumbnails}" IsChecked="{Binding BlogFile.DownloadVideoThumbnail, Mode=TwoWay}"
-                                  Grid.Column="0" Grid.ColumnSpan="2" Grid.Row="27" Margin="3,0,0,0" Padding="3,0,0,0">
+                                  Grid.Column="0" Grid.ColumnSpan="2" Grid.Row="29" Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
                                         <StackPanel>
@@ -576,7 +582,7 @@
                         </CheckBox>-->
 
                             <CheckBox Content="{x:Static p:Resources.DownloadUrlList2}"
-                                  IsChecked="{Binding BlogFile.DownloadUrlList, Mode=TwoWay}" Grid.Column="0" Grid.Row="29" Grid.ColumnSpan="5"
+                                  IsChecked="{Binding BlogFile.DownloadUrlList, Mode=TwoWay}" Grid.Column="0" Grid.Row="31" Grid.ColumnSpan="5"
                                   Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
@@ -591,7 +597,7 @@
 
                             <CheckBox Content="{x:Static p:Resources.ForceSize}"
                                   IsChecked="{Binding BlogFile.ForceSize, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="5"
-                                  Grid.Row="31" Margin="3,0,0,0" Padding="3,0,0,0">
+                                  Grid.Row="33" Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
                                         <StackPanel>
@@ -605,7 +611,7 @@
 
                             <CheckBox Content="{x:Static p:Resources.RegExPhotos}"
                                       IsChecked="{Binding BlogFile.RegExPhotos, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="5"
-                                      Grid.Row="33" Margin="3,0,0,0" Padding="3,0,0,0">
+                                      Grid.Row="35" Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
                                         <StackPanel>
@@ -619,7 +625,7 @@
 
                             <CheckBox Content="{x:Static p:Resources.RegExVideos}"
                                       IsChecked="{Binding BlogFile.RegExVideos, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="5"
-                                      Grid.Row="35" Margin="3,0,0,0" Padding="3,0,0,0">
+                                      Grid.Row="37" Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
                                         <StackPanel>
@@ -633,7 +639,7 @@
 
                             <CheckBox Content="{x:Static p:Resources.DumpCrawlerData}"
                                   IsChecked="{Binding BlogFile.DumpCrawlerData, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="5"
-                                  Grid.Row="37" Margin="3,0,0,0" Padding="3,0,0,0">
+                                  Grid.Row="39" Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
                                         <StackPanel>
@@ -647,7 +653,7 @@
 
                             <CheckBox Content="{x:Static p:Resources.ForceRescan}"
                                   IsChecked="{Binding BlogFile.ForceRescan, Mode=TwoWay}" Grid.Column="0" Grid.ColumnSpan="5"
-                                  Grid.Row="39" Margin="3,0,0,0" Padding="3,0,0,0">
+                                  Grid.Row="41" Margin="3,0,0,0" Padding="3,0,0,0">
                                 <CheckBox.ToolTip>
                                     <ToolTip>
                                         <StackPanel>