From cab20770390b6562fee509b04ca5ba8196208292 Mon Sep 17 00:00:00 2001 From: Fay Smash <2380642-FaySmash@users.noreply.gitlab.com> Date: Fri, 19 Jul 2024 17:07:48 +0200 Subject: [PATCH] Add support for indexing Streams --- .../migrations/0025_add_video_type_support.py | 20 ++++++++ tubesync/sync/models.py | 46 +++++++++++++------ tubesync/sync/templates/sync/source.html | 8 ++++ tubesync/sync/views.py | 2 +- 4 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 tubesync/sync/migrations/0025_add_video_type_support.py diff --git a/tubesync/sync/migrations/0025_add_video_type_support.py b/tubesync/sync/migrations/0025_add_video_type_support.py new file mode 100644 index 0000000..e1a901c --- /dev/null +++ b/tubesync/sync/migrations/0025_add_video_type_support.py @@ -0,0 +1,20 @@ +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0024_auto_20240717_1535'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='index_videos', + field=models.BooleanField(default=True, help_text='Index video media from this source', verbose_name='index videos'), + ), + migrations.AddField( + model_name='source', + name='index_streams', + field=models.BooleanField(default=False, help_text='Index live stream media from this source', verbose_name='index streams'), + ), + ] \ No newline at end of file diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index fc27ba7..00d8484 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -165,8 +165,8 @@ class Source(models.Model): } # Format used to create indexable URLs INDEX_URLS = { - SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos', - SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos', + SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/{type}', + SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/{type}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', } # Callback functions to get a list of media from the source @@ -274,6 +274,16 @@ class IndexSchedule(models.IntegerChoices): default=True, help_text=_('Download media from this source, if not selected the source will only be indexed') ) + index_videos = models.BooleanField( + _('index videos'), + default=False, + help_text=_('Index video media from this source') + ) + index_streams = models.BooleanField( + _('index streams'), + default=False, + help_text=_('Index live stream media from this source') + ) download_cap = models.IntegerField( _('download cap'), choices=CapChoices.choices, @@ -475,17 +485,16 @@ def create_url(obj, source_type, key): return url.format(key=key) @classmethod - def create_index_url(obj, source_type, key): + def create_index_url(obj, source_type, key, type): url = obj.INDEX_URLS.get(source_type) - return url.format(key=key) + return url.format(key=key, type=type) @property def url(self): return Source.create_url(self.source_type, self.key) - @property - def index_url(self): - return Source.create_index_url(self.source_type, self.key) + def get_index_url(self, type): + return Source.create_index_url(self.source_type, self.key, type) @property def format_summary(self): @@ -590,23 +599,32 @@ def is_regex_match(self, media_item_title): return True return bool(re.search(self.filter_text, media_item_title)) - def index_media(self): - ''' - Index the media source returning a list of media metadata as dicts. - ''' + def get_index(self, type): indexer = self.INDEXERS.get(self.source_type, None) if not callable(indexer): raise Exception(f'Source type f"{self.source_type}" has no indexer') - response = indexer(self.index_url) + response = indexer(self.get_index_url(type=type)) if not isinstance(response, dict): return [] - entries = response.get('entries', []) + entries = response.get('entries', []) + return entries + + def index_media(self): + ''' + Index the media source returning a list of media metadata as dicts. + ''' + entries = list() + if self.index_videos: + entries += self.get_index('videos') + # Playlists do something different that I have yet to figure out + if self.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: + if self.index_streams: + entries += self.get_index('streams') if settings.MAX_ENTRIES_PROCESSING: entries = entries[:settings.MAX_ENTRIES_PROCESSING] return entries - def get_media_thumb_path(instance, filename): fileid = str(instance.uuid) filename = f'{fileid.lower()}.jpg' diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 379cef8..19c195f 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -73,6 +73,14 @@

Source {{ source.name }}

Index schedule Index schedule
{{ source.get_index_schedule_display }} + + Index videos? + Index videos?
{% if source.index_videos %}{% else %}{% endif %} + + + Index streams? + Index streams?
{% if source.index_streams %}{% else %}{% endif %} + Download media? Download media?
{% if source.download_media %}{% else %}{% endif %} diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 8b60a07..94e9143 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -297,7 +297,7 @@ def get_success_url(self): class EditSourceMixin: model = Source fields = ('source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min', - 'media_format', 'index_schedule', 'download_media', 'download_cap', 'delete_old_media', + 'media_format', 'index_schedule', 'index_videos', 'index_streams', 'download_media', 'download_cap', 'delete_old_media', 'delete_removed_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'copy_channel_images', 'delete_removed_media', 'delete_files_on_disk', 'days_to_keep', 'source_resolution',