From ea9f1f2af8bf3d0d58b9d8fb88ae62903e643c1d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 26 Aug 2021 09:14:28 -0700 Subject: [PATCH] Temporarily revert "Multi period for DASH (#78)" This (manually) reverts the non-refactoring parts of commit e6152fdfe175dcf97a733cc5765ce95fcdaa7d26. For the v0.4.0 release, we will remove the multi-period DASH feature. It will then be restored and released together with multi-period HLS in v0.5.0. Change-Id: I450ca7b78d93f781d85a9561a6b747e4389d0126 --- CHANGELOG.md | 3 - config_files/input_multiperiod.yaml | 104 --------------------- streamer/controller_node.py | 35 +------ streamer/input_configuration.py | 36 +------- streamer/periodconcat_node.py | 138 ---------------------------- tests/tests.js | 79 ---------------- 6 files changed, 5 insertions(+), 390 deletions(-) delete mode 100644 config_files/input_multiperiod.yaml delete mode 100644 streamer/periodconcat_node.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 34e179a..5fac874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,9 +38,6 @@ - Restrict WebM formats to DASH, omit from HLS (https://github.com/google/shaka-streamer/issues/18) (https://github.com/google/shaka-streamer/pull/80) - - Multi period support for DASH - (https://github.com/google/shaka-streamer/issues/43) - (https://github.com/google/shaka-streamer/pull/78) - Automatic frame rate reduction (https://github.com/google/shaka-streamer/pull/77) - Fix missing members in docs, auto-link to types in config docs diff --git a/config_files/input_multiperiod.yaml b/config_files/input_multiperiod.yaml deleted file mode 100644 index 6e703db..0000000 --- a/config_files/input_multiperiod.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is a sample input configuration file for Shaka Streamer for multi-period stream. - -multiperiod_inputs_list: - - # List of inputs, this will be the first period in the final multi-period manifest. - - inputs: - # Name of the input file. - # This example can be downloaded from https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.4k.mkv - - name: Sintel.2010.4k.mkv - # The media type of the input. Can be audio or video. - media_type: video - - # A second track (audio) from the same input file. - - name: Sintel.2010.4k.mkv - media_type: audio - - # Several text tracks of different languages. - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Arabic.vtt - - name: Sintel.2010.Arabic.vtt - media_type: text - language: ar - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.English.vtt - - name: Sintel.2010.English.vtt - media_type: text - language: en - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Esperanto.vtt - - name: Sintel.2010.Esperanto.vtt - media_type: text - language: eo - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Spanish.vtt - - name: Sintel.2010.Spanish.vtt - media_type: text - language: es - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.French.vtt - - name: Sintel.2010.French.vtt - media_type: text - language: fr - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Chinese.vtt - - name: Sintel.2010.Chinese.vtt - media_type: text - language: zh - - # List of inputs, this will be the second period in the final multi-period manifest. - - inputs: - # Name of the input file. - # This example can be downloaded from https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.4k.mkv - - name: Sintel.2010.4k.mkv - # The media type of the input. Can be audio or video. - media_type: video - - # A second track (audio) from the same input file. - - name: Sintel.2010.4k.mkv - media_type: audio - - # Several text tracks of different languages. - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Arabic.vtt - - name: Sintel.2010.Arabic.vtt - media_type: text - language: ar - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.English.vtt - - name: Sintel.2010.English.vtt - media_type: text - language: en - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Esperanto.vtt - - name: Sintel.2010.Esperanto.vtt - media_type: text - language: eo - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Spanish.vtt - - name: Sintel.2010.Spanish.vtt - media_type: text - language: es - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.French.vtt - - name: Sintel.2010.French.vtt - media_type: text - language: fr - - # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Chinese.vtt - - name: Sintel.2010.Chinese.vtt - media_type: text - language: zh - diff --git a/streamer/controller_node.py b/streamer/controller_node.py index 005128d..429bb9f 100644 --- a/streamer/controller_node.py +++ b/streamer/controller_node.py @@ -38,7 +38,6 @@ from streamer.packager_node import PackagerNode from streamer.pipeline_configuration import PipelineConfig, StreamingMode from streamer.transcoder_node import TranscoderNode -from streamer.periodconcat_node import PeriodConcatNode import streamer.subprocessWindowsPatch # side-effects only from streamer.util import is_url from streamer.pipe import Pipe @@ -137,32 +136,13 @@ def start(self, output_location: str, if bucket_url: raise RuntimeError( 'Cloud bucket upload is incompatible with HTTP PUT support.') - - if self._input_config.multiperiod_inputs_list: - # TODO: Edit Multiperiod input list implementation to support HTTP outputs - raise RuntimeError( - 'Multiperiod input list support is incompatible with HTTP outputs.') # Note that we remove the trailing slash from the output location, because # otherwise GCS would create a subdirectory whose name is "". output_location = output_location.rstrip('/') - if self._input_config.inputs: - # InputConfig contains inputs only. - self._append_nodes_for_inputs_list(self._input_config.inputs, output_location) - else: - # InputConfig contains multiperiod_inputs_list only. - # Create one Transcoder node and one Packager node for each period. - for i, singleperiod in enumerate(self._input_config.multiperiod_inputs_list): - sub_dir_name = 'period_' + str(i) - self._append_nodes_for_inputs_list(singleperiod.inputs, output_location, sub_dir_name) - - if self._pipeline_config.streaming_mode == StreamingMode.VOD: - packager_nodes = [node for node in self._nodes if isinstance(node, PackagerNode)] - self._nodes.append(PeriodConcatNode( - self._pipeline_config, - packager_nodes, - output_location)) + # InputConfig contains inputs only. + self._append_nodes_for_inputs_list(self._input_config.inputs, output_location) if bucket_url: cloud_temp_dir = os.path.join(self._temp_dir, 'cloud') @@ -180,13 +160,12 @@ def start(self, output_location: str, return self - def _append_nodes_for_inputs_list(self, inputs: List[Input], output_location: str, - period_dir: Optional[str] = None) -> None: + def _append_nodes_for_inputs_list(self, inputs: List[Input], + output_location: str) -> None: """A common method that creates Transcoder and Packager nodes for a list of Inputs passed to it. Args: inputs (List[Input]): A list of Input streams. - period_dir (Optional[str]): A subdirectory name where a single period will be outputted to. If passed, this indicates that inputs argument is one period in a list of periods. """ @@ -251,12 +230,6 @@ def _append_nodes_for_inputs_list(self, inputs: List[Input], output_location: st self._pipeline_config, outputs)) - # If the inputs list was a period in multiperiod_inputs_list, create a nested directory - # and put that period in it. - if period_dir: - output_location = os.path.join(output_location, period_dir) - os.mkdir(output_location) - self._nodes.append(PackagerNode(self._pipeline_config, output_location, outputs)) diff --git a/streamer/input_configuration.py b/streamer/input_configuration.py index b41a05e..7a58786 100644 --- a/streamer/input_configuration.py +++ b/streamer/input_configuration.py @@ -313,42 +313,8 @@ def get_resolution(self) -> bitrate_configuration.VideoResolution: def get_channel_layout(self) -> bitrate_configuration.AudioChannelLayout: return bitrate_configuration.AudioChannelLayout.get_value(self.channel_layout) -class SinglePeriod(configuration.Base): - """An object repersenting one optional video stream and multiple audio and text streams""" - - inputs = configuration.Field(List[Input], required=True).cast() - class InputConfig(configuration.Base): """An object representing the entire input config to Shaka Streamer.""" - multiperiod_inputs_list = configuration.Field(List[SinglePeriod]).cast() - """A list of SinglePeriod objects""" - - inputs = configuration.Field(List[Input]).cast() + inputs = configuration.Field(List[Input], required=True).cast() """A list of Input objects""" - - def __init__(self, dictionary: Dict[str, Any]): - """A constructor to check that either inputs or mutliperiod_inputs_list is provided, - and produce a helpful error message in case both or none are provided. - - We need these checks before passing the input dictionary to the configuration.Base constructor, - because it does not check for this 'exclusive or-ing' relationship between fields. - """ - - assert isinstance(dictionary, dict), """Malformed Input Config File, - See some examples at https://github.com/google/shaka-streamer/tree/master/config_files. - """ - - if (dictionary.get('inputs') is not None - and dictionary.get('multiperiod_inputs_list') is not None): - raise configuration.ConflictingFields( - InputConfig, 'inputs', 'multiperiod_inputs_list') - - # Because these fields are not marked as required at the class level - # , we need to check ourselves that one of them is provided. - if not dictionary.get('inputs') and not dictionary.get('multiperiod_inputs_list'): - raise configuration.MissingRequiredExclusiveFields( - InputConfig, 'inputs', 'multiperiod_inputs_list') - - super().__init__(dictionary) - diff --git a/streamer/periodconcat_node.py b/streamer/periodconcat_node.py deleted file mode 100644 index 884f615..0000000 --- a/streamer/periodconcat_node.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Concatenates inputs into periods by creating a master DASH/HLS file.""" - -import os -import re -from typing import List -from xml.etree import ElementTree -from streamer import __version__ -from streamer.node_base import ProcessStatus, ThreadedNodeBase -from streamer.packager_node import PackagerNode -from streamer.pipeline_configuration import PipelineConfig, ManifestFormat, StreamingMode - - -class PeriodConcatNode(ThreadedNodeBase): - """A node that concatenates multiple DASH manifests and/or HLS playlists - when the input is a multiperiod_inputs_list and the output is to the local the system. - """ - - def __init__(self, - pipeline_config: PipelineConfig, - packager_nodes: List[PackagerNode], - output_location: str) -> None: - """Stores all relevant information needed for the period concatenation.""" - super().__init__(thread_name='periodconcat', continue_on_exception=False, sleep_time=3) - self._pipeline_config = pipeline_config - self._output_location = output_location - self._packager_nodes: List[PackagerNode] = packager_nodes - - def _thread_single_pass(self) -> None: - """Watches all the PackagerNode(s), if at least one of them is running it skips this - _thread_single_pass, if all of them are finished, it starts period concatenation, if one of - them is errored, it raises a RuntimeError. - """ - - for i, packager_node in enumerate(self._packager_nodes): - status = packager_node.check_status() - if status == ProcessStatus.Running: - return - elif status == ProcessStatus.Errored: - raise RuntimeError('Concatenation is stopped due to an error in PackagerNode#{}.'.format(i)) - - if ManifestFormat.DASH in self._pipeline_config.manifest_format: - self._dash_concat() - - if ManifestFormat.HLS in self._pipeline_config.manifest_format: - self._hls_concat() - - self._status = ProcessStatus.Finished - - def _dash_concat(self) -> None: - """Concatenates multiple single-period DASH manifests into one multi-period DASH manifest.""" - - def find(elem: ElementTree.Element, *args: str) -> ElementTree.Element: - """A better interface for the Element.find() method. - Use it only if it is guaranteed that the element we are searching for is inside, - Otherwise it will raise an AssertionError.""" - - full_path = '/'.join(['shaka-live:' + tag for tag in args]) - child_elem = elem.find(full_path, {'shaka-live': default_dash_namespace}) - - # elem.find() returns either an ElementTree.Element or None. - assert child_elem is not None, 'Unable to find: {} using the namespace: {}'.format( - full_path, default_dash_namespace) - return child_elem - - # Periods that are going to be collected from different MPD files. - periods: List[ElementTree.Element] = [] - - # Get the root of an MPD file that we will concatenate periods into. - concat_mpd = ElementTree.ElementTree(file=os.path.join( - self._packager_nodes[0].output_location, - self._pipeline_config.dash_output)).getroot() - - # Get the default namespace. - namespace_matches = re.search('\{([^}]*)\}', concat_mpd.tag) - assert namespace_matches is not None, 'Unable to find the default namespace.' - default_dash_namespace = namespace_matches.group(1) - - # Remove the 'mediaPresentationDuration' attribute. - concat_mpd.attrib.pop('mediaPresentationDuration') - # Remove the Period element in that MPD element. - concat_mpd.remove(find(concat_mpd, 'Period')) - - for packager_node in self._packager_nodes: - - mpd = ElementTree.ElementTree(file=os.path.join( - packager_node.output_location, - self._pipeline_config.dash_output)).getroot() - period = find(mpd, 'Period') - period.attrib['duration'] = mpd.attrib['mediaPresentationDuration'] - - # A BaseURL that will have the relative path to media file. - base_url = ElementTree.Element('BaseURL') - base_url.text = os.path.relpath(packager_node.output_location, self._output_location) + '/' - period.insert(0, base_url) - - periods.append(period) - - # Add the periods collected from all the files. - concat_mpd.extend(periods) - - # Write the period concat to the output_location. - with open(os.path.join( - self._output_location, - self._pipeline_config.dash_output), 'w') as master_dash: - - contents = "\n" - # TODO: Add Shaka-Packager version to this xml comment. - contents += "\n" - contents += "\n".format(__version__) - - # xml.ElementTree replaces the default namespace with 'ns0'. - # Register the DASH namespace back as the defualt namespace before converting to string. - ElementTree.register_namespace('', default_dash_namespace) - - # xml.etree.ElementTree already have an ElementTree().write() method, - # but it won't allow putting comments at the begining of the file. - contents += ElementTree.tostring(element=concat_mpd, encoding='unicode') - master_dash.write(contents) - - def _hls_concat(self) -> None: - """Concatenates multiple HLS playlists with #EXT-X-DISCONTINUITY.""" - - pass - \ No newline at end of file diff --git a/tests/tests.js b/tests/tests.js index e295309..c8459f5 100644 --- a/tests/tests.js +++ b/tests/tests.js @@ -145,8 +145,6 @@ describe('Shaka Streamer', () => { // TODO: Test is commented out until Packager outputs codecs for vtt in mp4. // muxedTextTests(hlsManifestUrl, '(hls)'); muxedTextTests(dashManifestUrl, '(dash)'); - - multiPeriodTests(dashManifestUrl, '(dash)'); }); function errorTests() { @@ -322,36 +320,6 @@ function errorTests() { })); }); - it('fails when both "inputs" and "multiperiod_inputs_list" are given', async() => { - const inputConfig = getBasicInputConfig(); - inputConfig.multiperiod_inputs_list = [ - getBasicInputConfig(), - getBasicInputConfig(), - ]; - const pipeline_config = { - streaming_mode: 'vod', - }; - - await expectAsync(startStreamer(inputConfig, pipeline_config)) - .toBeRejectedWith(jasmine.objectContaining({ - error_type: 'ConflictingFields', - field_name: 'inputs', - })); - }); - - it('fails when neither "inputs" nor "multiperiod_inputs_list" is given', async() => { - const inputConfig = {}; - const pipeline_config = { - streaming_mode: 'vod', - }; - - await expectAsync(startStreamer(inputConfig, pipeline_config)) - .toBeRejectedWith(jasmine.objectContaining({ - error_type: 'MissingRequiredExclusiveFields', - field_name: 'inputs', - })); - }); - it('fails when segment_per_file is false with a HTTP url output', async () => { const inputConfig = getBasicInputConfig(); const pipelineConfig = { @@ -365,19 +333,6 @@ function errorTests() { error_type: 'RuntimeError', })); }); - - it('fails when multiperiod_inputs_list is used with a HTTP url output', async () => { - const inputConfig = { - 'multiperiod_inputs_list': [ - getBasicInputConfig(), - ], - }; - - await expectAsync(startStreamer(inputConfig, minimalPipelineConfig, {}, outputHttpUrl)) - .toBeRejectedWith(jasmine.objectContaining({ - error_type: 'RuntimeError', - })); - }); } function resolutionTests(manifestUrl, format) { @@ -1363,37 +1318,3 @@ function muxedTextTests(manifestUrl, format) { ]); }); } - -function multiPeriodTests(manifestUrl, format) { - it('can process multiperiod_inputs_list ' + format, async() => { - const singleInputConfigDict = { - 'inputs': [ - { - 'name': TEST_DIR + 'Sintel.with.subs.mkv', - 'media_type': 'video', - // Keep this test short by only encoding 1s of content. - 'end_time': '0:01', - }, - ], - }; - const inputConfigDict = { - 'multiperiod_inputs_list': [ - singleInputConfigDict, - singleInputConfigDict, - ], - }; - const pipelineConfigDict = { - 'streaming_mode': 'vod', - 'resolutions': ['144p'], - 'audio_codecs': ['aac'], - 'video_codecs': ['h264'], - }; - - await startStreamer(inputConfigDict, pipelineConfigDict); - await player.load(manifestUrl); - - // Since we processed only 0:01s, the total duration shoud be 2s. - // Be more tolerant with float comparison, (D > 1.9 * length) instead of (D == 2 * length). - expect(video.duration).toBeGreaterThan(1.9); - }); -} \ No newline at end of file