Shaka Streamer Binaries (#87)

By default, Shaka Streamer will now expect you to install a secondary package (`shaka-streamer-binaries`) containing FFmpeg and Shaka Packager binaries.  There is a command line flag (`--use-system-binaries`) to use system-installed binaries instead.

Related documentation is forthcoming.

Issue #60

.gitignore

@@ -8,8 +8,11 @@ docs/build/
 build/
 dist/
 shaka_streamer.egg-info/
+shaka_streamer_binaries.egg-info/
 .idea/
 venv/
 dev/
 .vscode/
-packager.exe
+ffmpeg*
+ffprobe*
+packager*

binaries/build_wheels.py

@@ -0,0 +1,117 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A script that downloads ffmpeg, ffprobe, and packager static builds for all
+the platforms we build for and then builds distribution wheels for them.
+"""
+
+import os
+import shutil
+import subprocess
+import urllib.request
+
+import streamer_binaries
+
+
+# Version constants.
+# Change to download different versions.
+FFMPEG_VERSION = 'n4.4-1'
+PACKAGER_VERSION = 'v2.6.0'
+
+# A map of suffixes that will be combined with the binary download links
+# to achieve a full download link.  Different suffix for each platform.
+# Extend this dictionary to add more platforms.
+PLATFORM_SUFFIXES = {
+    # 64-bit Windows
+    'win_amd64': '-win-x64.exe',
+    # 64-bit Linux
+    'manylinux1_x86_64': '-linux-x64',
+    # Linux on ARM
+    'manylinux2014_aarch64': '-linux-arm64',
+    # 64-bit with 10.9 SDK
+    'macosx_10_9_x86_64': '-osx-x64',
+}
+
+FFMPEG_DL_PREFIX = 'https://github.com/joeyparrish/static-ffmpeg-binaries/releases/download/' + FFMPEG_VERSION
+PACKAGER_DL_PREFIX = 'https://github.com/google/shaka-packager/releases/download/' + PACKAGER_VERSION
+
+# The download links to each binary.  These download links
+# aren't complete, they miss the platfrom-specific suffix.
+BINARIES_DL = [
+    FFMPEG_DL_PREFIX + '/ffmpeg',
+    FFMPEG_DL_PREFIX + '/ffprobe',
+    PACKAGER_DL_PREFIX + '/packager',
+]
+
+
+def build_bdist_wheel(platform_name, platform_binaries):
+  """Builds a wheel distribution for `platform_name` adding the files
+  in `platform_binaries` to it using the `package_data` parameter."""
+
+  args = [
+      'python3', 'setup.py',
+      # Build binary as a wheel.
+      'bdist_wheel',
+      # Platform name to embed in generated filenames.
+      '--plat-name', platform_name,
+      # Temporary directory for creating the distribution.
+      '--bdist-dir', platform_name,
+      # Python tag to embed in the generated filenames.
+      '--python-tag', 'py3',
+      # Run quietly.
+      '--quiet',
+  ]
+  # After '--', we send the platform specific binaries that we want to include.
+  args += ['--']
+  args += platform_binaries
+  subprocess.check_call(args)
+  # Remove the build directory so that it is not reused by 'setup.py'.
+  shutil.rmtree('build')
+
+def download_binary(download_url: str, download_dir: str) -> str:
+  """Downloads a file and writes it to the file system.
+  Returns the file name.
+  """
+
+  binary_name = download_url.split('/')[-1]
+  binary_path = os.path.join(download_dir, binary_name)
+  print('downloading', binary_name, flush=True, end=' ')
+  urllib.request.urlretrieve(download_url, binary_path)
+  print('(finished)')
+  # Set executable permissions for the downloaded binaries.
+  default_permissions = 0o755
+  os.chmod(binary_path, default_permissions)
+  return binary_name
+
+
+def main():
+  # For each platform(OS+CPU), we download the its binaries and
+  # create a binary wheel distribution that contains the executable
+  # binaries specific to this platform.
+  for platform_name, suffix in PLATFORM_SUFFIXES.items():
+    binaries_to_include = []
+    # Use the `suffix` specific to this platfrom to achieve
+    # the full download link for each binary.
+    for binary_dl in BINARIES_DL:
+      download_link = binary_dl + suffix
+      binary_name = download_binary(download_url=download_link,
+                                    download_dir=streamer_binaries.__name__)
+      binaries_to_include.append(binary_name)
+    # Build a wheel distribution for this platform
+    # and include the binaries we have just downloaded.
+    build_bdist_wheel(platform_name, binaries_to_include)
+
+
+if  __name__ == '__main__':
+  main()

binaries/setup.py

@@ -0,0 +1,45 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import sys
+import setuptools # type: ignore
+
+import streamer_binaries
+
+separator_index = sys.argv.index('--')
+platform_binaries = sys.argv[separator_index + 1:]
+sys.argv = sys.argv[:separator_index]
+
+setuptools.setup(
+  name='shaka-streamer-binaries',
+  version=streamer_binaries.__version__,
+  author='Google',
+  description='A package containing FFmpeg, FFprobe, and Shaka Packager static builds.',
+  long_description=('An auxiliary package that provides platform-specific'
+                    ' binaries used by Shaka Streamer.'),
+  url='https://github.com/google/shaka-streamer/tree/master/binaries',
+  packages=[streamer_binaries.__name__,],
+  classifiers=[
+      'Programming Language :: Python :: 3',
+      'License :: OSI Approved :: Apache Software License',
+      'Operating System :: POSIX :: Linux',
+      'Operating System :: MacOS :: MacOS X',
+      'Operating System :: Microsoft :: Windows',
+  ],
+  package_data={
+      # Only add the corresponding platform specific binaries to the wheel.
+      streamer_binaries.__name__: platform_binaries,
+  }
+)

binaries/streamer_binaries/__init__.py

@@ -0,0 +1,50 @@
+import os
+
+__version__ = '0.5.0'
+
+
+# Module level variables.
+ffmpeg = ''
+"""The path to the installed FFmpeg binary."""
+ffprobe = ''
+"""The path to the installed FFprobe binary."""
+packager = ''
+"""The path to the installed Shaka Packager binary."""
+
+
+# Get the directory path where this __init__.py file resides.
+_dir_path = os.path.abspath(os.path.dirname(__file__))
+
+
+def _change_permissions_if_needed(file):
+  """This function will try to change the bundled executables permssions
+  as needed.  This is useful at build time, so to give the needed permissions
+  to all the binaries before packaging them into wheels.  It is also useful
+  at runtime to ensure that the executables we offer can be executed
+  as a subprocess.
+  """
+
+  executable_by_owner = 0o100
+  perms = os.stat(file).st_mode
+  # If it already has executable permssions, we don't chmod.
+  # As chmod may require root permssions.
+  if (perms | executable_by_owner) == perms:
+    return
+  # Else we will change the permissions to 0o755.
+  # Readable and executable by all + full permissions to owner.
+  default_permissions = 0o755 # rwxr-xr-x
+  # This might raise PermissionError.
+  os.chmod(file, default_permissions)
+
+
+# This will be executed at import time.
+for _file in os.listdir(_dir_path):
+  if _file.startswith('ffmpeg'):
+    ffmpeg = os.path.join(_dir_path, _file)
+    _change_permissions_if_needed(ffmpeg)
+  elif _file.startswith('ffprobe'):
+    ffprobe = os.path.join(_dir_path, _file)
+    _change_permissions_if_needed(ffprobe)
+  elif _file.startswith('packager'):
+    packager = os.path.join(_dir_path, _file)
+    _change_permissions_if_needed(packager)

run_end_to_end_tests.py

@@ -49,6 +49,7 @@
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 os.chdir(BASE_DIR)
 controller = None
+test_hermetics = False
 
 # Flask was unable to autofind the root_path correctly after an os.chdir() from another directory
 # Dunno why,refer to https://stackoverflow.com/questions/35864584/error-no-such-file-or-directory-when-using-os-chdir-in-flask
@@ -169,7 +170,8 @@ def start():
                      configs['input_config'],
                      configs['pipeline_config'],
                      configs['bitrate_config'],
-                     check_deps=False)
+                     check_deps=False,
+                     use_hermetic=test_hermetics)
   except Exception as e:
     # If the controller throws an exception during startup, we want to call
     # stop() to shut down any external processes that have already been started.
@@ -276,8 +278,14 @@ def main():
                       help='Number of trials to run')
   parser.add_argument('--reporters', nargs='+',
                       help='Enables specified reporters in karma')
+  parser.add_argument('--test-hermetics',
+                      action='store_true',
+                      help='Runs the tests using binaries from `shaka-streamer-binaries`')
   args = parser.parse_args()
 
+  global test_hermetics
+  test_hermetics = args.test_hermetics
+
   # Do static type checking on the project first.
   type_check_result = mypy_api.run(['streamer/'])
   if type_check_result[2] != 0:

shaka-streamer

@@ -49,18 +49,18 @@ def main():
   parser = argparse.ArgumentParser(description=description,
                                    formatter_class=CustomArgParseFormatter)
 
-  parser.add_argument('-i', '--input_config',
+  parser.add_argument('-i', '--input-config',
                       required=True,
                       help='The path to the input config file (required).')
-  parser.add_argument('-p', '--pipeline_config',
+  parser.add_argument('-p', '--pipeline-config',
                       required=True,
                       help='The path to the pipeline config file (required).')
-  parser.add_argument('-b', '--bitrate_config',
+  parser.add_argument('-b', '--bitrate-config',
                       help='The path to a config file which defines custom ' +
                            'bitrates and resolutions for transcoding. ' +
                            '(optional, see example in ' +
                            'config_files/bitrate_config.yaml)')
-  parser.add_argument('-c', '--cloud_url',
+  parser.add_argument('-c', '--cloud-url',
                       default=None,
                       help='The Google Cloud Storage or Amazon S3 URL to ' +
                            'upload to.  (Starts with gs:// or s3://)')
@@ -69,12 +69,16 @@ def main():
                       help='The output folder to write files to, or an HTTP ' +
                            'or HTTPS URL where files will be PUT.' +
                            'Used even if uploading to cloud storage.')
-  parser.add_argument('--skip_deps_check',
+  parser.add_argument('--skip-deps-check',
                       action='store_true',
-                      default=False,
                       help='Skip checks for dependencies and their versions. ' +
                            'This can be useful for testing pre-release ' +
                            'versions of FFmpeg or Shaka Packager.')
+  parser.add_argument('--use-system-binaries',
+                      action='store_true',
+                      help='Use FFmpeg, FFprobe and Shaka Packager binaries ' +
+                           'found in PATH instead of the ones offered by ' +
+                           'Shaka Streamer.')
 
   args = parser.parse_args()
 
@@ -99,7 +103,8 @@ def main():
   try:
     with controller.start(args.output, input_config_dict, pipeline_config_dict,
                           bitrate_config_dict, args.cloud_url,
-                          not args.skip_deps_check):
+                          not args.skip_deps_check,
+                          not args.use_system_binaries):
       # Sleep so long as the pipeline is still running.
       while True:
         status = controller.check_status()

streamer/autodetect.py

@@ -28,6 +28,9 @@
   InputType.EXTERNAL_COMMAND,
 ]
 
+# This module level variable might be set by the controller node
+# if the user chooses to use the shaka streamer bundled binaries.
+hermetic_ffprobe: Optional[str] = None
 
 def _probe(input: Input, field: str) -> Optional[str]:
   """Autodetect some feature of the input, if possible, using ffprobe.
@@ -46,7 +49,8 @@ def _probe(input: Input, field: str) -> Optional[str]:
 
   args: List[str] = [
       # Probe this input file
-      'ffprobe', input.name,
+      hermetic_ffprobe or 'ffprobe',
+      input.name,
   ]
 
   # Add any required input arguments for this input type