Allow local packages in requirements.txt dependency list.

CHANGES.md

@@ -50,6 +50,15 @@
 * ([#X](https://github.com/apache/beam/issues/X)).
 -->
 
+# [2.44.0] - Unreleased
+
+## New Features / Improvements
+
+* Local packages can now be used as dependencies in the requirements.txt file, rather
+  than requiring them to be passed separately via the `--extra_package` option.
+  ([#23684](https://github.com/apache/beam/pull/23684))
+
+
 # [2.43.0] - Unreleased
 
 ## Highlights

sdks/python/apache_beam/runners/portability/stager.py

@@ -224,9 +224,14 @@ def create_job_resources(options,  # type: PipelineOptions
               'The file %s cannot be found. It was specified in the '
               '--requirements_file command line option.' %
               setup_options.requirements_file)
+        extra_packages, thinned_requirements_file = (
+            Stager._extract_local_packages(setup_options.requirements_file))
+        if extra_packages:
+          setup_options.extra_packages = (
+              setup_options.extra_packages or []) + extra_packages
         resources.append(
             Stager._create_file_stage_to_artifact(
-                setup_options.requirements_file, REQUIREMENTS_FILE))
+                thinned_requirements_file, REQUIREMENTS_FILE))
         # Populate cache with packages from the requirement file option and
         # stage the files in the cache.
         if not use_beam_default_container:
@@ -683,6 +688,25 @@ def _remove_dependency_from_requirements(
 
     return tmp_requirements_filename
 
+  @staticmethod
+  def _extract_local_packages(requirements_file):
+    local_deps = []
+    pypi_deps = []
+    with open(requirements_file, 'r') as fin:
+      for line in fin:
+        dep = line.strip()
+        if os.path.exists(dep):
+          local_deps.append(dep)
+        else:
+          pypi_deps.append(dep)
+    if local_deps:
+      with tempfile.NamedTemporaryFile(suffix='-requirements.txt',
+                                       delete=False) as fout:
+        fout.write('\n'.join(pypi_deps).encode('utf-8'))
+        return local_deps, fout.name
+    else:
+      return [], requirements_file
+
   @staticmethod
   def _get_platform_for_default_sdk_container():
     """

website/www/site/content/en/documentation/sdks/python-pipeline-dependencies.md

@@ -36,15 +36,14 @@ If your pipeline uses public packages from the [Python Package Index](https://py
 
     This command creates a `requirements.txt` file that lists all packages that are installed on your machine, regardless of where they were installed from.
 
-2. Edit the `requirements.txt` file and leave only the packages that were installed from PyPI and are used in the workflow source. Delete all packages that are not relevant to your code.
+2. Edit the `requirements.txt` file and delete all packages that are not relevant to your code.
 
 3. Run your pipeline with the following command-line option:
 
         --requirements_file requirements.txt
 
     The runner will use the `requirements.txt` file to install your additional dependencies onto the remote workers.
 
-**Important:** Remote workers will install all packages listed in the `requirements.txt` file. Because of this, it's very important that you delete non-PyPI packages from the `requirements.txt` file, as stated in step 2. If you don't remove non-PyPI packages, the remote workers will fail when attempting to install packages from sources that are unknown to them.
 > **NOTE**: An alternative to `pip freeze` is to use a library like [pip-tools](https://github.com/jazzband/pip-tools) to compile all the dependencies required for the pipeline from a `--requirements_file`, where only top-level dependencies are mentioned.
 
 ## Custom Containers {#custom-containers}