Allow repeating an option that takes multiple values

Many command line arguments take multiple values. Previously, to utilize
this feature, all values must be specified after a single option flag
(e.g. --exclude-where 'region=A' 'region=B'). If options were set using
separate option flags (e.g. --exclude-where 'region=A' --exclude-where
'region=B'), only the last flag would be used, which is unintuitive.
This commit enables all option flags to be used.

Done across the codebase by adding action='extend' to all options that
use nargs='+' and did not already specify an action.

A side effect of action='extend' is that it extends instead of replacing
a default value defined in add_argument. Allow users to continue
overriding the default value by first defaulting to an empty list
constant, then setting the intended default value later during argument
validation.

augur/align.py

@@ -24,7 +24,7 @@ def register_arguments(parser):
     Kept as a separate function than `register_parser` to continue to support
     unit tests that use this function to create argparser.
     """
-    parser.add_argument('--sequences', '-s', required=True, nargs="+", metavar="FASTA", help="sequences to align")
+    parser.add_argument('--sequences', '-s', required=True, nargs="+", action="extend", metavar="FASTA", help="sequences to align")
     parser.add_argument('--output', '-o', default="alignment.fasta", help="output file (default: %(default)s)")
     parser.add_argument('--nthreads', type=nthreads_value, default=1,
                                 help="number of threads to use; specifying the value 'auto' will cause the number of available CPU cores on your system, if determinable, to be used")

augur/ancestral.py

@@ -315,7 +315,7 @@ def register_parser(parent_subparsers):
     )
     amino_acid_options_group.add_argument('--annotation',
                         help='GenBank or GFF file containing the annotation')
-    amino_acid_options_group.add_argument('--genes', nargs='+', help="genes to translate (list or file containing list)")
+    amino_acid_options_group.add_argument('--genes', nargs='+', action='extend', help="genes to translate (list or file containing list)")
     amino_acid_options_group.add_argument('--translations', type=str, help="translated alignments for each CDS/Gene. "
                            "Currently only supported for FASTA-input. Specify the file name via a "
                            "template like 'aa_sequences_%%GENE.fasta' where %%GENE will be replaced "

augur/argparse_.py

@@ -2,6 +2,7 @@
 Custom helpers for the argparse standard library.
 """
 from argparse import Action, ArgumentDefaultsHelpFormatter
+from typing import Any, List
 
 
 # Include this in an argument help string to suppress the automatic appending
@@ -18,6 +19,14 @@
 SKIP_AUTO_DEFAULT_IN_HELP = "%(default).0s"
 
 
+# We use nargs='+' with action='extend' for a more intuitive user experience,
+# but it has the side effect of extending instead of replacing a default value
+# defined in add_argument. Allow users to override the default value by first
+# defaulting to this constant, then setting the intended default value at a
+# later point in the code.
+DEFER_LIST_DEFAULT: List[Any] = []
+
+
 def add_default_command(parser):
     """
     Sets the default command to run when none is provided.

augur/clades.py

@@ -341,8 +341,8 @@ def parse_nodes(tree_file, node_data_files):
 def register_parser(parent_subparsers):
     parser = parent_subparsers.add_parser("clades", help=__doc__)
     parser.add_argument('--tree', required=True, help="prebuilt Newick -- no tree will be built if provided")
-    parser.add_argument('--mutations', required=True, metavar="NODE_DATA_JSON", nargs='+', help='JSON(s) containing ancestral and tip nucleotide and/or amino-acid mutations ')
-    parser.add_argument('--reference', nargs='+', help=SUPPRESS)
+    parser.add_argument('--mutations', required=True, metavar="NODE_DATA_JSON", nargs='+', action='extend', help='JSON(s) containing ancestral and tip nucleotide and/or amino-acid mutations ')
+    parser.add_argument('--reference', nargs='+', action='extend', help=SUPPRESS)
     parser.add_argument('--clades', required=True, metavar="TSV", type=str, help='TSV file containing clade definitions by amino-acid')
     parser.add_argument('--output-node-data', type=str,  metavar="NODE_DATA_JSON", help='name of JSON file to save clade assignments to')
     parser.add_argument('--membership-name', type=str, default="clade_membership", help='Key to store clade membership under; use "None" to not export this')

augur/curate/__init__.py

@@ -6,7 +6,7 @@
 from collections import deque
 from textwrap import dedent
 
-from augur.argparse_ import add_command_subparsers
+from augur.argparse_ import DEFER_LIST_DEFAULT, add_command_subparsers
 from augur.errors import AugurError
 from augur.io.json import dump_ndjson, load_ndjson
 from augur.io.metadata import DEFAULT_DELIMITERS, InvalidDelimiter, read_table_to_dict, read_metadata_with_sequences, write_records_to_tsv
@@ -53,7 +53,7 @@ def create_shared_parser():
         help="Name of the metadata column that contains the record identifier for reporting duplicate records. "
              "Uses the first column of the metadata file if not provided. "
              "Ignored if also providing a FASTA file input.")
-    shared_inputs.add_argument("--metadata-delimiters", default=DEFAULT_DELIMITERS, nargs="+",
+    shared_inputs.add_argument("--metadata-delimiters", nargs="+", action="extend",
         help="Delimiters to accept when reading a metadata file. Only one delimiter will be inferred.")
 
     shared_inputs.add_argument("--fasta",
@@ -138,6 +138,9 @@ def run(args):
     if args.metadata == '-':
         args.metadata = sys.stdin
 
+    if args.metadata_delimiters == DEFER_LIST_DEFAULT:
+        args.metadata_delimiters = DEFAULT_DELIMITERS
+
     if args.metadata and args.fasta:
         try:
             records = read_metadata_with_sequences(

augur/curate/format_dates.py

@@ -18,9 +18,9 @@ def register_parser(parent_subparsers):
         help=__doc__)
 
     required = parser.add_argument_group(title="REQUIRED")
-    required.add_argument("--date-fields", nargs="+",
+    required.add_argument("--date-fields", nargs="+", action="extend",
         help="List of date field names in the record that need to be standardized.")
-    required.add_argument("--expected-date-formats", nargs="+",
+    required.add_argument("--expected-date-formats", nargs="+", action="extend",
         help="Expected date formats that are currently in the provided date fields, " +
              "defined by standard format codes as listed at " +
              "https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes. " +

augur/distance.py

@@ -660,11 +660,11 @@ def get_distances_to_all_pairs(tree, sequences_by_node_and_gene, distance_map, e
 def register_parser(parent_subparsers):
     parser = parent_subparsers.add_parser("distance", help=first_line(__doc__))
     parser.add_argument("--tree", help="Newick tree", required=True)
-    parser.add_argument("--alignment", nargs="+", help="sequence(s) to be used, supplied as FASTA files", required=True)
-    parser.add_argument('--gene-names', nargs="+", type=str, help="names of the sequences in the alignment, same order assumed", required=True)
-    parser.add_argument("--attribute-name", nargs="+", help="name to store distances associated with the given distance map; multiple attribute names are linked to corresponding positional comparison method and distance map arguments", required=True)
-    parser.add_argument("--compare-to", nargs="+", choices=["root", "ancestor", "pairwise"], help="type of comparison between samples in the given tree including comparison of all nodes to the root (root), all tips to their last ancestor from a previous season (ancestor), or all tips from the current season to all tips in previous seasons (pairwise)", required=True)
-    parser.add_argument("--map", nargs="+", help="JSON providing the distance map between sites and, optionally, sequences present at those sites; the distance map JSON minimally requires a 'default' field defining a default numeric distance and a 'map' field defining a dictionary of genes and one-based coordinates", required=True)
+    parser.add_argument("--alignment", nargs="+", action="extend", help="sequence(s) to be used, supplied as FASTA files", required=True)
+    parser.add_argument('--gene-names', nargs="+", action="extend", type=str, help="names of the sequences in the alignment, same order assumed", required=True)
+    parser.add_argument("--attribute-name", nargs="+", action="extend", help="name to store distances associated with the given distance map; multiple attribute names are linked to corresponding positional comparison method and distance map arguments", required=True)
+    parser.add_argument("--compare-to", nargs="+", action="extend", choices=["root", "ancestor", "pairwise"], help="type of comparison between samples in the given tree including comparison of all nodes to the root (root), all tips to their last ancestor from a previous season (ancestor), or all tips from the current season to all tips in previous seasons (pairwise)", required=True)
+    parser.add_argument("--map", nargs="+", action="extend", help="JSON providing the distance map between sites and, optionally, sequences present at those sites; the distance map JSON minimally requires a 'default' field defining a default numeric distance and a 'map' field defining a dictionary of genes and one-based coordinates", required=True)
     parser.add_argument("--date-annotations", help="JSON of branch lengths and date annotations from augur refine for samples in the given tree; required for comparisons to earliest or latest date")
     parser.add_argument("--earliest-date", help="earliest date at which samples are considered to be from previous seasons (e.g., 2019-01-01). This date is only used in pairwise comparisons. If omitted, all samples prior to the latest date will be considered.")
     parser.add_argument("--latest-date", help="latest date at which samples are considered to be from previous seasons (e.g., 2019-01-01); samples from any date after this are considered part of the current season")

augur/export_v1.py

@@ -9,6 +9,7 @@
 from Bio import Phylo
 from argparse import SUPPRESS
 from collections import defaultdict
+from .argparse_ import DEFER_LIST_DEFAULT
 from .errors import AugurError
 from .io.metadata import DEFAULT_DELIMITERS, InvalidDelimiter, read_metadata
 from .utils import read_node_data, write_json, read_config, read_lat_longs, read_colors
@@ -312,7 +313,7 @@ def add_core_args(parser):
     core = parser.add_argument_group("REQUIRED")
     core.add_argument('--tree','-t', required=True, help="tree to perform trait reconstruction on")
     core.add_argument('--metadata', required=True, metavar="FILE", help="sequence metadata")
-    core.add_argument('--metadata-delimiters', default=DEFAULT_DELIMITERS, nargs="+",
+    core.add_argument('--metadata-delimiters', default=DEFER_LIST_DEFAULT, nargs="+", action="extend",
                       help="delimiters to accept when reading a metadata file. Only one delimiter will be inferred.")
     core.add_argument('--node-data', required=True, nargs='+', action="extend", help="JSON files with meta data for each node")
     core.add_argument('--output-tree', help="JSON file name that is passed on to auspice (e.g., zika_tree.json).")
@@ -341,7 +342,16 @@ def register_parser(parent_subparsers):
     return parser
 
 
+def validate_arguments(args):
+    if args.metadata_delimiters == DEFER_LIST_DEFAULT:
+        args.metadata_delimiters = DEFAULT_DELIMITERS
+
+    return args
+
+
 def run(args):
+    args = validate_arguments(args)
+
     print(dedent("""\
         DEPRECATION WARNING: augur export v1 is no longer maintained and will be removed in a future release.
         Read more on how to migrate to export v2: <https://docs.nextstrain.org/projects/augur/en/stable/releases/migrating-v5-v6.html>"""),file=sys.stderr)

augur/export_v2.py

@@ -11,6 +11,7 @@
 import re
 from Bio import Phylo
 
+from augur.argparse_ import DEFER_LIST_DEFAULT
 from .errors import AugurError
 from .io.file import open_file
 from .io.metadata import DEFAULT_DELIMITERS, DEFAULT_ID_COLUMNS, InvalidDelimiter, read_metadata
@@ -889,21 +890,21 @@ def register_parser(parent_subparsers):
     config.add_argument('--maintainers', metavar="name", action="append", nargs='+', help="Analysis maintained by, in format 'Name <URL>' 'Name2 <URL>', ...")
     config.add_argument('--build-url', type=str, metavar="url", help="Build URL/repository to be displayed by Auspice")
     config.add_argument('--description', metavar="description.md", help="Markdown file with description of build and/or acknowledgements to be displayed by Auspice")
-    config.add_argument('--geo-resolutions', metavar="trait", nargs='+', help="Geographic traits to be displayed on map")
-    config.add_argument('--color-by-metadata', metavar="trait", nargs='+', help="Metadata columns to include as coloring options")
-    config.add_argument('--metadata-columns', nargs="+",
+    config.add_argument('--geo-resolutions', metavar="trait", nargs='+', action='extend', help="Geographic traits to be displayed on map")
+    config.add_argument('--color-by-metadata', metavar="trait", nargs='+', action='extend', help="Metadata columns to include as coloring options")
+    config.add_argument('--metadata-columns', nargs="+", action="extend",
                                  help="Metadata columns to export in addition to columns provided by --color-by-metadata or colorings in the Auspice configuration file. " +
                                       "These columns will not be used as coloring options in Auspice but will be visible in the tree.")
-    config.add_argument('--panels', metavar="panels", nargs='+', choices=['tree', 'map', 'entropy', 'frequencies', 'measurements'], help="Restrict panel display in auspice. Options are %(choices)s. Ignore this option to display all available panels.")
+    config.add_argument('--panels', metavar="panels", nargs='+', action='extend', choices=['tree', 'map', 'entropy', 'frequencies', 'measurements'], help="Restrict panel display in auspice. Options are %(choices)s. Ignore this option to display all available panels.")
 
     optional_inputs = parser.add_argument_group(
         title="OPTIONAL INPUT FILES"
     )
     optional_inputs.add_argument('--node-data', metavar="JSON", nargs='+', action="extend", help="JSON files containing metadata for nodes in the tree")
     optional_inputs.add_argument('--metadata', metavar="FILE", help="Additional metadata for strains in the tree")
-    optional_inputs.add_argument('--metadata-delimiters', default=DEFAULT_DELIMITERS, nargs="+",
+    optional_inputs.add_argument('--metadata-delimiters', default=DEFER_LIST_DEFAULT, nargs="+", action="extend",
                                  help="delimiters to accept when reading a metadata file. Only one delimiter will be inferred.")
-    optional_inputs.add_argument('--metadata-id-columns', default=DEFAULT_ID_COLUMNS, nargs="+",
+    optional_inputs.add_argument('--metadata-id-columns', default=DEFER_LIST_DEFAULT, nargs="+", action="extend",
                                  help="names of possible metadata columns containing identifier information, ordered by priority. Only one ID column will be inferred.")
     optional_inputs.add_argument('--colors', metavar="FILE", help="Custom color definitions, one per line in the format `TRAIT_TYPE\\tTRAIT_VALUE\\tHEX_CODE`")
     optional_inputs.add_argument('--lat-longs', metavar="TSV", help="Latitudes and longitudes for geography traits (overrides built in mappings)")
@@ -1132,7 +1133,18 @@ def get_additional_metadata_columns(config, command_line_metadata_columns, metad
     return additional_metadata_columns
 
 
+def validate_arguments(args):
+    if args.metadata_delimiters == DEFER_LIST_DEFAULT:
+        args.metadata_delimiters = DEFAULT_DELIMITERS
+
+    if args.metadata_id_columns == DEFER_LIST_DEFAULT:
+        args.metadata_id_columns = DEFAULT_ID_COLUMNS
+
+    return args
+
+
 def run(args):
+    args = validate_arguments(args)
     configure_warnings()
     data_json = {"version": "v2", "meta": {"updated": time.strftime('%Y-%m-%d')}}