Merge upstream up to netflix/metaflow 2.5.1

.github/workflows/publish.yml

@@ -3,68 +3,8 @@ on:
   release:
     types: [published]
 jobs:
-  test:
-    name: ${{ matrix.lang }} tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        lang: [Python, R]
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Python 2.x
-      if: matrix.lang == 'Python'
-      uses: actions/setup-python@v1
-      with:
-        python-version: '2.x'
-
-    - name: Install Python 2.x dependencies
-      if: matrix.lang == 'Python'
-      run: |
-        python2 -m pip install --upgrade pip
-        python2 -m pip install tox
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.7'
-
-    - name: Install Python 3.x dependencies
-      run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install tox numpy
-
-    - name: Set up R 3.6
-      if: matrix.lang == 'R'
-      uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: '3.6.3' 
-
-    - name: Install R 3.6 system dependencies
-      if: matrix.lang == 'R' && matrix.os == 'ubuntu-latest'
-      run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev
-
-    - name: Install R 3.6 Rlang dependencies
-      if: matrix.lang == 'R'
-      run: |
-        python3 -m pip install . 
-        Rscript -e 'install.packages("devtools", repos="https://cloud.r-project.org", Ncpus=8)'
-        Rscript -e 'devtools::install_deps("R", dependencies=TRUE, repos="https://cloud.r-project.org", upgrade="default")'
-        R CMD INSTALL R
-        Rscript -e 'install.packages(c("data.table", "caret", "glmnet", "Matrix", "rjson"), repos="https://cloud.r-project.org", Ncpus=8)'   
-
-    - name: Execute R tests
-      if: matrix.lang == 'R'
-      run: |
-        cd R/tests
-        Rscript run_tests.R
-
-    - name: Execute Python tests
-      if: matrix.lang == 'python'
-      run: tox
-
+  test: 
+    uses: './.github/workflows/test.yml'
   deploy:
     needs: test
     runs-on: ubuntu-latest

.github/workflows/s3_test.yml

@@ -0,0 +1,43 @@
+name: pr-datastore-tests
+
+on:
+  push:
+    branches:
+    - master
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - labeled
+
+jobs:
+  test_data:
+    if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved')))
+    name: S3 data tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: refs/pull/${{ github.event.pull_request.number }}/merge
+        submodules: recursive
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+    - name: Install Python 3.x dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install tox numpy pytest click boto3 requests pylint pytest-benchmark
+    - name: Execute Data tests
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        METAFLOW_S3_TEST_ROOT: ${{ secrets.METAFLOW_S3_TEST_ROOT }}
+      run: |
+        cd test/data
+        PYTHONPATH=$(pwd)/../../ python3 -m pytest --benchmark-skip -x -s -v

.github/workflows/test.yml

@@ -7,32 +7,94 @@ on:
   pull_request:
     branches:
     - master
+<<<<<<< HEAD
     - feature/kfp
+=======
+  workflow_call:
+
+>>>>>>> master
 jobs:
-  test:
-    name: ${{ matrix.lang }} tests on ${{ matrix.os }}
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/[email protected]
+
+  Python:
+    name: core / Python ${{ matrix.ver }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
+<<<<<<< HEAD
         lang: [Python]
+=======
+        ver: ['3.5', '3.6', '3.7', '3.8', '3.9','3.10',]
+>>>>>>> master
 
     steps:
     - uses: actions/checkout@v2
 
+<<<<<<< HEAD
     - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
         python-version: '3.9'
+=======
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.ver }}
+>>>>>>> master
 
-    - name: Install Python 3.x dependencies
+    - name: Install Python ${{ matrix.ver }} dependencies
       run: |
         python3 -m pip install --upgrade pip
+<<<<<<< HEAD
         python3 -m pip install tox numpy black
 
     - name: Python Code Format Check
       run: black --target-version py39 --diff --check ./metaflow/plugins/kfp/*.py
+=======
+        python3 -m pip install tox numpy
 
     - name: Execute Python tests
-      if: matrix.lang == 'python'
       run: tox
+
+  R:
+    name: core / R ${{ matrix.ver }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ver: ['4.0', '4.1']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up ${{ matrix.ver }}
+      uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: ${{ matrix.ver }}
+
+    - name: Install R ${{ matrix.ver }} system dependencies
+      if: matrix.os == 'ubuntu-latest'
+      run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev
+
+    - name: Install R ${{ matrix.ver }} Rlang dependencies
+      run: |
+        python3 -m pip install . 
+        Rscript -e 'install.packages("devtools", repos="https://cloud.r-project.org", Ncpus=8)'
+        Rscript -e 'devtools::install_deps("R", dependencies=TRUE, repos="https://cloud.r-project.org", upgrade="default")'
+        R CMD INSTALL R
+        Rscript -e 'install.packages(c("data.table", "caret", "glmnet", "Matrix", "rjson"), repos="https://cloud.r-project.org", Ncpus=8)'   
+
+    - name: Execute R tests
+      run: |
+        cd R/tests
+        Rscript run_tests.R
+>>>>>>> master
+
+

.gitignore

@@ -19,6 +19,9 @@ R/.Rbuildignore
 .Rproj.user
 
 .DS_Store
+.env
+node_modules
+main.js.map
 
 .project
 .pydevproject

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+exclude: 'test/core/tests/card_timeout.py'
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+      - id: check-yaml
+      - id: check-json
+  - repo: https://github.com/ambv/black
+    rev: 21.9b0
+    hooks:
+      - id: black
+        language_version: python3
+        exclude: "^metaflow/_vendor/"

MANIFEST.in

@@ -1 +1,4 @@
 include LICENSE
+include metaflow/plugins/cards/card_modules/*.html
+include metaflow/plugins/cards/card_modules/*.js
+include metaflow/plugins/cards/card_modules/*.css

R/R/package.R

@@ -22,5 +22,5 @@ set_global_variable <- function(key, val, pos = 1) {
 #' @export
 metaflow <- function(cls, ...) {
   set_global_variable(cls, Flow$new(cls, list(...)))
-  get(cls)
+  get(cls, pos = 1)
 }

R/R/parameter.R

@@ -8,7 +8,7 @@
 #' @param flow metaflow object
 #' @param parameter name of the parameter
 #' @param required logical (defaults to FALSE) denoting if
-#' paramter is required as an argument to \code{run} the flow
+#' parameter is required as an argument to \code{run} the flow
 #' @param help optional help text
 #' @param default optional default value of the parameter
 #' @param type optional type of the parameter

R/README.md

@@ -10,7 +10,7 @@ Getting up and running with Metaflow is easy. Install Metaflow from [github](htt
 
 >```R
 >devtools::install_github("Netflix/metaflow", subdir="R")
->metaflow::install()
+>metaflow::install_metaflow()
 >```
 
 and access tutorials by typing:

R/inst/tutorials/04-helloaws/helloaws.Rmd

@@ -2,7 +2,7 @@
 title: "Episode 04-helloaws: Look Mom, We're in the Cloud"
 output: html_notebook
 ---
-In HellowAWSFlow, the 'start' and 'end' steps were run locally, while the 'hello' step was run remotely on AWS batch. Since we are using AWS, data artifacts and metdata were stored remotely. This means you can use the client to access information about any flow from anywhere. This notebook shows you how.
+In HellowAWSFlow, the 'start' and 'end' steps were run locally, while the 'hello' step was run remotely on AWS batch. Since we are using AWS, data artifacts and metadata were stored remotely. This means you can use the client to access information about any flow from anywhere. This notebook shows you how.
 
 ## Import the metaflow client
 ```{r}

R/inst/tutorials/05-statistics-redux/README.md

@@ -32,5 +32,5 @@ If you are using RStudio, you can replace the last line `run()` with
 and run by `source("stats.R")`.
 
 ##### Inspect the results:
-Open the R markdown file ```02-statistics/stats.Rmd``` in your RStudio and re-run the cells. You can acccess
+Open the R markdown file ```02-statistics/stats.Rmd``` in your RStudio and re-run the cells. You can access
 the artifacts stored in AWS S3 from your local RStudio session. 

R/inst/tutorials/07-autopilot/README.md

@@ -35,5 +35,5 @@ run(package_suffixes=".R,.csv", step_functions="trigger")
 for SFN trigger. You can then directly run `source("stats.R`)` in RStudio. 
 
 ##### Inspect the results:
-Open the R Markdown file```07-autopilot/stats.Rmd``` in your RStudio and re-run the cells. You can acccess
+Open the R Markdown file```07-autopilot/stats.Rmd``` in your RStudio and re-run the cells. You can access
 the artifacts stored in AWS S3 from your local RStudio session. 

R/tests/testthat/test-flow.R

@@ -1,5 +1,7 @@
 context("test-flow.R")
 
+teardown(if ("sqrt" %in% names(.GlobalEnv)) rm("sqrt", envir = .GlobalEnv))
+
 test_that("header() formatted correctly", {
   skip_if_no_metaflow()
   actual <- header("TestFlow")
@@ -88,3 +90,11 @@ test_that("get_functions() works", {
   )
   expect_equal(actual, expected)
 })
+
+test_that("flow names are assigned to global environment", {
+  expect_false("sqrt" %in% names(.GlobalEnv))
+  step(metaflow("sqrt"), step = "start")
+  expect_true("sqrt" %in% names(.GlobalEnv))
+  expect_s3_class(get("sqrt", envir = .GlobalEnv), "Flow")
+  expect_equal(base::sqrt(4), 2)
+})

README.md

@@ -46,4 +46,9 @@ There are several ways to get in touch with us:
 * Chat with us on: http://chat.metaflow.org 
 
 ## Contributing
+
 We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
+
+### Code style
+
+We use [black](https://black.readthedocs.io/en/stable/) as a code formatter. The easiest way to ensure your commits are always formatted with the correct version of `black` it is to use [pre-commit](https://pre-commit.com/): install it and then run `pre-commit install` once in your local copy of the repo. 

docs/Environment escape.md

@@ -143,7 +143,7 @@ and produces a JSON-able object (typically a dictionary with string keys and
 jsonable objects as values). The decoding is the reverse where a dictionary is
 taken from the channel and Python objects are returned.
 
-Transfering exceptions requires a tiny bit more work and this logic can be found
+Transferring exceptions requires a tiny bit more work and this logic can be found
 in ```exception_transferer.py```; this relies on ```data_transferer.py``` to do
 the actual encoding and decoding and ```exception_transferer.py``` merely takes
 care of the specificities of extracting the information needed from the