Data subsystem yt

.github/workflows/pythonbuild.yml

@@ -17,14 +17,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10"]
-        spark-version-suffix: ["", "-spark2"]
         exclude:
-          - python-version: 3.8
-            spark-version-suffix: "-spark2"
-          - python-version: 3.9
-            spark-version-suffix: "-spark2"
-          - python-version: 3.10
-            spark-version-suffix: "-spark2"
           # Ignore this test because we failed to install docker-py
           # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301.
           # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855
@@ -42,10 +35,10 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements files
-          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
         run: |
-          make setup${{ matrix.spark-version-suffix }}
+          make setup
           pip freeze
       - name: Test with coverage
         run: |
@@ -67,6 +60,7 @@ jobs:
           - flytekit-aws-batch
           - flytekit-aws-sagemaker
           - flytekit-bigquery
+          - flytekit-dask
           - flytekit-data-fsspec
           - flytekit-dbt
           - flytekit-deck-standard
@@ -156,13 +150,13 @@ jobs:
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.in') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip==21.2.4
-          pip install -r dev-requirements.txt
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.in
       - name: Lint
         run: |
           make lint

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
   hooks:
     - id: black
 - repo: https://github.com/PyCQA/isort
-  rev:  5.10.1
+  rev:  5.12.0
   hooks:
     - id: isort
       args: ["--profile", "black"]

.readthedocs.yml

@@ -18,3 +18,4 @@ sphinx:
 python:
   install:
     - requirements: doc-requirements.txt
+    - requirements: docs/requirements.txt

CODEOWNERS

@@ -1,3 +1,3 @@
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence.
-*       @wild-endeavor @kumare3 @eapolinario @pingsutw
+*       @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy

Dockerfile

@@ -4,6 +4,10 @@ FROM python:${PYTHON_VERSION}-slim-buster
 MAINTAINER Flyte Team <[email protected]>
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
 
+RUN useradd -u 1000 flytekit
+RUN chown flytekit: /root
+USER flytekit
+
 WORKDIR /root
 ENV PYTHONPATH /root
 

Makefile

@@ -22,11 +22,7 @@ update_boilerplate:
 
 .PHONY: setup
 setup: install-piptools ## Install requirements
-	pip-sync requirements.txt dev-requirements.txt
-
-.PHONY: setup-spark2
-setup-spark2: install-piptools ## Install requirements
-	pip-sync requirements-spark2.txt dev-requirements.txt
+	pip install -r dev-requirements.in
 
 .PHONY: fmt
 fmt: ## Format code with black and isort
@@ -35,11 +31,12 @@ fmt: ## Format code with black and isort
 
 .PHONY: lint
 lint: ## Run linters
-	mypy flytekit/core || true
-	mypy flytekit/types || true
-	mypy tests/flytekit/unit/core || true
-	# Exclude setup.py to fix error: Duplicate module named "setup"
-	mypy plugins --exclude setup.py || true
+	mypy flytekit/core
+	mypy flytekit/types
+	# allow-empty-bodies: Allow empty body in function.
+	# disable-error-code="annotation-unchecked": Remove the warning "By default the bodies of untyped functions are not checked".
+	# Mypy raises a warning because it cannot determine the type from the dataclass, despite we specified the type in the dataclass.
+	mypy --allow-empty-bodies --disable-error-code="annotation-unchecked" tests/flytekit/unit/core
 	pre-commit run --all-files
 
 .PHONY: spellcheck
@@ -62,18 +59,6 @@ unit_test:
 	pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \
 		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
 
-requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
-requirements-spark2.txt: requirements-spark2.in install-piptools
-	$(PIP_COMPILE) $<
-
-requirements.txt: export CUSTOM_COMPILE_COMMAND := make requirements.txt
-requirements.txt: requirements.in install-piptools
-	$(PIP_COMPILE) $<
-
-dev-requirements.txt: export CUSTOM_COMPILE_COMMAND := make dev-requirements.txt
-dev-requirements.txt: dev-requirements.in requirements.txt install-piptools
-	$(PIP_COMPILE) $<
-
 doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt
 doc-requirements.txt: doc-requirements.in install-piptools
 	$(PIP_COMPILE) $<
@@ -83,7 +68,7 @@ ${MOCK_FLYTE_REPO}/requirements.txt: ${MOCK_FLYTE_REPO}/requirements.in install-
 	$(PIP_COMPILE) $<
 
 .PHONY: requirements
-requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compile requirements
+requirements: doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compile requirements
 
 # TODO: Change this in the future to be all of flytekit
 .PHONY: coverage

dev-requirements.in

@@ -1,4 +1,4 @@
--c requirements.txt
+-r requirements.in
 
 git+https://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte
 coverage[toml]
@@ -12,8 +12,13 @@ codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
-tensorflow==2.8.1
+
+# Only install tensorflow if not running on an arm Mac.
+tensorflow==2.8.1; platform_machine!='arm64' or platform_system!='Darwin'
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1
 scikit-learn
+types-protobuf
+types-croniter
+types-mock