From bf9660a054aa5e133feedebee804bf955828d3e7 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 17 Apr 2021 13:25:23 +0300 Subject: [PATCH 1/2] Bunch of small refactorings and updates --- .pre-commit-config.yaml | 8 ++++---- .readthedocs.yml | 2 +- Makefile | 4 +++- README.md | 2 +- base-notebook/Dockerfile | 2 +- datascience-notebook/Dockerfile | 4 ++-- docs/using/common.md | 2 +- docs/using/recipes.md | 2 +- docs/using/running.md | 2 +- docs/using/specifics.md | 8 ++++---- examples/docker-compose/README.md | 2 +- test/helpers.py | 16 +++++++--------- test/test_packages.py | 14 +++++--------- 13 files changed, 32 insertions(+), 36 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bae8fa3002..6045755696 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.4.0 hooks: - id: check-yaml files: .*\.(yaml|yml)$ - repo: https://github.com/adrienverge/yamllint.git - rev: v1.23.0 + rev: v1.26.1 hooks: - id: yamllint args: ['-d {extends: relaxed, rules: {line-length: disable}}', '-s'] @@ -17,10 +17,10 @@ repos: - id: bashate args: ['--ignore=E006'] - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.9.1 hooks: - id: flake8 - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.5.4 + rev: v1.5.6 hooks: - id: autopep8 diff --git a/.readthedocs.yml b/.readthedocs.yml index 47c269bfec..4f766de72a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,6 @@ sphinx: formats: all python: - version: 3.7 + version: 3.8 install: - requirements: requirements-dev.txt diff --git a/Makefile b/Makefile index d3dc3c36d3..896035a560 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ git-commit: ## commit outstading git changes and push to remote @git config --global user.name "GitHub Actions" @git config --global user.email "actions@users.noreply.github.com" - @echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)" + @echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)" @cd $(LOCAL_PATH) && \ git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git && \ git checkout master && \ @@ -152,6 +152,8 @@ pull/%: DARGS?= pull/%: ## pull a jupyter image docker pull $(DARGS) $(OWNER)/$(notdir $@) +pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images + push/%: DARGS?= push/%: ## push all tags for a jupyter image docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@) diff --git a/README.md b/README.md index 888ad2e310..9a224b3b23 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ in working on the project. ## Jupyter Notebook Deprecation Notice Following [Jupyter Notebook notice](https://github.com/jupyter/notebook#notice), we encourage users to transition to JupyterLab. -This can be done by passing the environment variable `JUPYTER_ENABLE_LAB=yes` at container startup, +This can be done by passing the environment variable `JUPYTER_ENABLE_LAB=yes` at container startup, more information is available in the [documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#docker-options). At some point, JupyterLab will become the default for all of the Jupyter Docker stack images, however a new environment variable will be introduced to switch back to Jupyter Notebook if needed. diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index e35fdd2afb..4544a220ac 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -79,7 +79,7 @@ RUN chmod a+rx /usr/local/bin/fix-permissions # hadolint ignore=SC2016 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group # and make sure these dirs are writable by the `users` group. diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index 24a0133c7d..3488518069 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -62,10 +62,10 @@ RUN conda install --quiet --yes \ 'r-caret=6.0*' \ 'r-crayon=1.4*' \ 'r-devtools=2.4*' \ - 'r-forecast=8.14*' \ + 'r-forecast=8.14*' \ 'r-hexbin=1.28*' \ 'r-htmltools=0.5*' \ - 'r-htmlwidgets=1.5*' \ + 'r-htmlwidgets=1.5*' \ 'r-irkernel=1.1*' \ 'r-nycflights13=1.0*' \ 'r-randomforest=4.6*' \ diff --git a/docs/using/common.md b/docs/using/common.md index 6d3279057b..96f3cbccc3 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -116,7 +116,7 @@ mamba install some-package ### Using alternative channels -Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. +Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `conda` to use different channels. The examples below show how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages. diff --git a/docs/using/recipes.md b/docs/using/recipes.md index f60cd6b4b4..f64c51f8b5 100644 --- a/docs/using/recipes.md +++ b/docs/using/recipes.md @@ -130,7 +130,7 @@ RUN $CONDA_DIR/envs/${conda_env}/bin/python -m ipykernel install --user --name=$ fix-permissions /home/$NB_USER # any additional pip installs can be added by uncommenting the following line -# RUN $CONDA_DIR/envs/${conda_env}/bin/pip install +# RUN $CONDA_DIR/envs/${conda_env}/bin/pip install # prepend conda environment to path ENV PATH $CONDA_DIR/envs/${conda_env}/bin:$PATH diff --git a/docs/using/running.md b/docs/using/running.md index af34dd4acc..86a09dfdb7 100644 --- a/docs/using/running.md +++ b/docs/using/running.md @@ -76,7 +76,7 @@ Executing the command: jupyter notebook Pressing `Ctrl-C` shuts down the notebook server and immediately destroys the Docker container. Files written to `~/work` in the container remain touched. Any other changes made in the container are lost. -**Example 3** This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port. +**Example 3** This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port. ``` docker run -d -P --name notebook jupyter/all-spark-notebook diff --git a/docs/using/specifics.md b/docs/using/specifics.md index 245cf30ea7..651288baab 100644 --- a/docs/using/specifics.md +++ b/docs/using/specifics.md @@ -82,7 +82,7 @@ sc <- sparkR.session("local") # Sum of the first 100 whole numbers sdf <- createDataFrame(list(1:100)) dapplyCollect(sdf, - function(x) + function(x) { x <- sum(x)} ) # 5050 @@ -102,7 +102,7 @@ conf$spark.sql.catalogImplementation <- "in-memory" sc <- spark_connect(master = "local", config = conf) # Sum of the first 100 whole numbers -sdf_len(sc, 100, repartition = 1) %>% +sdf_len(sc, 100, repartition = 1) %>% spark_apply(function(e) sum(e)) # 5050 ``` @@ -171,7 +171,7 @@ sc <- sparkR.session("spark://master:7077") # Sum of the first 100 whole numbers sdf <- createDataFrame(list(1:100)) dapplyCollect(sdf, - function(x) + function(x) { x <- sum(x)} ) # 5050 @@ -190,7 +190,7 @@ conf$spark.sql.catalogImplementation <- "in-memory" sc <- spark_connect(master = "spark://master:7077", config = conf) # Sum of the first 100 whole numbers -sdf_len(sc, 100, repartition = 1) %>% +sdf_len(sc, 100, repartition = 1) %>% spark_apply(function(e) sum(e)) # 5050 ``` diff --git a/examples/docker-compose/README.md b/examples/docker-compose/README.md index 9c00f7ea53..d4a7d2e14c 100644 --- a/examples/docker-compose/README.md +++ b/examples/docker-compose/README.md @@ -85,7 +85,7 @@ NAME=your-notebook PORT=9001 WORK_VOLUME=our-work notebook/up.sh ### How do I run over HTTPS? -To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script. You must also provide a password, which will be used to secure the notebook server. You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script. +To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script. You must also provide a password, which will be used to secure the notebook server. You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script. ```bash PASSWORD=a_secret notebook/up.sh --secure diff --git a/test/helpers.py b/test/helpers.py index 867b19a913..70919c400f 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -75,7 +75,7 @@ def specified_packages(self): if self.specs is None: LOGGER.info("Grabing the list of specifications ...") self.specs = CondaPackageHelper._packages_from_json( - self._execute_command(CondaPackageHelper._conda_export_command(True)) + self._execute_command(CondaPackageHelper._conda_export_command(from_history=True)) ) return self.specs @@ -130,7 +130,7 @@ def _extract_available(lines): return ddict def check_updatable_packages(self, specifications_only=True): - """Check the updatables packages including or not dependencies""" + """Check the updatable packages including or not dependencies""" specs = self.specified_packages() installed = self.installed_packages() available = self.available_packages() @@ -145,9 +145,10 @@ def check_updatable_packages(self, specifications_only=True): current = min(inst_vs, key=CondaPackageHelper.semantic_cmp) newest = avail_vs[-1] if avail_vs and current != newest: - if CondaPackageHelper.semantic_cmp( - current - ) < CondaPackageHelper.semantic_cmp(newest): + if ( + CondaPackageHelper.semantic_cmp(current) < + CondaPackageHelper.semantic_cmp(newest) + ): self.comparison.append( {"Package": pkg, "Current": current, "Newest": newest} ) @@ -180,10 +181,7 @@ def try_int(version_str): def get_outdated_summary(self, specifications_only=True): """Return a summary of outdated packages""" - if specifications_only: - nb_packages = len(self.specs) - else: - nb_packages = len(self.installed) + nb_packages = len(self.specs if specifications_only else self.installed) nb_updatable = len(self.comparison) updatable_ratio = nb_updatable / nb_packages return f"{nb_updatable}/{nb_packages} ({updatable_ratio:.0%}) packages could be updated" diff --git a/test/test_packages.py b/test/test_packages.py index 6c44a66f12..ca6da5e756 100644 --- a/test/test_packages.py +++ b/test/test_packages.py @@ -7,12 +7,12 @@ This test module tests if R and Python packages installed can be imported. It's a basic test aiming to prove that the package is working properly. -The goal is to detect import errors that can be caused by incompatibilities between packages for example: +The goal is to detect import errors that can be caused by incompatibilities between packages, for example: - #1012: issue importing `sympy` - #966: isssue importing `pyarrow` -This module checks dynmamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfiles`. +This module checks dynamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfile`s. This means that it does not check dependencies. This choice is a tradeoff to cover the main requirements while achieving reasonable test duration. However it could be easily changed (or completed) to cover also dependencies `package_helper.installed_packages()` instead of `package_helper.specified_packages()`. @@ -86,10 +86,7 @@ def packages(package_helper): def package_map(package): """Perform a mapping between the python package name and the name used for the import""" - _package = package - if _package in PACKAGE_MAPPING: - _package = PACKAGE_MAPPING.get(_package) - return _package + return PACKAGE_MAPPING.get(package, package) def excluded_package_predicate(package): @@ -136,9 +133,8 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail for package in filtered_packages: LOGGER.info(f"Trying to import {package}") try: - assert ( - check_function(package_helper, package) == 0 - ), f"Package [{package}] import failed" + assert check_function(package_helper, package) == 0, \ + f"Package [{package}] import failed" except AssertionError as err: failures[package] = err if len(failures) > max_failures: From a8b7682fd6f15f6d9c06839aadfc1eec6cc2f08a Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 17 Apr 2021 18:20:37 +0300 Subject: [PATCH 2/2] Add newlines --- CODE_OF_CONDUCT.md | 2 +- all-spark-notebook/hooks/run_hook | 2 +- all-spark-notebook/test/data/local_pyspark.ipynb | 2 +- all-spark-notebook/test/data/local_sparkR.ipynb | 2 +- all-spark-notebook/test/data/local_sparklyr.ipynb | 2 +- all-spark-notebook/test/data/local_spylon.ipynb | 2 +- docs/Makefile | 2 +- docs/_static/custom.css | 2 +- docs/contributing/issues.md | 2 +- docs/contributing/lint.md | 2 +- docs/contributing/translations.md | 2 +- docs/using/common.md | 2 +- docs/using/specifics.md | 2 +- examples/make-deploy/Makefile | 2 +- minimal-notebook/test/data/notebook_math.ipynb | 2 +- pyspark-notebook/hooks/run_hook | 2 +- pytest.ini | 2 +- scipy-notebook/hooks/run_hook | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index eb72ecd0bc..a172249272 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1 +1 @@ -Please see the [Project Jupyter Code of Conduct](https://github.com/jupyter/governance/blob/master/conduct/code_of_conduct.md). \ No newline at end of file +Please see the [Project Jupyter Code of Conduct](https://github.com/jupyter/governance/blob/master/conduct/code_of_conduct.md). diff --git a/all-spark-notebook/hooks/run_hook b/all-spark-notebook/hooks/run_hook index cbfc0d6a46..a4182183d6 100755 --- a/all-spark-notebook/hooks/run_hook +++ b/all-spark-notebook/hooks/run_hook @@ -58,4 +58,4 @@ $(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c( \`\`\` $(docker run --rm ${IMAGE_NAME} apt list --installed) \`\`\` -EOF \ No newline at end of file +EOF diff --git a/all-spark-notebook/test/data/local_pyspark.ipynb b/all-spark-notebook/test/data/local_pyspark.ipynb index 66129f52e7..85be3b89c0 100644 --- a/all-spark-notebook/test/data/local_pyspark.ipynb +++ b/all-spark-notebook/test/data/local_pyspark.ipynb @@ -57,4 +57,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/all-spark-notebook/test/data/local_sparkR.ipynb b/all-spark-notebook/test/data/local_sparkR.ipynb index ecf7f7c1ec..aaa11ba4da 100644 --- a/all-spark-notebook/test/data/local_sparkR.ipynb +++ b/all-spark-notebook/test/data/local_sparkR.ipynb @@ -38,4 +38,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/all-spark-notebook/test/data/local_sparklyr.ipynb b/all-spark-notebook/test/data/local_sparklyr.ipynb index 8f4527243b..be6651ba3d 100644 --- a/all-spark-notebook/test/data/local_sparklyr.ipynb +++ b/all-spark-notebook/test/data/local_sparklyr.ipynb @@ -40,4 +40,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/all-spark-notebook/test/data/local_spylon.ipynb b/all-spark-notebook/test/data/local_spylon.ipynb index 0caf2f0a0b..7545d99c43 100644 --- a/all-spark-notebook/test/data/local_spylon.ipynb +++ b/all-spark-notebook/test/data/local_spylon.ipynb @@ -60,4 +60,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/docs/Makefile b/docs/Makefile index 7723800afa..b413321ed7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 7bfd656097..af20b7aff2 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,3 +1,3 @@ body div.sphinxsidebarwrapper p.logo { text-align: left; -} \ No newline at end of file +} diff --git a/docs/contributing/issues.md b/docs/contributing/issues.md index 0cd666ac45..edd8b13ae0 100644 --- a/docs/contributing/issues.md +++ b/docs/contributing/issues.md @@ -22,4 +22,4 @@ your problem. * If you have a general question about how to use the Jupyter Docker Stacks in your environment, in conjunction with other tools, with customizations, and so on, please post your question on the [Jupyter Discourse - site](https://discourse.jupyter.org). \ No newline at end of file + site](https://discourse.jupyter.org). diff --git a/docs/contributing/lint.md b/docs/contributing/lint.md index f16c4b432f..138e278fb9 100644 --- a/docs/contributing/lint.md +++ b/docs/contributing/lint.md @@ -111,4 +111,4 @@ RUN cd /tmp && echo "hello!" [rules]: https://github.com/hadolint/hadolint#rules [DL3006]: https://github.com/hadolint/hadolint/wiki/DL3006 [DL3008]: https://github.com/hadolint/hadolint/wiki/DL3008 -[pre-commit]: https://pre-commit.com/ \ No newline at end of file +[pre-commit]: https://pre-commit.com/ diff --git a/docs/contributing/translations.md b/docs/contributing/translations.md index 6cc0567390..b9cc1150b5 100644 --- a/docs/contributing/translations.md +++ b/docs/contributing/translations.md @@ -4,4 +4,4 @@ We are delighted when members of the Jupyter community want to help translate th 1. Follow the steps documented on the [Getting Started as a Translator](https://docs.transifex.com/getting-started-1/translators) page. 2. Look for *jupyter-docker-stacks* when prompted to choose a translation team. Alternatively, visit https://www.transifex.com/project-jupyter/jupyter-docker-stacks-1 after creating your account and request to join the project. -3. See [Translating with the Web Editor](https://docs.transifex.com/translation/translating-with-the-web-editor) in the Transifex documentation. \ No newline at end of file +3. See [Translating with the Web Editor](https://docs.transifex.com/translation/translating-with-the-web-editor) in the Transifex documentation. diff --git a/docs/using/common.md b/docs/using/common.md index 96f3cbccc3..8ccd0d635f 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -127,4 +127,4 @@ conda install --channel defaults humanize conda config --system --prepend channels defaults # install a package conda install humanize -``` \ No newline at end of file +``` diff --git a/docs/using/specifics.md b/docs/using/specifics.md index 651288baab..af6e4219e9 100644 --- a/docs/using/specifics.md +++ b/docs/using/specifics.md @@ -249,4 +249,4 @@ sess.run(hello) [sparkr]: https://spark.apache.org/docs/latest/sparkr.html [sparklyr]: https://spark.rstudio.com/ -[spark-conf]: https://spark.apache.org/docs/latest/configuration.html \ No newline at end of file +[spark-conf]: https://spark.apache.org/docs/latest/configuration.html diff --git a/examples/make-deploy/Makefile b/examples/make-deploy/Makefile index 71d16857ad..083a82c572 100644 --- a/examples/make-deploy/Makefile +++ b/examples/make-deploy/Makefile @@ -41,4 +41,4 @@ include softlayer.makefile # Preset notebook configurations include self-signed.makefile -include letsencrypt.makefile \ No newline at end of file +include letsencrypt.makefile diff --git a/minimal-notebook/test/data/notebook_math.ipynb b/minimal-notebook/test/data/notebook_math.ipynb index 0ee2ca07e6..3c400349d2 100644 --- a/minimal-notebook/test/data/notebook_math.ipynb +++ b/minimal-notebook/test/data/notebook_math.ipynb @@ -146,4 +146,4 @@ "metadata": {} } ] -} \ No newline at end of file +} diff --git a/pyspark-notebook/hooks/run_hook b/pyspark-notebook/hooks/run_hook index d4c6552108..a6df1233a9 100755 --- a/pyspark-notebook/hooks/run_hook +++ b/pyspark-notebook/hooks/run_hook @@ -48,4 +48,4 @@ $(docker run --rm ${IMAGE_NAME} conda list) \`\`\` $(docker run --rm ${IMAGE_NAME} apt list --installed) \`\`\` -EOF \ No newline at end of file +EOF diff --git a/pytest.ini b/pytest.ini index 106434fb9a..851895e4c7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,4 +5,4 @@ log_cli_level = INFO log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) log_cli_date_format=%Y-%m-%d %H:%M:%S markers = - info: marks tests as info (deselect with '-m "not info"') \ No newline at end of file + info: marks tests as info (deselect with '-m "not info"') diff --git a/scipy-notebook/hooks/run_hook b/scipy-notebook/hooks/run_hook index 783ffa926e..69c2175695 100755 --- a/scipy-notebook/hooks/run_hook +++ b/scipy-notebook/hooks/run_hook @@ -42,4 +42,4 @@ $(docker run --rm ${IMAGE_NAME} conda list) \`\`\` $(docker run --rm ${IMAGE_NAME} apt list --installed) \`\`\` -EOF \ No newline at end of file +EOF