feat: add install_dev_pandas, spark and compile requirements to makef…

…ile and add spark session as pytest fixture

.github/workflows/tests.yml

@@ -220,7 +220,6 @@ jobs:
         run: echo "ARROW_PRE_0_15_IPC_FORMAT=1" >> $GITHUB_ENV
       - run: echo "SPARK_LOCAL_IP=127.0.0.1" >> $GITHUB_ENV
       - run: make install
-      - run: make install-spark-ci
       - run: pip install -r requirements-spark.txt # Make sure the proper version of pandas is install after everything
       - run: make test_spark
 

Makefile

@@ -1,3 +1,7 @@
+VENV = venv
+PYTHON = $(VENV)/bin/python
+ACTIVATE = . $(VENV)/bin/activate
+
 .PHONY: docs examples
 
 docs:
@@ -10,7 +14,7 @@ test:
 	ydata_profiling -h
 
 test_spark:
-	pytest --spark_home=${SPARK_HOME} tests/backends/spark_backend/
+	pytest tests/backends/spark_backend/
 	ydata_profiling -h
 
 test_cov:
@@ -28,18 +32,32 @@ package:
 	python setup.py sdist bdist_wheel
 	twine check dist/*
 
+install_dev_pandas:
+	rm -rf $(VENV)
+	python -m venv $(VENV)
+	$(PYTHON) -m pip install --upgrade pip
+	$(PYTHON) -m pip install -r requirements-dev.txt
+	$(PYTHON) -m pip install -e .
+
+install_dev_spark:
+	rm -rf $(VENV)
+	python -m venv $(VENV)
+	$(PYTHON) -m pip install --upgrade pip
+	$(PYTHON) -m pip install -r requirements-spark.txt
+	$(PYTHON) -m pip install -e .
+
 install:
 	pip install -e .[notebook]
 
 install-docs: install ### Installs regular and docs dependencies
 	pip install -r requirements-docs.txt
 
-install-spark-ci:
-	sudo apt-get update
-	sudo apt-get -y install openjdk-8-jdk
-	curl https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \
-	--output ${SPARK_DIRECTORY}/spark.tgz
-	cd ${SPARK_DIRECTORY} && tar -xvzf spark.tgz && mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
+# compile requirements files
+compile:
+	$(PYTHON) -m piptools compile -o requirements.txt pyproject.toml
+	$(PYTHON) -m piptools compile --extra dev --extra pandas -o requirements-dev.txt pyproject.toml
+	$(PYTHON) -m piptools compile --extra dev --extra spark -o requirements-spark.txt pyproject.toml
+	$(PYTHON) -m piptools compile --extra dev -o requirements-test.txt pyproject.toml
 
 publish-docs: examples ### Publishes the documentation
 	mkdir docs/examples

README.md

@@ -217,6 +217,9 @@ Install it by navigating to the proper directory and running:
 ```sh
 pip install -e .
 ```
+or you can use command `make install_dev_pandas` to install requirements for pandas development 
+or `make install_dev_spark` for spark development.
+Than activate venv with command `source venv/bin/activate`. 
 
 The profiling report is written in HTML and CSS, which means a modern browser is required. 
 
@@ -225,8 +228,9 @@ You need [Python 3](https://python3statement.org/) to run the package. Other dep
 | Filename | Requirements|
 |----------|-------------|
 | [requirements.txt](https://github.com/ydataai/pandas-profiling/blob/master/requirements.txt) | Package requirements|
-| [requirements-dev.txt](https://github.com/ydataai/pandas-profiling/blob/master/requirements-dev.txt)  |  Requirements for development|
-| [requirements-test.txt](https://github.com/ydataai/pandas-profiling/blob/master/requirements-test.txt) | Requirements for testing|
+| [requirements-dev.txt](https://github.com/ydataai/pandas-profiling/blob/master/requirements-dev.txt)  |  Requirements for pandas development|
+| [requirements-spark.txt](https://github.com/ydataai/pandas-profiling/blob/master/requirements-spark.txt)  |  Requirements for spark development|
+| [requirements-test.txt](https://github.com/ydataai/pandas-profiling/blob/master/requirements-test.txt) | Requirements for tests|
 | [setup.py](https://github.com/ydataai/pandas-profiling/blob/master/setup.py) | Requirements for widgets etc. |
 
 ## 🔗 Integrations