Merge pull request #587 from locationtech/spark-3.2

Spark 3.2.1

.github/image/.dockerignore

@@ -0,0 +1,3 @@
+*
+!requirements-conda.txt
+!fix-permissions

.github/image/Dockerfile

@@ -0,0 +1,28 @@
+FROM adoptopenjdk/openjdk11:debian-slim
+
+# See: https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html
+RUN \
+  apt-get update && \
+  apt-get install -yq gpg && \
+  curl -s https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg  && \
+  install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
+  gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
+  echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list && \
+  apt-get update && \
+  apt-get install -yq --no-install-recommends conda && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+ENV CONDA_DIR=/opt/conda
+ENV PATH=$CONDA_DIR/bin:$PATH
+
+COPY requirements-conda.txt /tmp
+RUN \
+    conda install --quiet --yes --channel=conda-forge --file=/tmp/requirements-conda.txt && \
+    echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \
+    ldconfig && \
+    conda clean --all --force-pkgs-dirs --yes --quiet
+
+# Work-around for pyproj issue https://github.com/pyproj4/pyproj/issues/415
+ENV PROJ_LIB=/opt/conda/share/proj
+

.github/image/Makefile

@@ -0,0 +1,27 @@
+IMAGE_NAME=debian-openjdk-conda-gdal
+SHA=$(shell git log -n1 --format=format:"%H" | cut -c 1-7)
+VERSION?=$(SHA)
+HOST=docker.io
+REPO=$(HOST)/s22s
+FULL_NAME=$(REPO)/$(IMAGE_NAME):$(VERSION)
+
+.DEFAULT_GOAL := help
+help:
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+	@echo "Usage: make [target]"
+	@echo "Targets: "
+	@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\t\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+all: build push ## Build and then push image
+
+build: ## Build the docker image
+	docker build . -t ${FULL_NAME}
+
+login: ## Login to the docker registry
+	docker login
+
+push: login ## Push docker image to registry
+	docker push ${FULL_NAME}
+
+run: build ## Build image and launch shell
+	docker run --rm -it ${FULL_NAME} bash

.github/image/requirements-conda.txt

@@ -0,0 +1,5 @@
+python==3.8
+gdal==3.1.2
+libspatialindex
+rasterio[s3]
+rtree

.github/workflows/build-test.yml

@@ -0,0 +1,66 @@
+name: Build and Test
+
+on:
+  pull_request:
+    branches: ['**']
+  push:
+    branches: ['master', 'develop', 'release/*', 'spark-3.2']
+    tags: [v*]
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: coursier/cache-action@v6
+      - name: Setup JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 8
+          cache: sbt
+
+      # Do just the compilation stage to minimize sbt memory footprint
+      - name: Compile
+        run: sbt -v -batch compile test:compile it:compile
+
+      - name: Core tests
+        run: sbt -batch core/test
+
+      - name: Datasource tests
+        run: sbt -batch datasource/test
+
+      - name: Experimental tests
+        run: sbt -batch experimental/test
+
+      ## TODO: Update python build to be PEP 517 compatible
+      # - name: Install Conda dependencies
+      #   run: |
+      #     # $CONDA_DIR is an environment variable pointing to the root of the miniconda directory
+      #     $CONDA_DIR/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt
+      # - name: Create PyRasterFrames package
+      #   run: sbt -v -batch pyrasterframes/package
+      # - name: Python tests
+      #   run: sbt -batch pyrasterframes/test
+
+      - name: Collect artifacts
+        if: ${{ failure() }}
+        run: |
+          mkdir -p /tmp/core_dumps
+          ls -lh /tmp
+          cp core.* *.hs /tmp/core_dumps/ 2> /dev/null || true
+          cp ./core/*.log /tmp/core_dumps/ 2> /dev/null || true
+          cp -r /tmp/hsperfdata* /tmp/*.hprof /tmp/core_dumps/ 2> /dev/null || true
+          cp repo/core/core/* /tmp/core_dumps/ 2> /dev/null || true
+
+      - name: Upload core dumps
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: core-dumps
+          path: /tmp/core_dumps

.github/workflows/docs.yml

@@ -0,0 +1,68 @@
+name: Compile documentation
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    branches: ['**docs*']
+  push:
+    branches: ['master', 'release/*']
+  release:
+    types: [published]
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    container:
+      image: s22s/debian-openjdk-conda-gdal:6790f8d
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: coursier/cache-action@v6
+      - uses: olafurpg/setup-scala@v13
+        with:
+          java-version: [email protected]
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install Conda dependencies
+        run: |
+          # $CONDA_DIR is an environment variable pointing to the root of the miniconda directory
+          $CONDA_DIR/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt
+
+      - name: Build documentation
+        run: sbt makeSite
+
+      - name: Collect artifacts
+        if: ${{ failure() }}
+        run: |
+          mkdir -p /tmp/core_dumps
+          cp core.* *.hs /tmp/core_dumps 2> /dev/null || true
+          mkdir -p /tmp/markdown
+          cp pyrasterframes/target/python/docs/*.md /tmp/markdown 2> /dev/null || true
+
+      - name: Upload core dumps
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: core-dumps
+          path: /tmp/core_dumps
+
+      - name: Upload markdown
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: markdown
+          path: /tmp/markdown
+
+      - name: Upload rf-site
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: rf-site
+          path: docs/target/site

.jvmopts

@@ -0,0 +1,2 @@
+-Xms2g
+-Xmx4g

build.sbt

@@ -78,7 +78,7 @@ lazy val core = project
         ExclusionRule(organization = "com.github.mpilquist")
       ),
       scaffeine,
-      scalatest,
+      sparktestingbase excludeAll ExclusionRule("org.scala-lang.modules", "scala-xml_2.12"),
       `scala-logging`
     ),
     libraryDependencies ++= {

core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala

@@ -32,8 +32,7 @@ class RasterRefIT extends TestEnvironment {
   describe("practical subregion reads") {
     it("should construct a natural color composite") {
       import spark.implicits._
-      def scene(idx: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com" +
-        s"/c1/L8/176/039/LC08_L1TP_176039_20190703_20190718_01_T1/LC08_L1TP_176039_20190703_20190718_01_T1_B$idx.TIF")
+      def scene(idx: Int) = TestData.remoteCOGSingleBand(idx)
 
       val redScene = RFRasterSource(scene(4))
       // [west, south, east, north]