Patch open function with filesystem-specific interfaces

.github/workflows/superlinter.yml

@@ -0,0 +1,29 @@
+name: Super-Linter
+
+# Run this workflow every time a new commit pushed to your repository
+on: push
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  super-lint:
+    # Name the Job
+    name: Lint code base
+    # Set the type of machine to run on
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      # Runs the Super-Linter action
+      - name: Run Super-Linter
+        uses: github/[email protected]
+        env:
+          DEFAULT_BRANCH: master
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+          LINTER_RULES_PATH: /
+          PYTHON_BLACK_CONFIG_FILE: pyproject.toml
+          PYTHON_ISORT_CONFIG_FILE: pyproject.toml

.gitignore

@@ -136,3 +136,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# MinIO
+tests/data/.minio.sys/

README.md

@@ -18,4 +18,41 @@ Singer tap for the [dBase file format](https://en.wikipedia.org/wiki/.dbf).
 }
 ```
 
+## Filesystems
+
+### Local
+
+Example configuration:
+
+```json
+{
+  "path": "/files/*.dbf",
+  "fs_root": "file://data",
+  "ignore_missing_memofile": true
+}
+```
+
+### S3
+
+You need to install the package with the `s3` extra:
+
+```shell
+pip install 'tap-dbf[s3]'
+```
+
+Example configuration:
+
+```json
+{
+  "path": "/*.dbf",
+  "fs_root": "s3://someKey:someSecret@files?endpoint_url=http%3A//localhost%3A9000",
+  "ignore_missing_memofile": true
+}
+```
+
+## Roadmap
+
+- Google Drive filesystem
+- Dropbox filesystem
+
 [memofile]: https://en.wikipedia.org/wiki/.dbf#Memo_fields_and_the_.DBT_file

docker-compose.yaml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  minio:
+    image: minio/minio
+    ports:
+      - "9000:9000"
+    environment:
+      MINIO_ACCESS_KEY: someKey
+      MINIO_SECRET_KEY: someSecret
+    volumes:
+      - ./tests/data/:/data/
+    command: server /data

pyproject.toml

@@ -9,6 +9,8 @@ license = "Apache 2.0"
 python = ">=3.7,<3.9"
 singer-sdk = "0.2.0"
 dbfread = "^2.0.7"
+fs = "^2.4.13"
+fs-s3fs = {version = "^1.1.1", optional = true}
 
 [tool.poetry.dev-dependencies]
 black = "^21.5b1"
@@ -24,6 +26,9 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry.scripts]
 tap-dbf = 'tap_dbf.tap:cli'
 
+[tool.poetry.extras]
+s3 = ["fs-s3fs"]
+
 [tool.black]
 line-length = 88