Added some S3 documentation. Closes #442

doc/local-build.sh

@@ -36,6 +36,7 @@ setup_venv() {
   source "${venv_dir}/bin/activate" || die "could not activate virtualenv"
   pip install 'Sphinx==1.6.7' \
        'breathe' \
+       'sphinx_tabs' \
        'sphinx_rtd_theme' || die "could not install doc dependencies"
 }
 

doc/source/conf.py

@@ -45,7 +45,8 @@
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.intersphinx',
-    'breathe'
+    'breathe',
+    'sphinx_tabs.tabs'
 ]
 
 if readthedocs:
@@ -187,4 +188,4 @@
 # -- Custom setup -----------------------------------------------------------
 
 def setup(app):
-    app.add_stylesheet('custom.css')
+    app.add_stylesheet('custom.css')

doc/source/gensidebar.py

@@ -92,6 +92,7 @@ def write_api(project, desc, rst_page):
     write('Object Management', 'tiledb101/object-management')
     write('Storage Backends', 'tiledb101/storage-backends')
     write('Virtual Filesystem', 'tiledb101/virtual-filesystem')
+    write('Working with S3', 'tiledb101/s3')
     write('Language Bindings', 'tiledb101/language-bindings')
     write('Concurrency', 'tiledb101/concurrency')
     write('Consistency', 'tiledb101/consistency')

doc/source/tiledb101/s3.rst

@@ -0,0 +1,134 @@
+Working with S3
+===============
+
+This is a simple guide that demonstrates how to use TileDB on S3-compatible
+backends. We will first explain how to set up an AWS account and configure
+TileDB for `AWS S3 <https://aws.amazon.com/s3/>`_, and then show how to 
+set up and use `minio <https://minio.io>`_ locally.
+
+AWS
+---
+
+First, we need to set up an AWS account and generate access keys.
+
+1. Create a new `AWS account <https://portal.aws.amazon.com/billing/signup#/start>`_.
+
+2. Visit the `AWS console <https://aws.amazon.com/console/>`_ and sign in.
+
+3. On the AWS console, click on the ``Services`` drop-down menu and select ``Storage->S3``. You can create S3 buckets there.
+
+4. On the AWS console, click on the ``Services`` drop-down menu and select ``Security, Identity & Compliance->IAM``.
+
+5. Click on ``Users`` from the left-hand side menu, and then click on the ``Add User`` button. Provide the email or username of the user you wish to add, select the ``Programmatic Access`` checkbox and click on ``Next: Permissions``.
+
+6. Click on the ``Attach existing policies directly`` button, search for the S3-related policies and add the policy of your choice (e.g., full-access, read-only, etc.). Click on ``Next`` and then ``Create User``.
+
+7. Upon successful creation, the next page will show the user along with two keys: ``Access key ID`` and ``Secret access key``. Write down both these keys.
+
+8. Export these keys to your environment from a console.
+
+.. tabs::
+
+   .. group-tab:: Linux
+
+      | export AWS_ACCESS_KEY_ID=<your-access-key-id>
+      | export AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
+
+   .. group-tab:: Mac OSX
+
+      | export AWS_ACCESS_KEY_ID=<your-access-key-id>
+      | export AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
+
+   .. group-tab:: Windows
+
+      | # Powershell
+      | $env:AWS_ACCESS_KEY_ID = "<your-access-key-id>"
+      | $env:AWS_SECRET_ACCESS_KEY = "<your-secret-access-key>"
+      |
+      | # cmd.exe
+      | set AWS_ACCESS_KEY_ID=<your-access-key-id>
+      | set AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
+
+Now you are ready to start writing TileDB programs! When creating a TileDB
+context or a VFS object, you need to set up a configuration object with the
+following parameters for AWS S3 (supposing that your S3 buckets are on region
+``us-east-1`` - you can use set an arbitrary region):
+
+.. tabs::
+
+   .. code-tab:: c
+
+         tiledb_config_t *config = NULL;
+         tiledb_config_t *error = NULL;
+         tiledb_config_create(&config, &error);
+
+         tiledb_config_set(config, "vfs.s3.scheme", "https", &error);
+         tiledb_config_set(config, "vfs.s3.region", "us-east-1", &error);
+         tiledb_config_set(config, "vfs.s3.endpoint_override", "", &error);
+         tiledb_config_set(config, "vfs.s3.use_virtual_addressing", "true", &error);
+
+
+   .. code-tab:: c++
+
+         tiledb::Config config;
+
+         config["vfs.s3.scheme"] = "https";
+         config["vfs.s3.region"] = "us-east-1";
+         config["vfs.s3.endpoint_override"] = "";
+         config["vfs.s3.use_virtual_addressing"] = true;
+
+   .. code-tab:: py
+
+         config = tiledb.Config()
+
+         config["vfs.s3.scheme"] = "https";
+         config["vfs.s3.region"] = "us-east-1";
+         config["vfs.s3.endpoint_override"] = "";
+         config["vfs.s3.use_virtual_addressing"] = "true";
+
+.. note::
+    The above configuration parameters are currently set as shown in TileDB **by default**.
+    However, we suggest to always check whether the default values are the desired ones
+    for your application.
+
+minio
+-----
+
+`minio <https://minio.io>`_ is a lightweight S3-compliant object-store.
+Although it has many nice features, here we focus only on local deployment,
+which is very useful if you wish to quickly test your TileDB-S3 programs
+locally. See the `minio quickstart guide <https://docs.minio.io/docs/minio-quickstart-guide>`_
+for installation instructions.
+
+Once you get minio server running, you need to set the S3 configurations
+as follows (below, ``<port>`` stands for the port on which you are running the minio server):
+
+.. tabs::
+
+   .. code-tab:: c
+
+         tiledb_config_t *config = NULL;
+         tiledb_config_t *error = NULL;
+         tiledb_config_create(&config, &error);
+
+         tiledb_config_set(config, "vfs.s3.scheme", "http", &error);
+         tiledb_config_set(config, "vfs.s3.endpoint_override", "localhost:<port>", &error);
+         tiledb_config_set(config, "vfs.s3.use_virtual_addressing", "false", &error);
+
+
+   .. code-tab:: c++
+
+         tiledb::Config config;
+
+         config["vfs.s3.scheme"] = "http";
+         config["vfs.s3.endpoint_override"] = "localhost:<port>";
+         config["vfs.s3.use_virtual_addressing"] = false;
+
+   .. code-tab:: py
+
+         config = tiledb.Config()
+
+         config["vfs.s3.scheme"] = "http";
+         config["vfs.s3.endpoint_override"] = "localhost:<port>";
+         config["vfs.s3.use_virtual_addressing"] = "false";
+