From c01950f9efd33baaa2576b210d5d0e3d767fb9f3 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Tue, 8 Nov 2022 17:35:35 -0500 Subject: [PATCH] DOC: Document IPFS data upload Provide motivation and simplified contributor experience. --- .../Contribute/AccountHighlighted.png.cid | 1 - .../CMakeW3ExternalDataUpload.png.cid | 1 + .../Contribute/ClickOnItemHighlighted.png.cid | 1 - .../CreateNewKeyHighlighted.png.cid | 1 - .../FilesUploadedHighlighted.png.cid | 1 - .../FolderInformationHighlighted.png.cid | 1 - .../Contribute/LogInHighlighted.png.cid | 1 - .../Contribute/MyAccountHighlighted.png.cid | 1 - .../Contribute/MyFoldersHighlighted.png.cid | 1 - .../PersonalDataSpaceHighlighted.png.cid | 1 - .../PublicFolderHighlighted.png.cid | 1 - ...PublicFolderInformationHighlighted.png.cid | 1 - .../Contribute/ShowInfoHighlighted.png.cid | 1 - .../Contribute/ShowKeyHighlighted.png.cid | 1 - Documentation/Contribute/UploadBinaryData.rst | 257 +++++++----------- ...UploadHereFilesSelectedHighlighted.png.cid | 1 - .../Contribute/UploadHereHighlighted.png.cid | 1 - 17 files changed, 94 insertions(+), 179 deletions(-) delete mode 100644 Documentation/Contribute/AccountHighlighted.png.cid create mode 100644 Documentation/Contribute/CMakeW3ExternalDataUpload.png.cid delete mode 100644 Documentation/Contribute/ClickOnItemHighlighted.png.cid delete mode 100644 Documentation/Contribute/CreateNewKeyHighlighted.png.cid delete mode 100644 Documentation/Contribute/FilesUploadedHighlighted.png.cid delete mode 100644 Documentation/Contribute/FolderInformationHighlighted.png.cid delete mode 100644 Documentation/Contribute/LogInHighlighted.png.cid delete mode 100644 Documentation/Contribute/MyAccountHighlighted.png.cid delete mode 100644 Documentation/Contribute/MyFoldersHighlighted.png.cid delete mode 100644 Documentation/Contribute/PersonalDataSpaceHighlighted.png.cid delete mode 100644 Documentation/Contribute/PublicFolderHighlighted.png.cid delete mode 100644 Documentation/Contribute/PublicFolderInformationHighlighted.png.cid delete mode 100644 Documentation/Contribute/ShowInfoHighlighted.png.cid delete mode 100644 Documentation/Contribute/ShowKeyHighlighted.png.cid delete mode 100644 Documentation/Contribute/UploadHereFilesSelectedHighlighted.png.cid delete mode 100644 Documentation/Contribute/UploadHereHighlighted.png.cid diff --git a/Documentation/Contribute/AccountHighlighted.png.cid b/Documentation/Contribute/AccountHighlighted.png.cid deleted file mode 100644 index bcc9be608..000000000 --- a/Documentation/Contribute/AccountHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreidf3ycxbhwku7pzclbefwgei7mn5kf5d26qphn7vp22zfdxbyr6z4 diff --git a/Documentation/Contribute/CMakeW3ExternalDataUpload.png.cid b/Documentation/Contribute/CMakeW3ExternalDataUpload.png.cid new file mode 100644 index 000000000..decfecb7a --- /dev/null +++ b/Documentation/Contribute/CMakeW3ExternalDataUpload.png.cid @@ -0,0 +1 @@ +bafkreihtkjkiczpjvzbntaouq3sgqwhsnvcrg2ckqr2cmhlrfjzeshx2mu diff --git a/Documentation/Contribute/ClickOnItemHighlighted.png.cid b/Documentation/Contribute/ClickOnItemHighlighted.png.cid deleted file mode 100644 index 1ab6241be..000000000 --- a/Documentation/Contribute/ClickOnItemHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreidevbhrlmgrou64nxsbpm5cxyuig2ffizfpauk5tigjk7sw7vw3km diff --git a/Documentation/Contribute/CreateNewKeyHighlighted.png.cid b/Documentation/Contribute/CreateNewKeyHighlighted.png.cid deleted file mode 100644 index 7287e40af..000000000 --- a/Documentation/Contribute/CreateNewKeyHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreibroqibiijy23npn7pzysfixi7qd4tuht74q737x5ofvt6unpls7m diff --git a/Documentation/Contribute/FilesUploadedHighlighted.png.cid b/Documentation/Contribute/FilesUploadedHighlighted.png.cid deleted file mode 100644 index bb4220d02..000000000 --- a/Documentation/Contribute/FilesUploadedHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreicgfw2sejrlvhc5jst5jc4ivqhb2tglapyczmvvv4uy3rctkux7wy diff --git a/Documentation/Contribute/FolderInformationHighlighted.png.cid b/Documentation/Contribute/FolderInformationHighlighted.png.cid deleted file mode 100644 index 3100a3939..000000000 --- a/Documentation/Contribute/FolderInformationHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreig3kakq76kmlf6pzr5blys3ong4x4gdkpqh7x5gnq2okdb6g26aoa diff --git a/Documentation/Contribute/LogInHighlighted.png.cid b/Documentation/Contribute/LogInHighlighted.png.cid deleted file mode 100644 index 496dc06b5..000000000 --- a/Documentation/Contribute/LogInHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreia3nr53b73sm73fhsjwntljhdfhlqmxyys63gycci2entmuqhb4sm diff --git a/Documentation/Contribute/MyAccountHighlighted.png.cid b/Documentation/Contribute/MyAccountHighlighted.png.cid deleted file mode 100644 index bd7c3f82c..000000000 --- a/Documentation/Contribute/MyAccountHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreigtmra374etrzoft3xc7efdpzunapthclqcqa5254s5xpvkkhvqbq diff --git a/Documentation/Contribute/MyFoldersHighlighted.png.cid b/Documentation/Contribute/MyFoldersHighlighted.png.cid deleted file mode 100644 index 226726ef2..000000000 --- a/Documentation/Contribute/MyFoldersHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreihp5zbsckvs2wttvxcnhrehl3vjctvy24dwjsclm57uu6aogasj6e diff --git a/Documentation/Contribute/PersonalDataSpaceHighlighted.png.cid b/Documentation/Contribute/PersonalDataSpaceHighlighted.png.cid deleted file mode 100644 index f6157cce2..000000000 --- a/Documentation/Contribute/PersonalDataSpaceHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreiejuuubtr3ybrygle3w2zmziwkf3tjvg3mucaadmf2cu2zkjgi2la diff --git a/Documentation/Contribute/PublicFolderHighlighted.png.cid b/Documentation/Contribute/PublicFolderHighlighted.png.cid deleted file mode 100644 index 96b4c574e..000000000 --- a/Documentation/Contribute/PublicFolderHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreihsokhw7iju3mksfnqogok4y5eopiarrpciqzkcflnrw6lyn5vrji diff --git a/Documentation/Contribute/PublicFolderInformationHighlighted.png.cid b/Documentation/Contribute/PublicFolderInformationHighlighted.png.cid deleted file mode 100644 index c5a9aafb0..000000000 --- a/Documentation/Contribute/PublicFolderInformationHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreiaz53sqsfk2vtet4j7bvxepvzlkqlj7fd7sfuypy74vmvnaoussru diff --git a/Documentation/Contribute/ShowInfoHighlighted.png.cid b/Documentation/Contribute/ShowInfoHighlighted.png.cid deleted file mode 100644 index 55e994a2a..000000000 --- a/Documentation/Contribute/ShowInfoHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreifcomk2hyrdrev3rq5qxsmbbphjqsfvr3cxl25c4rju3uwhfkelsi diff --git a/Documentation/Contribute/ShowKeyHighlighted.png.cid b/Documentation/Contribute/ShowKeyHighlighted.png.cid deleted file mode 100644 index 42ec74def..000000000 --- a/Documentation/Contribute/ShowKeyHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreicxpppekuzj2z4uvmetjplxpalbeqxgokjx737boavu4uzmgscc7a diff --git a/Documentation/Contribute/UploadBinaryData.rst b/Documentation/Contribute/UploadBinaryData.rst index 3d1e7835e..d882f6904 100644 --- a/Documentation/Contribute/UploadBinaryData.rst +++ b/Documentation/Contribute/UploadBinaryData.rst @@ -6,202 +6,131 @@ Upload Binary Data Motivation ---------- -Since every local Git_ repository contains a copy of the entire project history, -it is important to avoid adding large binary files directly to the repository. -Large binary files added and removed throughout a project's history will cause -the repository to become bloated, take up too much disk space, require excessive -time and bandwidth to download, etc. - -A `solution to this problem`_ which has been adopted by this project is to store -binary files, such as images, in a separate location outside the Git repository, -then download the files at build time with CMake_. - -A "content link" file contains an identifying `SHA512 hash`_. The content link -is stored in the Git_ repository at the path where the file would exist, but -with a ".sha512" extension appended to the file name. CMake will find these -content link files at *build* time, download them from a list of server -resources, and create symlinks or copies of the original files at the +Since every local Git_ repository contains a copy of the entire project +history, it is important to avoid adding large binary files directly to the +repository. Large binary files added and removed throughout a project's +history will cause the repository to become bloated, take up too much disk +space, require excessive time and bandwidth to download, etc. + +A `solution to this problem`_ which has been adopted by this project, is to +store binary files such as images in a separate location outside the Git +repository. Then, download the files at build time with CMake_. + +A "content link" file contains an identifying `Content Identifier (CID)`_. The +content link is stored in the Git_ repository at the path where the file would +exist, but with a `.cid`` extension appended to the file name. CMake will +find these content link files at *build* time, download them from a list of +server resources, and create symlinks or copies of the original files at the corresponding location in the *build tree*. +The `Content Identifier (CID)`_ is self-describing hash following the +`multiformats`_ standard created by the Interplanetary Filesystem (`IPFS`_) +community. A file with a CID for its filename is content-verifable. Locating +files according to their CID makes content-addressed, as opposed to +location-addressed, data exchange possible. This practice is the foundation of +the decentralized web, also known as the dWeb or Web3. By adopting Web3, we +gain: + +- Permissionless data uploads +- Robust, redundant storage +- Local and peer-to-peer storage +- Scalability +- Sustainability + +Contributors to the examples upload their data through an easy-to-use, +permissionless, free service, `web3.storage`_. + +Data used in the examples Git repository is periodically tracked in a +`dedicated Datalad repository`_ and stored across redundant locations so it +can be retrieved from any of the following: + +- Local `IPFS`_ nodes +- Peer `IPFS`_ nodes +- `web3.storage`_ +- `estuary.tech`_ +- `pinata.cloud`_ +- Kitware's HTTP Server + Prerequisites ------------- -The `data.kitware.com`_ server is an ITK community resource where any +`web3.storage`_ is a decentralized IPFS storage provider where any ITK community member can upload binary data files. There are two methods available to upload data files: -1. The `Girder web interface`_. -2. The `girder-cli` command line executable that comes with the - girder-client_ Python package. - -Before uploading data, please visit `data.kitware.com`_ and -register for an account. +1. The CMake ExternalData Web3 upload browser interface. +2. The `w3` command line executable that comes with the + `@web3-storage/w3`_ Node.js NPM package. -Once files have been uploaded to your account, they will be publicly -available and accessible since data is content addressed. At release time, -the release manager will upload and archive repository data references in the -`ITK collection`_ and other redundant storage locations. +Once files have been uploaded to your account, they will be publicly available +and accessible since data is content addressed on the IPFS peer-to-peer +network. At release time, the release manager will upload and archive +repository data references in other redundant storage locations. Upload Via the Web Interface ---------------------------- +Use the `CMake ExternalData Web3 Upload`_ tool to upload your data to the +InterPlanetary Filesystem and download the corresponding CMake content link +file. -.. figure:: LogInHighlighted.png - :alt: Log in welcome page - :align: center - :width: 400 - - After logging in, you will be presented with the welcome page. Click on the - **personal data space** link. - -.. figure:: PersonalDataSpaceHighlighted.png - :alt: Personal data space - :align: center - :width: 400 - - Next, select the **Public** folder of your personal data space. - -.. figure:: PublicFolderHighlighted.png - :alt: Public folder +.. figure:: CMakeW3ExternalDataUpload.png + :alt: CMake ExternalData Web3 Upload :align: center - :width: 400 - - Click the **green upload button**. + :width: 500 + :target: https://cmake-w3-externaldata-upload.on.fleek.co/ -.. figure:: UploadHereHighlighted.png - :alt: The Upload files dialog - :align: center - :width: 400 + `CMake ExternalData Web3 Upload`_ - Click the **Browse or drop files** to select the files to upload. +Add the file to the examples repository in your example's directory. Next time +CMake configuration runs, it will find the new content link. During the next +project build, the data file corresponding to the content link will be +downloaded into the build tree. -.. figure:: UploadHereFilesSelectedHighlighted.png - :alt: The Upload files dialog with files selected - :align: center - :width: 400 +Upload Via CMake and Node.js CLI +-------------------------------- - Click **Start Upload** to upload the file to the server. +Install the `w3` CLI with the `@web3-storage/w3`_ `Node.js`_ package: -Next, proceed to `Download the Content Link`_. +.. code-block:: shell -Upload Via Python Script ------------------------- + $ npm install --location=global @web3-storage/w3 -A Python script to upload files from the command line, `girder-cli`, is -available with the girder-client_ python package. To install it:: +Login in and create an API token at `web3.storage`_ then pass it into `w3 token`: - python -m pip install girder-client +.. code-block:: shell -To upload files with the `girder-cli` script, we need to obtain an API key and -a parent folder id from the web interface. + $ w3 token + ? Paste your API token for api.web3.storage › -.. figure:: MyAccountHighlighted.png - :alt: My account link - :align: center - :width: 400 + ⁂ API token saved - After logging in, select **My account** from the user drop down. - -.. figure:: AccountHighlighted.png - :alt: API key tab - :align: center - :width: 400 - - Next, select the **API keys** tab. - -.. figure:: CreateNewKeyHighlighted.png - :alt: Create new key - :align: center - :width: 400 - - Create a new API key if one is not available. - -.. figure:: ShowKeyHighlighted.png - :alt: Create new key - :align: center - :width: 400 - - The **show** link will show the key, which can be copied into the command - line. - -.. figure:: MyFoldersHighlighted.png - :alt: My Folders link - :align: center - :width: 400 +Create an `w3externaldata` bash/zsh function: - Next, select **My Folders** from the user drop down. - -.. figure:: PersonalDataSpaceHighlighted.png - :alt: Personal data space - :align: center - :width: 400 - - Next, select the **Public** folder of your personal data space. - -.. figure:: PublicFolderInformationHighlighted.png - :alt: Public folder information - :align: center - :width: 400 - - Click the **i** button for information about the folder. - -.. figure:: FolderInformationHighlighted.png - :alt: Public folder information modal - :align: center - :width: 400 - - The **Unique ID** can be copied into the command line. - -Use both the API key and the folder ID when calling `girder-cli`. For example, - -.. code-block:: bash - - girder-cli \ - --api-key 12345ALongSetOfCharactersAndNumbers \ - --api-url https://data.kitware.com/api/v1 \ - upload \ - 58becaee8d777f0aefede556 \ - /tmp/cthead1.png - -Next, proceed to `Download the Content Link`_. - -Download the Content Link -------------------------- - -.. figure:: FilesUploadedHighlighted.png - :alt: File has been uploaded - :align: center - :width: 400 - - Click on the file that has been uploaded. - -.. figure:: ClickOnItemHighlighted.png - :alt: Item has been clicked - :align: center - :width: 400 - - Click on the **i** button for further information. - -.. figure:: ShowInfoHighlighted.png - :alt: File information - :align: center - :width: 400 +.. code-block:: shell - Finally, click on the **Download key file** icon to download the key file. + $ function w3externaldata() { w3 put $1 --no-wrap | tail -n 1 | awk -F "/ipfs/" '{print $2}' | tee $1.cid } -Move the content link file to the source tree at the location -where the actual file is desired in the build tree. Stage the new file to -your commit:: +Call the function with the file to be uploaded. This command will generate the +`.cid` content link: - git add -- path/to/file.sha512 +.. code-block:: shell + $ w3externaldata + # Packed 1 file (0.3MB) + ⁂ Stored 1 file + bafkreifpfhcc3gc7zo2ds3ktyyl5qrycwisyaolegp47cl27i4swxpa2ey .. _CMake: https://cmake.org/ .. _Git: https://git-scm.com/ -.. _Insight Community mailing list: https://itk.org/mailman/listinfo/community -.. _ITK collection: https://data.kitware.com/#collection/57b5c9e58d777f126827f5a1 -.. _SHA512 hash: https://en.wikipedia.org/wiki/SHA-2 -.. _data.kitware.com: https://data.kitware.com/ -.. _Girder web interface: https://girder.readthedocs.io/en/latest/user-guide.html -.. _girder-client: https://girder.readthedocs.io/en/latest/python-client.html#the-command-line-interface .. _solution to this problem: https://blog.kitware.com/cmake-externaldata-using-large-files-with-distributed-version-control/ +.. _Content Identifier (CID): https://proto.school/anatomy-of-a-cid +.. _multiformats: https://multiformats.io/ +.. _IPFS: https://ipfs.io/ +.. _web3.storage: https://web3.storage/ +.. _dedicated Datalad repository: https://github.com/InsightSoftwareConsortium/ITKSphinxExamplesData +.. _estuary.tech: https://estuary.tech +.. _pinata.cloud: https://pinata.cloud +.. _CMake ExternalData Web3 Upload: https://cmake-w3-externaldata-upload.on.fleek.co/ +.. _@web3-storage/w3: https://www.npmjs.com/package/@web3-storage/w3 +.. _Node.js: https://nodejs.org/ diff --git a/Documentation/Contribute/UploadHereFilesSelectedHighlighted.png.cid b/Documentation/Contribute/UploadHereFilesSelectedHighlighted.png.cid deleted file mode 100644 index ee7093111..000000000 --- a/Documentation/Contribute/UploadHereFilesSelectedHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreigz7nbgfaz42liwa5dqsgux5ahjytetwpdtqjulzadbxxlmlzmn6y diff --git a/Documentation/Contribute/UploadHereHighlighted.png.cid b/Documentation/Contribute/UploadHereHighlighted.png.cid deleted file mode 100644 index 2d1bc1535..000000000 --- a/Documentation/Contribute/UploadHereHighlighted.png.cid +++ /dev/null @@ -1 +0,0 @@ -bafkreihr2vl46wcfe23ip33tz4c52ykvlhhyxa6k57tb3hic6xnkvbzszi