diff --git a/Documentation/GitCheatSheet.pdf b/Documentation/GitCheatSheet.pdf index f3eeee31d9d..c6297107e42 100644 Binary files a/Documentation/GitCheatSheet.pdf and b/Documentation/GitCheatSheet.pdf differ diff --git a/Documentation/GitCheatSheet.tex b/Documentation/GitCheatSheet.tex index 54ff891e502..22b6bc3df25 100644 --- a/Documentation/GitCheatSheet.tex +++ b/Documentation/GitCheatSheet.tex @@ -126,12 +126,12 @@ \subsection*{Topic Branch Lifecycle} \parbox[t]{0.6\cvmain}{% \texttt{git data-upload DataFile.png\\ \# Add DATA\{DataFile.png\} to CMakeLists.txt\\ - git add -- DataFile.png.sha512 CMakeLists.txt\\ + git add -- DataFile.png.cid CMakeLists.txt\\ git commit --amend} } \parbox[t]{0.38\cvmain}{% Upload a testing data input or baseline image to - \href{https://data.kitware.com}{data.kitware.com}.\\ + \href{https://content-link-upload.itk.org}{content-link-upload.itk.org}.\\ Reference the content link in the CMake configuration.\\ Add the content link and updated CMake configuration. } diff --git a/Utilities/GitSetup/setup-girder-api-key b/Utilities/GitSetup/setup-girder-api-key deleted file mode 100755 index cb706ce4941..00000000000 --- a/Utilities/GitSetup/setup-girder-api-key +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -#============================================================================= -# Copyright 2018 Kitware, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -# Run this script to set up the Girder API keys. This is used by the -# Utilities/UploadBinaryData.sh script to perform authenticated uploads of -# binary testing data. - -die() { - echo 1>&2 "$@" ; exit 1 -} - -json_field() { - local key=$1 - local json=$2 - echo $json | awk 'BEGIN { FS="\""; RS="," }; { if ($2 == "'$key'") {print $4} }' -} - - -# Make sure we are inside the repository. -cd "${BASH_SOURCE%/*}" && - -protocol=$(git config -f config --get girder.protocol || echo "https") && -host=$(git config -f config --get girder.host || echo "data.kitware.com") && -site=$(git config -f config --get girder.site || echo "$protocol://$host") - -api_key=$(git config --get girder.api-key || echo '') - -# Tell user about current configuration. -if test -n "$api_key"; then - echo 'The Girder API key is currently set to "'"${api_key:0:5}"'..."' && - read -ep 'Reconfigure Girder API key? [y/N]: ' ans && - if [ "$ans" == "y" ] || [ "$ans" == "Y" ]; then - setup=1 - else - setup='' - fi -else - echo 'A Girder API key is not configured to upload testing data. -' && - read -ep 'Configure Git to contribute testing data to '"$host"'? [Y/n]: ' ans && - if [ "$ans" == "n" ] || [ "$ans" == "N" ]; then - exit 0 - else - setup=1 - fi -fi && - -setup_instructions='Once registered, login to Girder at - - '"$site/#?dialog=login"' - -Then visit - - (upper right) > My account > API keys > Create new key - -To create a new API key. -' - -# Perform setup if necessary. -if test -n "$setup"; then - echo 'Register for an account at - - '"$site/#?dialog=register"' - -'"$setup_instructions" && - read -ep "Girder API key?: " gak && - git config girder.api-key "$gak" && - api_key=$(git config --get girder.api-key) && - echo 'The Girder API key is now configured to - - '"${api_key:0:5}..."' -' -fi && - -# Optionally test GitHub access. -if test -n "$api_key"; then - read -ep 'Test access to Girder via API key? [y/N]: ' ans && - if [ "$ans" == "y" ] || [ "$ans" == "Y" ]; then - echo -n 'Testing Girder access via API key...' - token="" - token_response=$(curl -s -X POST --header 'Content-Length: 0' --header 'Content-Type: application/json' --header 'Accept: application/json' "https://data.kitware.com/api/v1/api_key/token?key=${api_key}&duration=1" || die "Could not retrieve token from API key.") - token=$(json_field "token" "${token_response}") - if test -z "$token"; then - echo 'failed.' && - die 'Could not access Girder via your API key.' - else - echo 'passed.' - fi - fi -fi diff --git a/Utilities/GitSetup/setup-git-aliases b/Utilities/GitSetup/setup-git-aliases index a10fb2baa19..9eeb519535b 100755 --- a/Utilities/GitSetup/setup-git-aliases +++ b/Utilities/GitSetup/setup-git-aliases @@ -14,6 +14,3 @@ git config alias.gerrit-push \ # Useful alias to see what commits are on the current branch with respect to # upstream/master git config alias.prepush 'log --graph --stat upstream/master..' - -# Alias to the script to upload testing data -git config alias.data-upload "!bash Utilities/UploadBinaryData.sh" diff --git a/Utilities/SetupForDevelopment.sh b/Utilities/SetupForDevelopment.sh index cfa79814c01..9ca63bc9fcf 100755 --- a/Utilities/SetupForDevelopment.sh +++ b/Utilities/SetupForDevelopment.sh @@ -27,8 +27,6 @@ Utilities/GitSetup/setup-git-aliases && echo && echo 'Failed to setup origin. Run this again to retry.') && echo && (Utilities/GitSetup/setup-github || echo 'Failed to setup GitHub. Run this again to retry.') && echo && -(Utilities/GitSetup/setup-girder-api-key || - echo 'Failed to setup the Girder API key. Run this again to retry.') && echo && Utilities/GitSetup/tips && Utilities/GitSetup/github-tips diff --git a/Utilities/UploadBinaryData.sh b/Utilities/UploadBinaryData.sh deleted file mode 100755 index 503599173ee..00000000000 --- a/Utilities/UploadBinaryData.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash -#========================================================================== -# -# Copyright NumFOCUS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0.txt -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#==========================================================================*/ - -# Utility functions -print_help() { -cat << EOF -Usage: $0 [--folder-id ] [ ...] - - -Use this script to upload binary testing data for ITK and related projects. - -Binary data, e.g. test input images, are not stored in Git because they -will bloat a Git repository's size. - - -To use the script: - -1. Sign up for an account at https://data.kitware.com -2. Place the binary file at the desired location in the Git repository. -3. Run this script, and pass in the binary file(s) as arguments to script. -4. In test/CMakeLists.txt, use the itk_add_test macro and reference the file - path with \`DATA\` and braces, e.g.: DATA{}. -5. Re-build ITK, and the testing data will be downloaded into the build tree. - - -If the Git `girder.api-key` config or `GIRDER_API_KEY` environmental variable -is not set, a prompt will appear for your username and password. The API key -can be created in the data.kitware.com user account web browser interface. - -The script will authenticate to data.kitware.com, upload the file to your -user account's Public folder, and create a *.sha512 CMake ExternalData -content link file. To specify a different folder, use the --folder flag. -After the content link has been created, add the *.sha512 file to your -git commit. The binary file will be removed from the source tree following -upload. -EOF -} - -die() { - echo "$@" 1>&2; exit 1 -} - -json_field() { - local key=$1 - local json=$2 - echo $json | awk 'BEGIN { FS="\""; RS="," }; { if ($2 == "'$key'") {print $4} }' -} - - -# Parse arguments -help=false -folder_id="" -while test $# -gt 0; -do - opt="$1"; - case "$opt" in - "-h"|"--help") - shift; - help=true - break;; - "-f"|"--folder-id") - shift; - folder_id="$1" - shift;; - *) - break;; - esac -done -binary_files="$@" - -if test "${binary_files}" = "" || $help; then - print_help - exit 1 -fi - - -# Check for required dependencies -if ! type curl > /dev/null; then - die "Please install the curl executable." -fi -if ! type wc > /dev/null; then - die "Please install the wc executable." -fi - - -# Authenticate -token="" -git_config_api_key=$(git config --get girder.api-key || echo '') -if test -n "${git_config_api_key}"; then - token_response=$(curl -s -X POST --header 'Content-Length: 0' --header 'Content-Type: application/json' --header 'Accept: application/json' "https://data.kitware.com/api/v1/api_key/token?key=${git_config_api_key}&duration=1" || die "Could not retrieve token from API key.") - token=$(json_field "token" "${token_response}") -fi -if test -z "${token}" -a -n "${GIRDER_API_KEY}"; then - token_response=$(curl -s -X POST --header 'Content-Length: 0' --header 'Content-Type: application/json' --header 'Accept: application/json' "https://data.kitware.com/api/v1/api_key/token?key=${GIRDER_API_KEY}&duration=1" || die "Could not retrieve token from API key.") - token=$(json_field "token" "${token_response}") -fi -if test -z "${token}"; then - if ! type base64 > /dev/null; then - die "Please install the base64 executable." - fi - echo "Please provide your" - echo "" - echo " https://data.kitware.com" - echo "" - read -p "username: " username - read -p "password: " -s password - basic_content=$(echo -n "${username}:${password}" | base64) - token_response=$(curl -s -X GET --header "Girder-Authorization: Basic ${basic_content}" --header 'Accept: */*' --header 'Host: data.kitware.com' --header 'Referer: https://data.kitware.com' 'https://data.kitware.com/api/v1/user/authentication' || die "Could not retrieve token from username / password.") - token=$(json_field "token" "${token_response}") -fi -if test -z "${token}"; then - die "Could not authenticate to https://data.kitware.com" -fi - - -# Get user / folder -user_id_response=$(curl -s -X GET --header 'Accept: application/json' --header "Girder-Token: ${token}" 'https://data.kitware.com/api/v1/user/me' || die 'Could not get user id.') -user_id=$(json_field "_id" "${user_id_response}") - -if test -z "$folder_id"; then - folder_id_response=$(curl -s -X GET --header 'Accept: application/json' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/folder?parentType=user&parentId=${user_id}&name=Public&limit=3&sort=lowerName&sortdir=1" || die 'Could not get folder id.') - folder_id=$(json_field "_id" "${folder_id_response}") -fi - - -# Upload files and create content links -generated_content_links="" -md5_content_link_conflicts="" -for binary_file in $binary_files; do - if test ! -e $binary_file; then - die "$binary_file does not exist." - fi - item_name=$(basename "$binary_file") - - create_item_response=$(curl -s -X POST --header 'Content-Length: 0' --header 'Content-Type: application/x-www-form-urlencoded' --header 'Accept: application/json' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/item?folderId=${folder_id}&name=${item_name}&description=ITK%20testing%20data%20uploaded%20by%20ITK%2FUtilities%2FUploadBinaryData.sh&reuseExisting=true" || die 'Could not create item.') - item_id=$(json_field "_id" "${create_item_response}") - - size=$(wc -c < "$binary_file" | sed 's/^[[:space:]]*//') - echo "Uploading ${item_name}..." - upload_file_response=$(curl -X POST --data-binary "@${binary_file}" --header 'Content-Type: application/json' --header 'Accept: application/json' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/file?parentType=item&parentId=${item_id}&name=${item_name}&size=${size}" || die 'Could not upload file.') - file_id=$(json_field "_id" "${upload_file_response}") - - curl -s -X GET --output "${binary_file}.sha512" --header 'Accept: text/plain' --header "Girder-Token: ${token}" "https://data.kitware.com/api/v1/file/${file_id}/hashsum_file/sha512" || die 'Could not get file sha512sum.' - generated_content_links="$generated_content_links ${binary_file}.sha512" - if type cmake > /dev/null; then - cmake_sha512sum=$(cmake -E sha512sum "$binary_file" 2> /dev/null) - # If sufficient CMake version, ... - if test $? -eq 0; then - local_sha512=$(echo $cmake_sha512sum | awk '{print $1}') - remote_sha512=$(cat "${binary_file}.sha512") - if test "$local_sha512" != "$remote_sha512"; then - die "Local file hash does not match uploaded file hash." - fi - fi - fi - - md5_content_link="${binary_file}.md5" - if test -e "$md5_content_link"; then - md5_content_link_conflicts="$md5_content_link_conflicts $md5_content_link" - fi - rm $binary_file -done - - -# Recommend next steps -cat << EOF - -Testing data upload complete. - -Now run: - - git add --$generated_content_links - -EOF -if test -n "$md5_content_link_conflicts"; then -cat << EOF -and: - - git rm --$md5_content_link_conflicts -EOF -fi