From 8961457247f48ed7f55f6da347854f1cd6656d55 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 5 Dec 2019 15:21:16 +0000 Subject: [PATCH] [FAB-16711] Update azure pipeline Merge npm publish changes from release-1.4 and delete unused Jenkins scripts Signed-off-by: James Taylor --- ci/azure-pipelines.yml | 85 +++--- rush.json | 2 +- tools/scripts/Jenkins_Scripts/CI_Script.sh | 258 ------------------ .../Jenkins_Scripts/Publish_NPM_Modules.sh | 87 ------ 4 files changed, 39 insertions(+), 393 deletions(-) delete mode 100755 tools/scripts/Jenkins_Scripts/CI_Script.sh delete mode 100755 tools/scripts/Jenkins_Scripts/Publish_NPM_Modules.sh diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index 12f900a3..a49eabe8 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -4,7 +4,7 @@ # # fabric-chaincode-node azure pipeline configuration. # -name: RELEASE-$(Date:yyyyMMdd)$(Rev:.rrr) +name: $(SourceBranchName)-$(Date:yyyyMMdd)$(Rev:.rrr) # Daily build for final quality # cf https://crontab.guru/#0_23_*_*_* @@ -17,14 +17,13 @@ schedules: always: true -# Trigger on the master and the release-1.4 branches only -# Stndard Pull Request and Merge builds +# Trigger on the master branch +# Standard Pull Request and Merge builds # Any git tag will also trigger a build that should do publishing trigger: branches: include: - 'master' - - 'release-1.4' tags: include: - '*' @@ -41,6 +40,8 @@ variables: value: fabric-chaincode-node - name: pipeline value: ci + - name: node_version_spec + value: '12.x' # Build on Ubuntu pool: @@ -51,24 +52,20 @@ pool: # Keeping it simple and explict whilst we gain experience stages: - # output some basic information for reference - # Not for use every time, but this can be useful template to keep - # - stage: info - # jobs: - # - job: env_info - # steps: - # - script: | - # env | sort - # Primary stage of building, linting and unit tests - stage: Build_and_Test jobs: + # output some basic information for reference + - job: displayenv + steps: + - script: | + env | sort - job: main displayName: 'Build, lint, UT' steps: - task: NodeTool@0 inputs: - versionSpec: '10.x' + versionSpec: $(node_version_spec) - script: | set -ev node common/scripts/install-run-rush.js install @@ -131,7 +128,7 @@ stages: steps: - task: NodeTool@0 inputs: - versionSpec: '10.x' + versionSpec: $(node_version_spec) - task: DownloadPipelineArtifact@2 inputs: artifact: nodeenv-docker-image @@ -191,7 +188,7 @@ stages: steps: - task: NodeTool@0 inputs: - versionSpec: '10.x' + versionSpec: $(node_version_spec) - task: DownloadPipelineArtifact@2 inputs: artifact: node-tgz @@ -204,8 +201,7 @@ stages: npm audit displayName: 'Run npm audit' - # Publish the snapshot images etc. - # Original source of these is https://github.com/hyperledger/ci-management/blob/master/jjb/fabric-chaincode-java/shell/include-raw-publish-docker-jar.sh + # Publish the snapshot images - job: snapshot_publish condition: and(succeeded(),eq(variables['Build.Reason'], 'IndividualCI')) dependsOn: fvt @@ -219,50 +215,45 @@ stages: NEXUS_USERNAME: $(nexus-user) NEXUS_PASSWORD: $(nexus-password) NEXUS_URL: nexus3.hyperledger.org:10003/hyperledger - - script: | - node common/scripts/install-run-rush.js install - cd ${BUILD_SOURCEBRANCH}/docs/apidocs - CHAINCODE_NODE_COMMIT=$(git rev-parse --short HEAD) - echo "---------> CHAINCODE_NODE_COMMIT:" $CHAINCODE_NODE_COMMIT - TARGET_REPO=$CHAINCODE_NODE_USERNAME.github.io.git - git config --global user.email "fabric.shim@gmail.com" - git config --global user.name "fabric-shim" - git clone https://github.com/$CHAINCODE_NODE_USERNAME/$TARGET_REPO - - # build docs - DOCS_ROOT=$CHAINCODE_NODE_USERNAME.github.io gulp -f docs.js docs - - cd $CHAINCODE_NODE_USERNAME.github.io - git add . - git commit -m "CHAINCODE_NODE commit - $CHAINCODE_NODE_COMMIT" - git config remote.gh-pages.url https://$CHAINCODE_NODE_USERNAME:$CHAINCODE_NODE_PASSWORD@github.com/$CHAINCODE_NODE_USERNAME/$TARGET_REPO - - # Push API docs to Target repository - git push gh-pages master - - env: - CHAINCODE_NODE_USERNAME: fabric-shim - + displayName: 'docker push to nexus' # Publish a new version, triggered by a git tag - stage: Publish_tag + dependsOn: Build_and_Test condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) jobs: - job: update_version steps: - script: | - set -ev env | sort echo "Update the version number" + VERSION=$(jq '.version' $(Build.SourcesDirectory)/package.json | sed -r "s/\"([0-9]?[0-9]\.[0-9]?[0-9]\.[0-9]?[0-9]).*/\1/") + echo Version is :${VERSION}: echo "Make sure release notes are present" echo "Make sure change history is present" - job: npm_publish + displayName: 'npm publish' steps: + - task: NodeTool@0 + inputs: + versionSpec: $(node_version_spec) + - download: current + artifact: node-tgz - script: | - set -ev - echo "Setup .npmrc" - echo "Use pipeline secret to login" - echo "publish" + touch $(Agent.TempDirectory)/.npmrc + echo '##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]$(Agent.TempDirectory)/.npmrc' + displayName: 'create user .npmrc file' + - script: | + npm config set registry https://registry.npmjs.org/ + npm config ls + displayName: 'set npmjs registry' + - task: npmAuthenticate@0 + inputs: + workingFile: '$(Agent.TempDirectory)/.npmrc' + customEndpoint: 'fabric-chainode-node-npm' + - script: | + find $(Pipeline.Workspace)/node-tgz/ -maxdepth 1 -type f -name 'fabric-*.tgz' -exec npm publish {} --tag latest \; + displayName: 'npm publish' - job: jsdoc_publish steps: - script: | diff --git a/rush.json b/rush.json index 683c6161..09dda8da 100644 --- a/rush.json +++ b/rush.json @@ -62,7 +62,7 @@ * Specify a SemVer range to ensure developers use a Node.js version that is appropriate * for your repo. */ - "nodeSupportedVersionRange": ">=10.13.0 <11.0.0", + "nodeSupportedVersionRange": ">=10.13.0 <12.14.0", /** * Odd-numbered major versions of Node.js are experimental. Even-numbered releases * spend six months in a stabilization period before the first Long Term Support (LTS) version. diff --git a/tools/scripts/Jenkins_Scripts/CI_Script.sh b/tools/scripts/Jenkins_Scripts/CI_Script.sh deleted file mode 100755 index f5ee0c0d..00000000 --- a/tools/scripts/Jenkins_Scripts/CI_Script.sh +++ /dev/null @@ -1,258 +0,0 @@ -#!/bin/bash -# -# Copyright IBM Corp All Rights Reserved -# -# SPDX-License-Identifier: Apache-2.0 -# - -export BASE_FOLDER=$WORKSPACE/gopath/src/github.com/hyperledger -export NEXUS_URL=nexus3.hyperledger.org:10001 -export ORG_NAME="hyperledger/fabric" -export CONTAINER_LIST=(ca orderer peer0.org1) - -# error check -err_and_exit() { - echo -e "\033[31m $1" "\033[0m" - docker images | grep hyperledger && docker ps -a - # Write ca, orderer, peer logs - for CONTAINER in ${CONTAINER_LIST[*]}; do - docker logs $CONTAINER.example.com >& $CONTAINER.log - done - # Write cli container logs into cli.log file - docker logs cli >& cli.log - # Write couchdb container logs into couchdb.log file - docker logs couchdb >& couchdb.log - # Copy debug log - cp /tmp/fabric-shim/logs/*.log $WORKSPACE - exit 1 -} - -Parse_Arguments() { - while [ $# -gt 0 ]; do - case $1 in - --env_Info) - env_Info - ;; - --clean_Environment) - clean_Environment - ;; - --pull_Docker_Images) - pull_Docker_Images - ;; - --e2e_Tests) - e2e_Tests - ;; - --fullBuild) - fullBuild - ;; - --publish_NpmModules) - publish_NpmModules - ;; - --publish_ApiDocs) - publish_ApiDocs - ;; - --publish_Nodeenv_Image) - publish_Nodeenv_Image - ;; - esac - shift - done -} - -clean_Environment() { -echo "-----------> Clean Docker Containers & Images, unused/lefover build artifacts" -function clearContainers () { - CONTAINER_IDS=$(docker ps -aq) - if [ -z "$CONTAINER_IDS" ] || [ $CONTAINER_IDS == " " ]; then - echo "---- No containers available for deletion ----" - else - docker rm -f $CONTAINER_IDS || true - docker ps -a - fi -} - -function removeUnwantedImages() { - DOCKER_IMAGES_SNAPSHOTS=$(docker images | grep snapshot | grep -v grep | awk '{print $1":" $2}') - - if [ -z "$DOCKER_IMAGES_SNAPSHOTS" ] || [ $DOCKER_IMAGES_SNAPSHOTS == " " ]; then - echo "---- No snapshot images available for deletion ----" - else - docker rmi -f $DOCKER_IMAGES_SNAPSHOTS || true - fi - DOCKER_IMAGE_IDS=$(docker images | grep -v 'couchdb\|kafka\|zookeeper\|cello' | awk '{print $3}') - - if [ -z "$DOCKER_IMAGE_IDS" ] || [ "$DOCKER_IMAGE_IDS" = " " ]; then - echo "---- No images available for deletion ----" - else - docker rmi -f $DOCKER_IMAGE_IDS || true - docker images - fi -} - -# Delete nvm prefix & then delete nvm -rm -rf $HOME/.node-gyp/ $HOME/.npm/ $HOME/.npmrc || true - -ls -l /tmp/fabric-shim/chaincode/hyperledger/fabric -ls -l /tmp/fabric-shim/chaincode/hyperledger - -# Remove /tmp/fabric-shim -docker run -v /tmp:/tmp library/alpine rm -rf /tmp/fabric-shim || true - -ls -l /tmp/fabric-shim/chaincode/hyperledger/fabric -ls -l /tmp/fabric-shim/chaincode/hyperledger - -# remove tmp/hfc and hfc-key-store data -rm -rf /home/jenkins/npm /tmp/fabric-shim /tmp/hfc* /tmp/npm* /home/jenkins/kvsTemp /home/jenkins/.hfc-key-store || true - -rm -rf /var/hyperledger/* - -clearContainers -removeUnwantedImages -} - -env_Info() { - # This function prints system info - - #### Build Env INFO - echo "-----------> Build Env INFO" - # Output all information about the Jenkins environment - uname -a - cat /etc/*-release - env - gcc --version - docker version - docker info - docker-compose version - pgrep -a docker - docker images - docker ps -a -} - -# pull fabric, ca images from nexus -pull_Docker_Images() { - for IMAGES in peer orderer tools ccenv ca baseos; do - docker pull $NEXUS_URL/$ORG_NAME-$IMAGES:${IMAGE_TAG} > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo -e "\033[31m FAILED to pull docker images" "\033[0m" - exit 1 - fi - echo "\033[32m ----------> pull $IMAGES image" "\033[0m" - echo - docker tag $NEXUS_URL/$ORG_NAME-$IMAGES:${IMAGE_TAG} $ORG_NAME-$IMAGES - docker tag $NEXUS_URL/$ORG_NAME-$IMAGES:${IMAGE_TAG} $ORG_NAME-$IMAGES:${ARCH}-${VERSION} - docker tag $NEXUS_URL/$ORG_NAME-$IMAGES:${IMAGE_TAG} $ORG_NAME-$IMAGES:${VERSION} - docker rmi -f $NEXUS_URL/$ORG_NAME-$IMAGES:${IMAGE_TAG} - done - echo - docker images | grep hyperledger/fabric -} - -# Install NPM -install_Npm() { - echo "-------> ARCH:" $ARCH - if [[ $ARCH == "amd64" ]]; then - echo -e "\033[32m npm version ------> $(npm -v)" "\033[0m" - echo -e "\033[32m node version ------> $(node -v)" "\033[0m" - npm install -g gulp - fi -} - -# run sdk e2e tests -e2e_Tests() { - - echo -e "\033[32m Execute Chaincode Node Integration Tests" "\033[0m" - cd ${WORKSPACE}/gopath/src/github.com/hyperledger/fabric-chaincode-node - - # Install NPM before start the tests - install_Npm - - echo "#################################################" - echo -e "\033[32m ------> Run Headless Tests" "\033[0m" - echo "#################################################" - - gulp test-headless || err_and_exit "ERROR!!! test-headless failed" - - echo "#################################################################" - echo -e "\033[32m ------> Run Integration and Scenario Tests" "\033[0m" - echo "#################################################################" - - gulp docker-image-build || err_and_exit "ERROR!!! docker-image-build failed" - docker images | grep hyperledger && docker ps -a - - gulp verdaccio-start || err_and_exit "ERROR!!! verdaccio-start failed" - - DEVMODE=false gulp channel-init || err_and_exit "ERROR!!! channel-init failed" - gulp test-e2e || err_and_exit "ERROR!!! test-e2e failed" - - echo "###############################################" - echo -e "\033[32m ------> Run DevMode tests" "\033[0m" - echo "###############################################" - - DEVMODE=true gulp channel-init || err_and_exit "ERROR!!! channel-init failed" - gulp test-devmode || err_and_exit "ERROR!!! test-devmode failed" - - echo "###############################################" - echo -e "\033[32m ------> Run InvCtrl tests" "\033[0m" - echo "###############################################" - - DEVMODE=true gulp channel-init || err_and_exit "ERROR!!! channel-init failed" - gulp test-invctrl || err_and_exit "ERROR!!! test-invctrl failed" - - echo "#############################################" - echo -e "\033[32m ------> Tests Complete" "\033[0m" - echo "#############################################" -} - -# Publish nodeenv docker image after successful merge -publish_Nodeenv_Image() { - echo - echo -e "\033[32m -----------> Publish nodeenv docker image" "\033[0m" - # 10003 points to docker.snapshot - DOCKER_REPOSITORY=nexus3.hyperledger.org:10003 - # SETTINGS_FILE stores the nexus credentials - USER=$(xpath -e "//servers/server[id='$DOCKER_REPOSITORY']/username/text()" "$SETTINGS_FILE") - PASS=$(xpath -e "//servers/server[id='$DOCKER_REPOSITORY']/password/text()" "$SETTINGS_FILE") - docker login $DOCKER_REPOSITORY -u "$USER" -p "$PASS" - # tag nodeenv latest tag to nexus3 repo - docker tag hyperledger/fabric-nodeenv $DOCKER_REPOSITORY/hyperledger/fabric-nodeenv:$ARCH-latest - docker tag hyperledger/fabric-nodeenv $DOCKER_REPOSITORY/hyperledger/fabric-nodeenv:$ARCH-$VERSION-stable - # Push nodeenv image to nexus3 docker.snapshot - docker push $DOCKER_REPOSITORY/hyperledger/fabric-nodeenv:$ARCH-latest - docker push $DOCKER_REPOSITORY/hyperledger/fabric-nodeenv:$ARCH-$VERSION-stable - docker images -} - -# Publish npm modules after successful merge on amd64 -publish_NpmModules() { - echo - echo -e "\033[32m -----------> Publish npm modules from amd64" "\033[0m" - ./Publish_NPM_Modules.sh -} - -# Publish NODE_SDK API docs after successful merge on amd64 -publish_ApiDocs() { - echo - echo -e "\033[32m -----------> Publish NODE_SDK API docs after successful merge on amd64" "\033[0m" - ${WORKSPACE}/gopath/src/github.com/hyperledger/fabric-chaincode-node/docs/apidocs/Publish_API_Docs.sh -} - -fullBuild() { - - echo -e "\033[32m Execute Chaincode Node Full Build" "\033[0m" - cd ${WORKSPACE}/gopath/src/github.com/hyperledger/fabric-chaincode-node - - # Install NPM before start the tests - install_Npm - - node common/scripts/install-run-rush.js install - node common/scripts/install-run-rush.js update - node common/scripts/install-run-rush.js rebuild - - # equivalent to the test-e2e - node common/scripts/install-run-rush.js start-fabric - node common/scripts/install-run-rush.js start-verdaccio - node common/scripts/install-run-rush.js test:fv - -} - -Parse_Arguments $@ diff --git a/tools/scripts/Jenkins_Scripts/Publish_NPM_Modules.sh b/tools/scripts/Jenkins_Scripts/Publish_NPM_Modules.sh deleted file mode 100755 index c88c77c9..00000000 --- a/tools/scripts/Jenkins_Scripts/Publish_NPM_Modules.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -e -# -# Copyright IBM Corp All Rights Reserved -# -# SPDX-License-Identifier: Apache-2.0 -# - -###################################### -# Publish npm module after merge commit -# npm publish --tag $CURRENT_TAG -###################################### - -npmPublish() { - if [[ "$CURRENT_TAG" = *"skip"* ]]; then - echo "----> Don't publish npm modules on skip tag" - elif [[ "$CURRENT_TAG" = *"unstable"* ]]; then - echo - echo "======> Incremented UNSTABLE VERSION:" $UNSTABLE_INCREMENT - - # Get last digit of the unstable version of $CURRENT_TAG - UNSTABLE_INCREMENT=$(echo $UNSTABLE_INCREMENT| rev | cut -d '.' -f 1 | rev) - echo "======> UNSTABLE_INCREMENT:" $UNSTABLE_INCREMENT - - # Append last digit with the package.json version - export UNSTABLE_INCREMENT_VERSION=$RELEASE_VERSION.$UNSTABLE_INCREMENT - echo "======> UNSTABLE_INCREMENT_VERSION:" $UNSTABLE_INCREMENT_VERSION - - # Replace existing version with $UNSTABLE_INCREMENT_VERSION - sed -i 's/\(.*\"version\"\: \"\)\(.*\)/\1'$UNSTABLE_INCREMENT_VERSION\"\,'/' package.json - - # Update the dependencies with that refer to any fabric-* module to be the matching number - sed -i 's/\(.*\"fabric-.*\"\)\(2.0.0-snapshot\)/\1'$UNSTABLE_INCREMENT_VERSION'/' package.json - npm publish --tag $CURRENT_TAG - - else - # Publish node modules on latest tag - echo -e "\033[32m ======> RELEASE_VERSION: $RELEASE_VERSION" "\033[0m" - echo - echo -e "\033[32m ======> CURRENT_TAG: $CURRENT_TAG" "\033[0m" - npm publish --tag $CURRENT_TAG - fi -} - -ROOT=$WORKSPACE/gopath/src/github.com/hyperledger/fabric-chaincode-node - -npm config set //registry.npmjs.org/:_authToken=$NPM_TOKEN - -cd ${ROOT} - -CURRENT_TAG=$(cat package.json | grep tag | awk -F\" '{ print $4 }') -echo -e "\033[32m ======> CURRENT_TAG: $CURRENT_TAG" "\033[0m" - -RELEASE_VERSION=$(cat package.json | grep version | awk -F\" '{ print $4 }') -echo -e "\033[32m ======> Current RELEASE_VERSION:" "\033[0m" - -# lock step versions to fabric-shim -UNSTABLE_VER=$(npm dist-tags ls fabric-shim | awk "/$CURRENT_TAG"":"/'{ - ver=$NF - sub(/.*\./,"",rel) - sub(/\.[[:digit:]]+$/,"",ver) - print ver}') - -echo "======> UNSTABLE VERSION:" $UNSTABLE_VER -# Increment unstable version here -# lock step versions to fabric-shim -UNSTABLE_INCREMENT=$(npm dist-tags ls fabric-shim | awk "/$CURRENT_TAG"":"/'{ - ver=$NF - rel=$NF - sub(/.*\./,"",rel) - sub(/\.[[:digit:]]+$/,"",ver) - print ver"."rel+1}') - -cd ${ROOT}/apis/fabric-shim-api -echo -e "\033[32m ======> fabric-shim-api" "\033[0m" -npmPublish - -cd ${ROOT}/libraries/fabric-shim -echo -e "\033[32m ======> fabric-shim" "\033[0m" -npmPublish - -cd ${ROOT}/libraries/fabric-shim-crypto -echo -e "\033[32m ======> fabric-shim-crypto" "\033[0m" -npmPublish - -cd ${ROOT}/apis/fabric-contract-api -echo -e "\033[32m ======> fabric-contract-api" "\033[0m" -npmPublish