From d8773c2437674622a38e40894b54ee00af780feb Mon Sep 17 00:00:00 2001 From: Sachin Holla <51310506+sachinholla@users.noreply.github.com> Date: Sat, 4 Jul 2020 06:03:03 +0530 Subject: [PATCH] Telemetry and mgmt-framework build fixes (#130) 1) Use buildimage-vs-image artifacts for sonic-mgmt-framework builds. It requires buster packages. 2) Changed telemetry build script to install redis directly through apt. 3) Changed mgmt-framework to make it similar to telemetry build script's layout -- install deps and compile sonic-mgmt-common by itself, without depending on sonic-mgmt-common's build script. Signed-off-by: Sachin Holla --- .../sonic-mgmt-common-build-pr/Jenkinsfile | 2 +- .../sonic-mgmt-framework-build-pr/Jenkinsfile | 3 +- .../sonic-mgmt-framework-build/Jenkinsfile | 1 - .../sonic-telemetry-build-pr/Jenkinsfile | 5 +++ .../common/sonic-telemetry-build/Jenkinsfile | 5 +++ .../sonic-mgmt-framework-build/build.sh | 42 +++++++++++++++++-- .../common/sonic-mgmt-framework-build/test.sh | 1 + scripts/common/sonic-telemetry-build/build.sh | 9 +--- 8 files changed, 53 insertions(+), 15 deletions(-) diff --git a/jenkins/common/sonic-mgmt-common-build-pr/Jenkinsfile b/jenkins/common/sonic-mgmt-common-build-pr/Jenkinsfile index 8ff3ae60158e..3774dd7533ea 100644 --- a/jenkins/common/sonic-mgmt-common-build-pr/Jenkinsfile +++ b/jenkins/common/sonic-mgmt-common-build-pr/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common', refspec: '+refs/pull/*:refs/remotes/origin/pr/*']]]) } - copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false) + copyArtifacts(projectName: 'vs/buildimage-vs-image', filter: '**/*.deb', target: 'buildimage', flatten: false) } } diff --git a/jenkins/common/sonic-mgmt-framework-build-pr/Jenkinsfile b/jenkins/common/sonic-mgmt-framework-build-pr/Jenkinsfile index eb237a55443a..4837983cb4e4 100644 --- a/jenkins/common/sonic-mgmt-framework-build-pr/Jenkinsfile +++ b/jenkins/common/sonic-mgmt-framework-build-pr/Jenkinsfile @@ -15,13 +15,12 @@ pipeline { branches: [[name: 'refs/heads/master']], userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common']]]) } - copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false) + copyArtifacts(projectName: 'vs/buildimage-vs-image', filter: '**/*.deb', target: 'buildimage', flatten: false) } } stage('Build') { steps { - sh './scripts/common/sonic-mgmt-common-build/build.sh' sh './scripts/common/sonic-mgmt-framework-build/build.sh' } } diff --git a/jenkins/common/sonic-mgmt-framework-build/Jenkinsfile b/jenkins/common/sonic-mgmt-framework-build/Jenkinsfile index 2ff51b7fa99c..50cfc35af1d0 100644 --- a/jenkins/common/sonic-mgmt-framework-build/Jenkinsfile +++ b/jenkins/common/sonic-mgmt-framework-build/Jenkinsfile @@ -32,7 +32,6 @@ pipeline { stage('Build') { steps { - sh './scripts/common/sonic-mgmt-common-build/build.sh' sh './scripts/common/sonic-mgmt-framework-build/build.sh' } } diff --git a/jenkins/common/sonic-telemetry-build-pr/Jenkinsfile b/jenkins/common/sonic-telemetry-build-pr/Jenkinsfile index 62f999d113cd..acf2348630fe 100644 --- a/jenkins/common/sonic-telemetry-build-pr/Jenkinsfile +++ b/jenkins/common/sonic-telemetry-build-pr/Jenkinsfile @@ -10,6 +10,11 @@ pipeline { userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-telemetry', refspec: '+refs/pull/*:refs/remotes/origin/pr/*']]]) } + dir('sonic-mgmt-common') { + checkout([$class: 'GitSCM', + branches: [[name: 'refs/heads/master']], + userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common']]]) + } copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false) } } diff --git a/jenkins/common/sonic-telemetry-build/Jenkinsfile b/jenkins/common/sonic-telemetry-build/Jenkinsfile index ed7aaf38678a..4a079102d300 100644 --- a/jenkins/common/sonic-telemetry-build/Jenkinsfile +++ b/jenkins/common/sonic-telemetry-build/Jenkinsfile @@ -21,6 +21,11 @@ pipeline { branches: [[name: 'refs/heads/master']], userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-telemetry']]]) } + dir('sonic-mgmt-common') { + checkout([$class: 'GitSCM', + branches: [[name: 'refs/heads/master']], + userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-mgmt-common']]]) + } copyArtifacts(projectName: '../vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false) } } diff --git a/scripts/common/sonic-mgmt-framework-build/build.sh b/scripts/common/sonic-mgmt-framework-build/build.sh index 1832b25e2f13..5cc66bfb49a2 100755 --- a/scripts/common/sonic-mgmt-framework-build/build.sh +++ b/scripts/common/sonic-mgmt-framework-build/build.sh @@ -1,9 +1,43 @@ #!/bin/bash -ex # Build script for sonic-mgmt-framework. -# Assumes sonic-mgmt-common is already cloned & built. -# Dependencies of sonic-mgmt-common would have been installed already. -# Only sonic-mgmt-framework dependencies will have to be installed here. + +[[ ! -z ${DISTRO} ]] || DISTRO=buster + +# REDIS +sudo apt-get install -y redis-server +sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf +sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf +sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf +sudo service redis-server start + +# LIBYANG +sudo dpkg -i buildimage/target/debs/${DISTRO}/libyang*.deb + +# Install from "requirement" files in sonic-mgmt-framework/tools/test directory. +pushd sonic-mgmt-framework/tools/test + +[[ ! -f apt_requirements.txt ]] || \ + sed 's/#.*//' apt_requirements.txt | xargs -r sudo apt-get install -y + +[[ ! -f python2_requirements.txt ]] || \ + sudo pip install --no-cache-dir -r python2_requirements.txt + +[[ ! -f python3_requirements.txt ]] || \ + sudo pip3 install --no-cache-dir -r python3_requirements.txt + +popd + + +# Build sonic-mgmt-common + +pushd sonic-mgmt-common + +NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc + +popd + +# Build sonic-mgmt-framework pushd sonic-mgmt-framework @@ -12,4 +46,4 @@ dpkg-buildpackage -rfakeroot -b -us -uc popd mkdir -p target -cp sonic-mgmt-framework*.deb target/ +cp *.deb target/ diff --git a/scripts/common/sonic-mgmt-framework-build/test.sh b/scripts/common/sonic-mgmt-framework-build/test.sh index bbadd9320e36..aa2ed112ba81 100755 --- a/scripts/common/sonic-mgmt-framework-build/test.sh +++ b/scripts/common/sonic-mgmt-framework-build/test.sh @@ -10,6 +10,7 @@ DEBDIR=$(realpath sonic-mgmt-common/debian/sonic-mgmt-common) pushd sonic-mgmt-framework/build/tests/rest export CVL_SCHEMA_PATH=${DEBDIR}/usr/sbin/schema +export YANG_MODELS_PATH=${DEBDIR}/usr/models/yang ./server.test -test.v -logtostderr || STATUS=1 diff --git a/scripts/common/sonic-telemetry-build/build.sh b/scripts/common/sonic-telemetry-build/build.sh index 705d473c9dc2..575ec754351c 100755 --- a/scripts/common/sonic-telemetry-build/build.sh +++ b/scripts/common/sonic-telemetry-build/build.sh @@ -4,9 +4,7 @@ sudo dpkg -i buildimage/target/debs/stretch/libhiredis*.deb # Install REDIS -sudo apt-get install -y liblua5.1-0 lua-bitop lua-cjson -sudo dpkg -i buildimage/target/debs/stretch/redis-tools_*.deb -sudo dpkg -i buildimage/target/debs/stretch/redis-server_*.deb +sudo apt-get install -y redis-server sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf @@ -15,14 +13,11 @@ sudo service redis-server start # Install libyang sudo dpkg -i buildimage/target/debs/stretch/libyang*.deb -# Clone sonic-mgmt-common repository -git clone https://github.com/Azure/sonic-mgmt-common.git - # Build sonic-mgmt-common first pushd sonic-mgmt-common -dpkg-buildpackage -rfakeroot -b -us -uc +NO_TEST_BINS=1 dpkg-buildpackage -rfakeroot -b -us -uc popd