Skip to content

Commit

Permalink
chore(ci) build nightly releases from jenkins instead of travis-ci (#167
Browse files Browse the repository at this point in the history
)

* chore(ci) [skip travis] move nightly releases to Jenkins

* [skip travis]

* [skip travis] split plugin tests out and login to docker when building the docker test image

* [skip travis] try a different way of defining the KONG_VERSION env

* [skip travis] skip the problematic builds

* [skip travis] move the daily deploys out of travis.yml

* [skip travis] wip debugging a sporadically failing test

* fix(tests) adjust how we run the report mock server for a more reliable test

* chore(ci) debug the environment variables available in jenkins [skip travis]

* chore(ci) set the repository os name environment variable [skip travis]

* test(reports) adjust how we check if the report server can run

* chore(ci) adjust the jenkins setup [skip travis]

* chore(wip) remove the integration tests to focus on getting the nightly releases to work

* fix(ci) adjust how set set the bintray credentials [skip travis]

* wip -- debugging daily releases to bintray [skip travis]

* chore(ci) run only the xenial release [skip travis]

* chore(ci) re-enable tests and other distribution releases

* chore(ci) add the CI cron trigger

chore(dependency) bump the kong-build-tools dependency (#168)

chore(dependencies) adjust kong-build-tools dependency pin (#169)

* chore(dependency) bump the kong-build-tools dependency

* chore(ci) unpin the jenkins build from the kong-build-tools branch

chore(nightly) build nightly arm release (#171)

chore(ci) adjust cache settings for xenail nightly builds (#173)
  • Loading branch information
hutchic committed Nov 21, 2019
1 parent c6b245c commit 5ba8f81
Show file tree
Hide file tree
Showing 4 changed files with 250 additions and 82 deletions.
75 changes: 0 additions & 75 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ cache:
stages:
- lint and unit
- test
- Deploy daily build
- Release

jobs:
Expand Down Expand Up @@ -149,79 +148,5 @@ jobs:
- make release
env: PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=1 KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=${TRAVIS_TAG}
if: tag IS present AND tag ~= 1.
- stage: deploy daily build
install: skip
script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=trusty KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=xenial KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=deb RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=bionic KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=deb RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=stretch KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=deb RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=jessie KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=deb RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=buster KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=centos RESTY_IMAGE_TAG=6 KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=centos RESTY_IMAGE_TAG=7 KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=apk RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=latest KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=6 KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=rhel RESTY_IMAGE_TAG=7 KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
- script:
- make setup-kong-build-tools
- pushd ../kong-build-tools && make setup-ci && popd
- make nightly-release
env: PACKAGE_TYPE=rpm RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=latest KONG_PACKAGE_NAME=${PWD##*/} KONG_VERSION=`date +%Y-%m-%d` REPOSITORY_NAME=${PWD##*/}-nightly REPOSITORY_OS_NAME=$TRAVIS_BRANCH
if: type=cron
script:
- .ci/run_tests.sh
236 changes: 236 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
pipeline {
agent none
environment{
UPDATE_CACHE = "true"
DOCKER_CREDENTIALS = credentials('dockerhub')
DOCKER_USERNAME = "${env.DOCKER_CREDENTIALS_USR}"
DOCKER_PASSWORD = "${env.DOCKER_CREDENTIALS_PSW}"
KONG_PACKAGE_NAME = 'kong'
REPOSITORY_OS_NAME = "${env.BRANCH_NAME}"
}
triggers {
cron('@daily')
}
stages {
stage('Build Kong') {
agent {
node {
label 'docker-compose'
}
}
environment {
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
}
steps {
sh 'make setup-kong-build-tools'
sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
dir('../kong-build-tools') { sh 'make kong-test-container' }
}
}
stage('Integration Tests') {
parallel {
stage('dbless') {
agent {
node {
label 'docker-compose'
}
}
environment {
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
TEST_DATABASE = "off"
TEST_SUITE = "dbless"
}
steps {
sh 'make setup-kong-build-tools'
dir('../kong-build-tools'){
sh 'make test-kong'
}
}
}
stage('postgres') {
agent {
node {
label 'docker-compose'
}
}
environment {
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
TEST_DATABASE = 'postgres'
}
steps {
sh 'make setup-kong-build-tools'
dir('../kong-build-tools'){
sh 'make test-kong'
}
}
}
stage('postgres plugins') {
agent {
node {
label 'docker-compose'
}
}
environment {
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
TEST_DATABASE = 'postgres'
TEST_SUITE = 'plugins'
}
steps {
sh 'make setup-kong-build-tools'
dir('../kong-build-tools'){
sh 'make test-kong'
}
}
}
stage('cassandra') {
agent {
node {
label 'docker-compose'
}
}
environment {
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
TEST_DATABASE = 'cassandra'
}
steps {
sh 'make setup-kong-build-tools'
dir('../kong-build-tools'){
sh 'make test-kong'
}
}
}
}
}
stage('Nightly Releases') {
when {
allOf {
triggeredBy 'TimerTrigger'
anyOf { branch 'master'; branch 'next' }
}
}
parallel {
stage('Ubuntu Xenial Release') {
agent {
node {
label 'docker-compose'
}
}
options {
retry(2)
}
environment {
PACKAGE_TYPE = 'deb'
RESTY_IMAGE_BASE = 'ubuntu'
RESTY_IMAGE_TAG = 'xenial'
CACHE = 'false'
UPDATE_CACHE = 'true'
USER = 'travis'
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
BINTRAY_USR = 'kong-inc_travis-ci@kong'
BINTRAY_KEY = credentials('bintray_travis_key')
AWS_ACCESS_KEY = credentials('AWS_ACCESS_KEY')
AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
DOCKER_MACHINE_ARM64_NAME = "jenkins-kong-${env.BUILD_NUMBER}"
}
steps {
sh 'make setup-kong-build-tools'
sh 'mkdir -p $HOME/bin'
sh 'sudo ln -s $HOME/bin/kubectl /usr/local/bin/kubectl'
sh 'sudo ln -s $HOME/bin/kind /usr/local/bin/kind'
dir('../kong-build-tools'){ sh 'make setup-ci' }
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` make nightly-release'
}
post {
always {
dir('../kong-build-tools'){ sh 'make cleanup_build' }
}
}
}
stage('Ubuntu Releases') {
agent {
node {
label 'docker-compose'
}
}
environment {
PACKAGE_TYPE = 'deb'
RESTY_IMAGE_BASE = 'ubuntu'
RESTY_IMAGE_TAG = 'xenial'
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
BINTRAY_USR = 'kong-inc_travis-ci@kong'
BINTRAY_KEY = credentials('bintray_travis_key')
DOCKER_MACHINE_ARM64_NAME = "jenkins-kong-${env.BUILD_NUMBER}"
}
steps {
sh 'make setup-kong-build-tools'
sh 'mkdir -p $HOME/bin'
sh 'sudo ln -s $HOME/bin/kubectl /usr/local/bin/kubectl'
sh 'sudo ln -s $HOME/bin/kind /usr/local/bin/kind'
dir('../kong-build-tools'){ sh 'make setup-ci' }
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=trusty BUILDX=false make nightly-release'
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=bionic BUILDX=false make nightly-release'
}
}
stage('Centos Releases') {
agent {
node {
label 'docker-compose'
}
}
environment {
PACKAGE_TYPE = 'rpm'
RESTY_IMAGE_BASE = 'centos'
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
REDHAT_CREDENTIALS = credentials('redhat')
REDHAT_USERNAME = "${env.REDHAT_USR}"
REDHAT_PASSWORD = "${env.REDHAT_PSW}"
BINTRAY_USR = 'kong-inc_travis-ci@kong'
BINTRAY_KEY = credentials('bintray_travis_key')
}
steps {
sh 'make setup-kong-build-tools'
sh 'mkdir -p $HOME/bin'
sh 'sudo ln -s $HOME/bin/kubectl /usr/local/bin/kubectl'
sh 'sudo ln -s $HOME/bin/kind /usr/local/bin/kind'
dir('../kong-build-tools'){ sh 'make setup-ci' }
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=6 make nightly-release'
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=7 make nightly-release'
}
}
stage('Debian Releases') {
agent {
node {
label 'docker-compose'
}
}
environment {
PACKAGE_TYPE = 'deb'
RESTY_IMAGE_BASE = 'debian'
KONG_SOURCE_LOCATION = "${env.WORKSPACE}"
KONG_BUILD_TOOLS_LOCATION = "${env.WORKSPACE}/../kong-build-tools"
BINTRAY_USR = 'kong-inc_travis-ci@kong'
BINTRAY_KEY = credentials('bintray_travis_key')
}
steps {
sh 'make setup-kong-build-tools'
sh 'mkdir -p $HOME/bin'
sh 'sudo ln -s $HOME/bin/kubectl /usr/local/bin/kubectl'
sh 'sudo ln -s $HOME/bin/kind /usr/local/bin/kind'
dir('../kong-build-tools'){ sh 'make setup-ci' }
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=jessie make nightly-release'
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=stretch make nightly-release'
sh 'REPOSITORY_NAME=`basename ${GIT_URL%.*}`-nightly KONG_VERSION=`date +%Y-%m-%d` RESTY_IMAGE_TAG=buster make nightly-release'
}
}
}
}
}
}
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ RESTY_VERSION ?= `grep RESTY_VERSION $(KONG_SOURCE_LOCATION)/.requirements | awk
RESTY_LUAROCKS_VERSION ?= `grep RESTY_LUAROCKS_VERSION $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
RESTY_OPENSSL_VERSION ?= `grep RESTY_OPENSSL_VERSION $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
RESTY_PCRE_VERSION ?= `grep RESTY_PCRE_VERSION $(KONG_SOURCE_LOCATION)/.requirements | awk -F"=" '{print $$2}'`
KONG_BUILD_TOOLS ?= '2.0.4'
KONG_BUILD_TOOLS ?= '2.0.5'
KONG_VERSION ?= `cat $(KONG_SOURCE_LOCATION)/kong-*.rockspec | grep tag | awk '{print $$3}' | sed 's/"//g'`
OPENRESTY_PATCHES_BRANCH ?= master
KONG_NGINX_MODULE_BRANCH ?= master
Expand All @@ -43,8 +43,8 @@ setup-ci:
.ci/setup_env.sh

setup-kong-build-tools:
-rm -rf kong-build-tools; \
git clone https://github.com/Kong/kong-build-tools.git $(KONG_BUILD_TOOLS_LOCATION); fi
-rm -rf $(KONG_BUILD_TOOLS_LOCATION)
-git clone https://github.com/Kong/kong-build-tools.git $(KONG_BUILD_TOOLS_LOCATION)
cd $(KONG_BUILD_TOOLS_LOCATION); \
git reset --hard $(KONG_BUILD_TOOLS); \

Expand Down
15 changes: 11 additions & 4 deletions spec/helpers.lua
Original file line number Diff line number Diff line change
Expand Up @@ -770,20 +770,25 @@ local function udp_server(port, n, timeout)
return thread
end


local function mock_reports_server(opts)
local localhost = "127.0.0.1"
local threads = require "llthreads2.ex"
local server_port = constants.REPORTS.STATS_PORT
opts = opts or {}

local thread = threads.new({
function(port, host, opts)
local socket = require "socket"
local server = assert(socket.tcp())
server:settimeout(360)
assert(server:setoption("reuseaddr", true))
assert(server:bind(host, port))
local counter = 0
while not server:bind(host, port) do
counter = counter + 1
if counter > 5 then
error('could not bind successfully')
end
socket.sleep(1)
end
assert(server:listen())
local data = {}
local handshake_done = false
Expand Down Expand Up @@ -839,7 +844,9 @@ local function mock_reports_server(opts)
local sock = ngx.socket.tcp()
sock:settimeout(0.01)
while true do
if sock:connect(localhost, server_port) then
if not thread:alive() then
error('the reports thread died')
elseif sock:connect(localhost, server_port) then
sock:send("\\START\n")
local ok = sock:receive()
sock:close()
Expand Down

0 comments on commit 5ba8f81

Please sign in to comment.