diff --git a/CHANGELOG.md b/CHANGELOG.md index 87de8ba6..f0ff7b4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ - Added a new stage to the lesson, "STIG Compliance checking with custom scripts". This lesson goes through building a python script to check the SNMP vulnerabilities that were done with NAPALM and jSNAPY in the to previous stages. [#238](https://github.com/nre-learning/nrelabs-curriculum/pull/238) - Added a lesson on BASH [Commit #1fe7b94](https://github.com/nre-learning/nrelabs-curriculum/commit/1fe7b94454e880b1a468b1d1742d2911139359ab) - Remove platform images from curriculum [#245](https://github.com/nre-learning/nrelabs-curriculum/pull/245) - +- Modified images to support a standard build process [#247](https://github.com/nre-learning/nrelabs-curriculum/pull/247) ## v0.3.2 - April 19, 2019 diff --git a/images/asterisk/Makefile b/images/asterisk/Makefile new file mode 100644 index 00000000..9853f774 --- /dev/null +++ b/images/asterisk/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/asterisk:$(TARGET_VERSION) . + docker push antidotelabs/asterisk:$(TARGET_VERSION) diff --git a/images/container-vqfx/Makefile b/images/container-vqfx/Makefile index a9b15f5c..3b3aaf12 100644 --- a/images/container-vqfx/Makefile +++ b/images/container-vqfx/Makefile @@ -1,19 +1,27 @@ -# Copyright (c) 2018, Juniper Networks, Inc. -# All rights reserved. +# SHELL=/bin/bash -all: build +TARGET_VERSION ?= latest -build: src/cosim.tgz - ./build.sh +all: docker -src/cosim.tgz: - cd vagrant && ./extract_pfe_files.sh ../src +docker: -run: - docker run -ti --rm --name vqfx --privileged antidote/vqfx:18.4R1.8 + ./download_packages.sh -run1: - docker run -ti --rm --name vqfx --privileged antidote/vqfx:18.1R1.9 + TARGET_VERSION=$(TARGET_VERSION) ./build.sh -stop: - docker kill vqfx || true + # The below might be useful for building an image with snapshots. + # Currently leaving this commented out so that we can finish the rest of the pipeline without this added + # complexity, not to mention that some infrastructure improvements (both prod and selfmedicate) might mean + # we don't need to do the snapshot trick. However, if it proves necessary in the future, this does work, + # we just need to add the logic to copy the resulting disk file out and into its own container, so we don't + # create a monster image. + # + # docker kill vqfx-full || true + # docker rm vqfx-full || true + # sleep 5 + # docker run -d --name vqfx-full -v $$(pwd):/cap --rm -p 2222:22 -p 4001:4000 -p 5001:5000 --privileged antidotelabs/vqfx-full:$(TARGET_VERSION) + # ./wait_for_live.sh + # docker exec vqfx-full /create_snapshot.expect + + docker push antidotelabs/container-vqfx:$(TARGET_VERSION) diff --git a/images/container-vqfx/build.sh b/images/container-vqfx/build.sh index de37f5a4..02c4aaf0 100755 --- a/images/container-vqfx/build.sh +++ b/images/container-vqfx/build.sh @@ -1,12 +1,18 @@ #!/bin/bash -target=antidotelabs/vqfx-full +target=antidotelabs/container-vqfx ocpkg=$(ls junos-openconfig-*.tgz) docker build -f src/Dockerfile -t container-vqfx src for image in *.img; do - version=$(echo "${image%.*}" | cut -d- -f5) + + # Originally, we got the tag from the Junos version. However, we are removing this now + # in favor of the Antidote-relevant tag. + # + # version=$(echo "${image%.*}" | cut -d- -f5) + version=$TARGET_VERSION + echo "Building container $target:$version ... " docker build -f Dockerfile.junos --build-arg image=$image --build-arg ocpkg=$ocpkg -t $target:$version . done diff --git a/images/container-vqfx/download_packages.sh b/images/container-vqfx/download_packages.sh new file mode 100755 index 00000000..bdd8b0ed --- /dev/null +++ b/images/container-vqfx/download_packages.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This script uses the gsutil utility from Google Cloud +# to download files from the relevant bucket. See +# https://cloud.google.com/sdk/docs/ for instructions on installing +# the Google Cloud SDK, which includes this utility. + +declare -a arr=( + "container-vqfx/junos-openconfig-0.0.0.10-1-signed.tgz" + "container-vqfx/cosim.tgz" + "container-vqfx/jinstall-vqfx-10-f-18.1R1.9.img" +) + +for i in "${arr[@]}" +do + gsutil cp "gs://nrelabs-curriculum-base-images/$i" "./$i" + mv "$i" ./ +done + +rm -rf container-vqfx/ +mv cosim.tgz src/ diff --git a/images/container-vqfx/src/Dockerfile b/images/container-vqfx/src/Dockerfile index 569232ec..de9a2a8d 100644 --- a/images/container-vqfx/src/Dockerfile +++ b/images/container-vqfx/src/Dockerfile @@ -8,7 +8,8 @@ RUN rm -f /root/pecosim/*.tgz FROM ubuntu:18.04 RUN export DEBIAN_FRONTEND=noninteractive \ - && apt-get update && apt-get install -y -q qemu-kvm qemu-utils dosfstools pwgen telnet \ + && apt-get update && apt-get install -y -q qemu-kvm qemu-utils dosfstools \ + pwgen telnet screen socat expect vim tcl tclsh \ ca-certificates netbase libpcap0.8 \ tcpdump macchanger gettext-base net-tools ethtool\ file iproute2 docker.io \ @@ -19,7 +20,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ COPY --from=cosim /root/pecosim /root/pecosim COPY create_config_drive.sh launch.sh \ - create_apply_group.sh fix_network_order.sh / + create_apply_group.sh fix_network_order.sh create_snapshot.expect / RUN chmod a+rx /*.sh diff --git a/images/container-vqfx/src/create_snapshot.expect b/images/container-vqfx/src/create_snapshot.expect new file mode 100755 index 00000000..232cbeee --- /dev/null +++ b/images/container-vqfx/src/create_snapshot.expect @@ -0,0 +1,67 @@ +#!/usr/bin/expect -f +# +# This Expect script was generated by autoexpect on Mon Jul 22 07:12:58 2019 +# Expect and autoexpect were both written by Don Libes, NIST. +# +# Note that autoexpect does not guarantee a working script. It +# necessarily has to guess about certain things. Two reasons a script +# might fail are: +# +# 1) timing - A surprising number of programs (rn, ksh, zsh, telnet, +# etc.) and devices discard or ignore keystrokes that arrive "too +# quickly" after prompts. If you find your new script hanging up at +# one spot, try adding a short sleep just before the previous send. +# Setting "force_conservative" to 1 (see below) makes Expect do this +# automatically - pausing briefly before sending each character. This +# pacifies every program I know of. The -c flag makes the script do +# this in the first place. The -C flag allows you to define a +# character to toggle this mode off and on. + +set force_conservative 0 ;# set to 1 to force conservative mode even if + ;# script wasn't run conservatively originally +if {$force_conservative} { + set send_slow {1 .1} + proc send {ignore arg} { + sleep .1 + exp_send -s -- $arg + } +} + +# +# 2) differing output - Some programs produce different output each time +# they run. The "date" command is an obvious example. Another is +# ftp, if it produces throughput statistics at the end of a file +# transfer. If this causes a problem, delete these patterns or replace +# them with wildcards. An alternative is to use the -p flag (for +# "prompt") which makes Expect only look for the last line of output +# (i.e., the prompt). The -P flag allows you to define a character to +# toggle this mode off and on. +# +# Read the man page for more info. +# +# -Don + + +set timeout -1 +spawn telnet 127.0.0.1 4000 +match_max 100000 +expect -exact "Trying 127.0.0.1...\r +Connected to 127.0.0.1.\r +Escape character is '^\]'.\r +QEMU 2.11.1 monitor - type 'help' for more information\r +(qemu) " +send -- "stop\r" +expect -exact "stop\r +s\[K\[Dst\[K\[D\[Dsto\[K\[D\[D\[Dstop\[K\r +(qemu) \r +(qemu) " +send -- "savevm speedy\r" +expect -exact "savevm speedy\r +s\[K\[Dsa\[K\[D\[Dsav\[K\[D\[D\[Dsave\[K\[D\[D\[D\[Dsavev\[K\[D\[D\[D\[D\[Dsavevm\[K\[D\[D\[D\[D\[D\[Dsavevm \[K\[D\[D\[D\[D\[D\[D\[Dsavevm s\[K\[D\[D\[D\[D\[D\[D\[D\[Dsavevm sp\[K\[D\[D\[D\[D\[D\[D\[D\[D\[Dsavevm spe\[K\[D\[D\[D\[D\[D\[D\[D\[D\[D\[Dsavevm spee\[K\[D\[D\[D\[D\[D\[D\[D\[D\[D\[D\[Dsavevm speed\[K\[D\[D\[D\[D\[D\[D\[D\[D\[D\[D\[D\[Dsavevm speedy\[K\r +(qemu) \r +(qemu) " +send -- "" +expect -exact "^\]\r +telnet> " +send -- "quit\r" +expect eof diff --git a/images/container-vqfx/wait_for_live.sh b/images/container-vqfx/wait_for_live.sh new file mode 100755 index 00000000..18487788 --- /dev/null +++ b/images/container-vqfx/wait_for_live.sh @@ -0,0 +1,21 @@ +#!/bin/bash + + +until sshpass -p antidotepassword ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -q antidote@localhost -p 2222 exit +do + echo "return is $?, waiting for 0" + sleep 5 +done + +echo "Device is Live" + +while [[ $xeifs -eq 24 ]] +do + # Wait until PFE is active + xeifs=$((sshpass -p antidotepassword ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -q antidote@localhost -p 2222 show interfaces terse | grep xe-0 | wc -l)) + + echo "xeifs is $xeifs, waiting for 24" + sleep 1 +done + +echo "cosim is live" diff --git a/images/netbox/Makefile b/images/netbox/Makefile new file mode 100644 index 00000000..e594b421 --- /dev/null +++ b/images/netbox/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/netbox:$(TARGET_VERSION) . + docker push antidotelabs/netbox:$(TARGET_VERSION) diff --git a/images/newvmx/Dockerfile b/images/newvmx/Dockerfile deleted file mode 100644 index b4a9abd5..00000000 --- a/images/newvmx/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM marcelwiget/vmx-docker-light - -COPY launch.sh / -RUN chmod a+rx /*.sh - -VOLUME /u /var/run/docker.sock - -ENTRYPOINT ["/launch.sh"] \ No newline at end of file diff --git a/images/newvmx/launch.sh b/images/newvmx/launch.sh deleted file mode 100755 index 432bc5ea..00000000 --- a/images/newvmx/launch.sh +++ /dev/null @@ -1,427 +0,0 @@ -#!/bin/bash -# Copyright (c) 2017, Juniper Networks, Inc. -# All rights reserved. -# - -VCPMEM="${VCPMEM:-1024}" -VCPU="${VCPU:-1}" - -echo "Juniper Networks vMX Docker Light Container" - -set -e # exit immediately if something goes wrong -/system_check.sh - -echo "/u contains the following files:" -ls /u - -while getopts "c:m:l:V:ep:i:" opt; do - case "$opt" in - V) VCPCPU=$OPTARG - ;; - m) VCPMEM=$OPTARG - ;; - e) EMULATED=1 - ;; - p) PUBLICKEY=$OPTARG - ;; - c) CONFIG=$OPTARG - ;; - i) LO_IP=$OPTARG - ;; - l) LICENSE=$OPTARG - ;; - esac -done - -CONFIG="${CONFIG-config.txt}" - -shift "$((OPTIND-1))" - -if [ ! -z "$1" ]; then - IMAGE=$1 - shift -fi - -if [ -z "$IMAGE" ]; then - # no image given, check if we have one in /u - IMAGE=$(cd /u && ls junos-*qcow2 | tail -1) -fi - -if [ ! -f "/u/$IMAGE" ]; then - echo "vMX file $IMAGE not found" - exit 1 -fi - -if [ -z "$PUBLICKEY" ]; then - PUBLICKEY=$(cd /u && ls id_*.pub | tail -1) -fi - -if [ -z "$LICENSE" ]; then - LICENSE=$(cd /u && ls license*txt | tail -1) -fi - -if [ ! -f "/u/$PUBLICKEY" ]; then - echo "WARNING: Can't read ssh public key file $PUBLICKEY" - echo "Access limited to root password" -else - SSHPUBLIC=$(cat /u/$PUBLICKEY) -fi - -if [ ! -z "$PCI" ]; then - echo -n "checking for igb_uio kernel module ..." - ls /sys/bus/pci/drivers/igb_uio/ >/dev/null 2>/dev/null - echo "ok" -fi - -for pci in $PCI; do - if [ ${#pci} -lt 8 ]; then - pci="0000:$pci" - fi - if [ -e "/sys/bus/pci/drivers/igb_uio/$pci" ]; then - echo "$pci already bound to igb_uio" - else - echo -n "Binding $pci to igb_uio ..." - echo -n $pci > /sys/bus/pci/drivers/ixgbe/unbind || true - echo -n "8086 10fb" > /sys/bus/pci/drivers/igb_uio/new_id - if [ -e "/sys/bus/pci/drivers/igb_uio/$pci" ]; then - echo "$pci bound to igb_uio" - else - echo -n $pci > /sys/bus/pci/drivers/igb_uio/bind - fi - echo "done" - fi -done - - -# fix network interface order due to https://github.com/docker/compose/issues/4645 -ifconfig -a -/fix_network_order.sh -ifconfig -a -ip route -ip -6 route - -ROOTPASSWORD=$(pwgen 24 1) -SALT=$(pwgen 8 1) -HASH=$(openssl passwd -1 -salt $SALT $ROOTPASSWORD) -myip=$(ifconfig eth0|grep 'inet addr'|cut -d: -f2|awk '{print $1}') -# there is a small chance of getting 2 global IPv6 addresses temporarly, pick just the last one with tail -1 -myipv6=$(ifconfig eth0|grep Global|awk '{print $3}'|tail -1|cut -d/ -f1) -# extract container name via id -hostname=$(docker ps --format '{{.Names}}' -f id=$HOSTNAME || echo $HOSTNAME) -if [ -z "$hostname" ]; then - hostname=$HOSTNAME -fi - -if [ "$LO_IP" -ge 1 -a "$LO_IP" -le 255 ]; then - id=$LO_IP -else - id=$(echo $hostname|rev | cut -d'_' -f 1 | rev) # get index, e.g. 3 from vmxdockerlight_vmx_3 -fi -echo "Loopback IP last octet: $id" - -export myip myipv6 hostname id -echo "-----------------------------------------------------------------------" -echo "vMX $hostname ($myip $myipv6) root password to $ROOTPASSWORD" -echo "-----------------------------------------------------------------------" -echo "" - -if [[ "$IMAGE" =~ \.qcow2$ ]]; then - echo "using qcow2 image $IMAGE" - cp /u/$IMAGE /tmp/ - VCPIMAGE=$IMAGE -else - echo "extracting qcow2 image from $IMAGE ..." - tar zxvf /u/$IMAGE -C /tmp/ --wildcards vmx/images/junos*qcow2 2>/dev/null - VCPIMAGE=$(ls /tmp/vmx*/images/junos*qcow2) - mv $VCPIMAGE /tmp/ - VCPIMAGE=${VCPIMAGE##*/} -fi - -if [ ! -f "/u/$LICENSE" ]; then - echo "Warning: No license file found ($LICENSE)" -fi -echo "LICENSE=$LICENSE" - -until ifconfig eth0; do - echo "waiting for eth0 to be attached ..." - sleep 5 -done - -# instead of using rcp, this method uses scp instead by -# creating a local ssh keypair and adding it to the junos config -#ln -s /usr/bin/scp /usr/bin/rcp -#ssh-keygen -t rsa -f /root/.ssh/id_rsa -N '' -#PUBKEY=$(cat /root/.ssh/id_rsa.pub) -#cp /u/$CONFIG /tmp/ -#cat >> /tmp/$CONFIG < /tmp/$CONFIG <> /tmp/$CONFIG -fi - -if [ ! -z "$SSHPUBLIC" ]; then - SSHUSER=$(echo $SSHPUBLIC | cut -d' ' -f3 | cut -d'@' -f1) - if [ $SSHUSER == 'root' ]; then - SSHUSER="lab" - fi - echo "adding super-user $SSHUSER with public key $SSHPUBLIC to config" - cat >> /tmp/$CONFIG <> /tmp/$CONFIG <> /tmp/$CONFIG < fxp0" - cat >> /tmp/$CONFIG < ge-0/0/$index" - if [ -z "$NOIP" ]; then - cat >> /tmp/$CONFIG <> /tmp/$CONFIG <> /tmp/$CONFIG <> /tmp/$CONFIG <> /tmp/$CONFIG - for nameserver in $nameservers; do - echo " $nameserver;" >> /tmp/$CONFIG - done - echo "} }" >> /tmp/$CONFIG -fi - -cat >> /tmp/$CONFIG </dev/null - -echo "Starting PFE with PCI=($PCI) ..." -PCI=$PCI sh /start_pfe.sh & - -echo "Booting VCP ($ENABLEKVM) ..." -qemu-system-x86_64 --version - -cd /tmp -qemu-system-x86_64 -M pc $ENABLEKVM -smp 1 -m $VCPMEM \ - -smbios type=0,vendor=Juniper \ - -smbios type=1,manufacturer=VMX,product=VM-vcp_vmx1-161-re-0,version=0.1.0 \ - -no-user-config \ - -no-shutdown \ - -drive if=ide,file=$VCPIMAGE -drive if=ide,file=$HDDIMAGE \ - -drive if=ide,file=$CFGDRIVE \ - -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 \ - -netdev type=tap,id=tc0,ifname=fxp0,script=no,downscript=no \ - -device virtio-net-pci,netdev=tc0,mac=$mymac \ - -netdev type=tap,id=tc1,ifname=em1,script=no,downscript=no \ - -device virtio-net-pci,netdev=tc1 \ - -netdev type=tap,id=tc2,ifname=em2,script=no,downscript=no \ - -device virtio-net-pci,netdev=tc2 \ - -netdev type=tap,id=tc3,ifname=em3,script=no,downscript=no \ - -device virtio-net-pci,netdev=tc3 \ - -nographic || true diff --git a/images/pjsua-lindsey/Makefile b/images/pjsua-lindsey/Makefile new file mode 100644 index 00000000..301eae32 --- /dev/null +++ b/images/pjsua-lindsey/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/pjsua-lindsey:$(TARGET_VERSION) . + docker push antidotelabs/pjsua-lindsey:$(TARGET_VERSION) diff --git a/images/pjsua-lindsey/push.sh b/images/pjsua-lindsey/push.sh deleted file mode 100755 index 45a4591d..00000000 --- a/images/pjsua-lindsey/push.sh +++ /dev/null @@ -1,2 +0,0 @@ -docker build -t antidotelabs/pjsua-lindsey . -docker push antidotelabs/pjsua-lindsey diff --git a/images/salt/Makefile b/images/salt/Makefile new file mode 100644 index 00000000..71d79837 --- /dev/null +++ b/images/salt/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/salt:$(TARGET_VERSION) . + docker push antidotelabs/salt:$(TARGET_VERSION) diff --git a/images/selfservice-flask-app/Makefile b/images/selfservice-flask-app/Makefile new file mode 100644 index 00000000..00479fe7 --- /dev/null +++ b/images/selfservice-flask-app/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/selfservice-flask-app:$(TARGET_VERSION) . + docker push antidotelabs/selfservice-flask-app:$(TARGET_VERSION) diff --git a/images/st2/Makefile b/images/st2/Makefile new file mode 100644 index 00000000..18970942 --- /dev/null +++ b/images/st2/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/stackstorm:$(TARGET_VERSION) . + docker push antidotelabs/stackstorm:$(TARGET_VERSION) diff --git a/images/terraform/Makefile b/images/terraform/Makefile new file mode 100644 index 00000000..fd372a15 --- /dev/null +++ b/images/terraform/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/terraform:$(TARGET_VERSION) . + docker push antidotelabs/terraform:$(TARGET_VERSION) diff --git a/images/utility-vm/Makefile b/images/utility-vm/Makefile new file mode 100644 index 00000000..787fe213 --- /dev/null +++ b/images/utility-vm/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/utility-vm:$(TARGET_VERSION) . + docker push antidotelabs/utility-vm:$(TARGET_VERSION) diff --git a/images/utility/Makefile b/images/utility/Makefile new file mode 100644 index 00000000..30793a02 --- /dev/null +++ b/images/utility/Makefile @@ -0,0 +1,9 @@ +# SHELL=/bin/bash + +TARGET_VERSION ?= latest + +all: docker + +docker: + docker build -t antidotelabs/utility:$(TARGET_VERSION) . + docker push antidotelabs/utility:$(TARGET_VERSION)