greenbone-nvt-sync

#!/bin/sh
# Copyright (C) 2009-2020 Greenbone Networks GmbH
#
# SPDX-License-Identifier: GPL-2.0-or-later
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

# This script updates the local Network Vulnerability Tests (NVTs) from the
# Greenbone Security Feed (GSF) or the Greenbone Community Feed (GCF). 

VERSION=20.8.0

# SETTINGS
# ========

# PRIVATE_SUBDIR defines a subdirectory of the NVT directory that is excluded
# from the feed sync. This is where to place your own NVTs.
if [ -z "$PRIVATE_SUBDIR" ]
then
  PRIVATE_SUBDIR="private"
fi

# RSYNC_DELETE controls whether files which are not part of the repository will
# be removed from the local directory after synchronization. The default value
# for this setting is
# "--delete --exclude \"$PRIVATE_SUBDIR/\"",
# which means that files which are not part of the feed or private directory
# will be deleted.
RSYNC_DELETE="--delete --exclude $PRIVATE_SUBDIR/"

# RSYNC_SSH_OPTS contains options which should be passed to ssh for the rsync
# connection to the repository.
RSYNC_SSH_OPTS="-o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\""

# RSYNC_COMPRESS specifies the compression level to use for the rsync connection.
RSYNC_COMPRESS="--compress-level=9"

# RSYNC_CHMOD specifies the permissions to chmod the files to.
RSYNC_CHMOD="--perms --chmod=Fugo+r,Fug+w,Dugo-s,Dugo+rx,Dug+w"

# Verbosity flag for rsync. "-q" means a quiet rsync, "-v" a verbose rsync.
RSYNC_VERBOSE="-q"

# RSYNC_OPTIONS controls the general parameters for the rsync connection.
RSYNC_OPTIONS="--links --times --omit-dir-times $RSYNC_VERBOSE --recursive --partial --progress"

# Script and feed information which will be made available to user through
# command line options and automated tools.
# Script name which will be used for logging
SCRIPT_NAME="greenbone-nvt-sync"

# Result of selftest () is stored here. If it is not 0, the selftest has failed
# and the sync script is unlikely to work.
SELFTEST_FAIL=0

# Port to use for synchronization. Default value is 24.
PORT=24

# Directory where the OpenVAS configuration is located
OPENVAS_SYSCONF_DIR="/usr/local/etc/openvas"

# Directory where the feed update lock file will be placed.
OPENVAS_FEED_LOCK_PATH="/usr/local/var/run/feed-update.lock"

# Location of the GSF Access Key
ACCESS_KEY="/usr/local/etc/gvm/gsf-access-key"

# If ENABLED is set to 0, the sync script will not perform a synchronization.
ENABLED=1

# LOG_CMD defines the command to use for logging. To have logger log to stderr
# as well as syslog, add "-s" here. The logging facility is checked. In case of error
# all will be logged in the standard error and the socket error check will be
# disabled.
LOG_CMD="logger -t $SCRIPT_NAME"

check_logger () {
  logger --socket-error=on -p daemon.info -t $SCRIPT_NAME "Checking logger" --no-act 1>/dev/null 2>&1
  if [ $? -gt 0 ]
  then
    LOG_CMD="logger --socket-error=off -s -t $SCRIPT_NAME"
    $LOG_CMD -p daemon.warning "The log facility is not working as expected. All messages will be written to the standard error stream."
  fi
}
check_logger


# Source configuration file if it is readable
[ -r $OPENVAS_SYSCONF_DIR/greenbone-nvt-sync.conf ] && . $OPENVAS_SYSCONF_DIR/greenbone-nvt-sync.conf

# NVT_DIR is the place where the NVTs are located.
if [ -z "$NVT_DIR" ]
then
  NVT_DIR="/usr/local/var/lib/openvas/plugins"
fi

log_write () {
  $LOG_CMD -p daemon.notice $1
}

log_debug () {
  $LOG_CMD -p daemon.debug "$1"
}

log_info () {
  $LOG_CMD -p daemon.info "$1"
}

log_notice () {
  $LOG_CMD -p daemon.notice "$1"
}

log_warning () {
  $LOG_CMD -p daemon.warning "$1"
}

log_err () {
  $LOG_CMD -p daemon.err "$1"
}

stderr_write ()
{
  echo "$1" > /dev/stderr
}

# Read the general information about the feed origin from
# the file "plugin_feed_info.inc" inside the feed directory.
get_feed_info ()
{
  INFOFILE="$NVT_DIR/plugin_feed_info.inc"
  if [ -r $INFOFILE ] ; then
    FEED_VERSION=`grep PLUGIN_SET $INFOFILE | sed -e 's/[^0-9]//g'`
    FEED_NAME=`awk -F\" '/PLUGIN_FEED/ { print $2 }' $INFOFILE`
    FEED_VENDOR=`awk -F\" '/FEED_VENDOR/ { print $2 }' $INFOFILE`
    FEED_HOME=`awk -F\" '/FEED_HOME/ { print $2 }' $INFOFILE`
    FEED_PRESENT=1
  else
    FEED_PRESENT=0
  fi

  if [ -z "$FEED_NAME" ] ; then
    FEED_NAME="Unidentified Feed"
  fi

  if [ -z "$FEED_VENDOR" ] ; then
    FEED_VENDOR="Unidentified Vendor"
  fi

  if [ -z "$FEED_HOME" ] ; then
    FEED_HOME="Unidentified Feed Homepage"
  fi
}

# Prevent that root executes this script
if [ "`id -u`" -eq "0" ]
then
  stderr_write "$0 must not be executed as privileged user root"
  stderr_write
  stderr_write "Unlike the actual scanner the sync routine does not need privileges."
  stderr_write "Accidental execution as root would prevent later overwriting of"
  stderr_write "files with a non-privileged user."

  log_err "Denied to run as root"
  #exit 1
fi

# Always try to get the information when started.
# This also ensures variables like FEED_PRESENT are set.
get_feed_info

# Determine whether a GSF access key is present. If yes,
# then use the Greenbone Security Feed. Else use the
# Greenbone Community Feed.
if [ -e $ACCESS_KEY ]
then
  RESTRICTED=1
else
  RESTRICTED=0

  if [ -z "$COMMUNITY_NVT_RSYNC_FEED" ]; then
    COMMUNITY_NVT_RSYNC_FEED=rsync://feed.community.greenbone.net:/nvt-feed
    # An alternative syntax which might work if the above doesn't:
    # COMMUNITY_NVT_RSYNC_FEED=rsync@feed.community.greenbone.net::/nvt-feed
  fi
fi

RSYNC=`command -v rsync`

if [ -z "$TMPDIR" ]; then
  SYNC_TMP_DIR=/tmp
  # If we have mktemp, create a temporary dir (safer)
  if [ -n "`which mktemp`" ]; then
    SYNC_TMP_DIR=`mktemp -t -d greenbone-nvt-sync.XXXXXXXXXX` || { echo "ERROR: Cannot create temporary directory for file download" >&2; exit 1 ; }
    trap "rm -rf $SYNC_TMP_DIR" EXIT HUP INT TRAP TERM
  fi
else
  SYNC_TMP_DIR="$TMPDIR"
fi

# Initialize this indicator variable with default assuming the
# feed is not up-to-date.
FEED_CURRENT=0

# This function uses gos-state-manager to get information about the settings.
# If gos-state-manager is not installed the values of the settings can not be
# retrieved.
#
# Input: option
# Output: value as string or empty String if gos-state-manager is not installed
#         or option not set
get_value ()
{
  value=""
  key=$1
  if which gos-state-manager 1>/dev/null 2>&1
  then
    if gos-state-manager get "$key.value" 1>/dev/null 2>&1
    then
      value="$(gos-state-manager get "$key.value")"
    fi
  fi
  echo "$value"
}

# Creates a restricted access copy of the access key if necessary.
setup_temp_access_key () {
  if [ -e "$ACCESS_KEY" ]
  then
    FILE_ACCESS=`stat -c%a "$ACCESS_KEY" | cut -c2-`
  fi
  if [ -n "$FILE_ACCESS" ] && [ "00" != "$FILE_ACCESS" ]
  then
    TEMP_ACCESS_KEY_DIR=`mktemp -d`
    TEMP_ACCESS_KEY="$TEMP_ACCESS_KEY_DIR/gsf-access-key"
    cp "$ACCESS_KEY" "$TEMP_ACCESS_KEY"
    chmod 400 "$TEMP_ACCESS_KEY"
  else
    TEMP_ACCESS_KEY_DIR=""
    TEMP_ACCESS_KEY="$ACCESS_KEY"
  fi
}

# Deletes the read-only copy of the access key.
cleanup_temp_access_key () {
  if [ -n "$TEMP_ACCESS_KEY_DIR" ]
  then
    rm -rf "$TEMP_ACCESS_KEY_DIR"
  fi
  TEMP_ACCESS_KEY_DIR=""
  TEMP_ACCESS_KEY=""
}

is_feed_current () {
  if [ -z "$FEED_VERSION" ]
  then
    log_write "Could not determine feed version."
    FEED_CURRENT=0
    return $FEED_CURRENT
  fi

  if [ -z "$RSYNC" ]
  then
    log_notice "rsync not available, skipping feed version test"
    FEED_CURRENT=0
    rm -rf $FEED_INFO_TEMP_DIR
    cleanup_temp_access_key
    return 0
  fi

  FEED_INFO_TEMP_DIR=`mktemp -d`

  if [ -e $ACCESS_KEY ]
  then
    gsmproxy=$(get_value proxy_feed | sed -r -e 's/^.*\/\///' -e 's/:([0-9]+)$/ \1/')
    syncport=$(get_value syncport)
    if [ "$syncport" ]
    then
      PORT="$syncport"
    fi

    read feeduser < $ACCESS_KEY
    custid=`awk -F@ 'NR > 1 { exit }; { print $1 }' $ACCESS_KEY`
    if [ -z "$feeduser" ] || [ -z "$custid" ]
    then
      log_err "Could not determine credentials, aborting synchronization."
      exit 1
    fi

    setup_temp_access_key

    if [ "$gsmproxy" = "proxy_feed" ] || [ -z "$gsmproxy" ]
    then
      RSYNC_SSH_PROXY_CMD=""
    else
      if [ -e $OPENVAS_SYSCONF_DIR/proxyauth ] && [ -r $OPENVAS_SYSCONF_DIR/proxyauth ]
      then
        RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p $OPENVAS_SYSCONF_DIR/proxyauth\""
      else
        RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p\""
      fi
    fi

    rsync -e "ssh $RSYNC_SSH_OPTS $RSYNC_SSH_PROXY_CMD -p $PORT -i $TEMP_ACCESS_KEY" $RSYNC_OPTIONS $RSYNC_DELETE $RSYNC_COMPRESS $RSYNC_CHMOD "$feeduser"plugin_feed_info.inc $FEED_INFO_TEMP_DIR

    if [ $? -ne 0 ]
    then
      log_err "Error: rsync failed."
      rm -rf "$FEED_INFO_TEMP_DIR"
      exit 1
    fi
  else
    # Sleep for five seconds (a previous feed might have been synced a few seconds before) to prevent
    # IP blocking due to network equipment in between keeping the previous connection too long open.
    sleep 5
    log_notice "No Greenbone Security Feed access key found, falling back to Greenbone Community Feed"
    eval "$RSYNC -ltvrP \"$COMMUNITY_NVT_RSYNC_FEED/plugin_feed_info.inc\" \"$FEED_INFO_TEMP_DIR\""
    if [ $? -ne 0 ]
    then
      log_err "rsync failed, aborting synchronization."
      rm -rf "$FEED_INFO_TEMP_DIR"
      exit 1
    fi
  fi

  FEED_VERSION_SERVER=`grep PLUGIN_SET $FEED_INFO_TEMP_DIR/plugin_feed_info.inc | sed -e 's/[^0-9]//g'`

  if [ -z "$FEED_VERSION_SERVER" ]
  then
    log_err "Could not determine server feed version."
    rm -rf $FEED_INFO_TEMP_DIR
    cleanup_temp_access_key
    exit 1
  fi
  # Check against FEED_VERSION
  if [ $FEED_VERSION -lt $FEED_VERSION_SERVER ] ; then
    FEED_CURRENT=0
  else
    FEED_CURRENT=1
  fi
  # Cleanup
  rm -rf "$FEED_INFO_TEMP_DIR"
  cleanup_temp_access_key

  return $FEED_CURRENT
}

do_rsync_community_feed () {
  # Sleep for five seconds (a previous feed might have been synced a few seconds before) to prevent
  # IP blocking due to network equipment in between keeping the previous connection too long open.
  sleep 5
  log_notice "Configured NVT rsync feed: $COMMUNITY_NVT_RSYNC_FEED"
  mkdir -p "$NVT_DIR"
  eval "$RSYNC -ltvrP $RSYNC_DELETE \"$COMMUNITY_NVT_RSYNC_FEED\" \"$NVT_DIR\" --exclude=plugin_feed_info.inc"
  if [ $? -ne 0 ] ; then
    log_err "rsync failed."
    exit 1
  fi
  # Sleep for five seconds (after the above rsync call) to prevent IP blocking due
  # to network equipment in between keeping the previous connection too long open.
  sleep 5
  eval "$RSYNC -ltvrP $RSYNC_DELETE \"$COMMUNITY_NVT_RSYNC_FEED/plugin_feed_info.inc\" \"$NVT_DIR\""
  if [ $? -ne 0 ] ; then
    log_err "rsync failed."
    exit 1
  fi
}

sync_nvts(){
  if [ $ENABLED -ne 1 ]
  then
    log_write "NVT synchronization is disabled, exiting."
    exit 0
  fi

  if [ -e $ACCESS_KEY ]
  then
    log_write "Synchronizing NVTs from the Greenbone Security Feed into $NVT_DIR..."
    if [ $FEED_PRESENT -eq 1 ] ; then
      FEEDCOUNT=`grep -E "nasl$|inc$" $NVT_DIR/md5sums | wc -l`
      log_write "Current status: Using $FEED_NAME at version $FEED_VERSION ($FEEDCOUNT NVTs)"
    else
      log_write "Current status: No feed installed."
    fi
    notsynced=1
    retried=0

    mkdir -p "$NVT_DIR"
    read feeduser < $ACCESS_KEY
    custid=`awk -F@ 'NR > 1 { exit }; { print $1 }' $ACCESS_KEY`
    if [ -z "$feeduser" ] || [ -z "$custid" ]
    then
      log_err "Could not determine credentials, aborting synchronization."
      exit 1
    fi

    setup_temp_access_key

    while [ $notsynced -eq 1 ]
    do

      gsmproxy=$(get_value proxy_feed | sed -r -e 's/^.*\/\///' -e 's/:([0-9]+)$/ \1/')
      syncport=$(get_value syncport)
      if [ "$syncport" ]
      then
        PORT="$syncport"
      fi

      if [ "$gsmproxy" = "proxy_feed" ] || [ -z "$gsmproxy" ]
      then
        RSYNC_SSH_PROXY_CMD=""
      else
        if [ -e $OPENVAS_SYSCONF_DIR/proxyauth ] && [ -r $OPENVAS_SYSCONF_DIR/proxyauth ]; then
          RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p $OPENVAS_SYSCONF_DIR/proxyauth\""
        else
          RSYNC_SSH_PROXY_CMD="-o \"ProxyCommand corkscrew $gsmproxy %h %p\""
        fi
      fi
      rsync -e "ssh $RSYNC_SSH_OPTS $RSYNC_SSH_PROXY_CMD -p $PORT -i $TEMP_ACCESS_KEY" --exclude=plugin_feed_info.inc $RSYNC_OPTIONS $RSYNC_DELETE $RSYNC_COMPRESS $RSYNC_CHMOD $feeduser $NVT_DIR
      if [ $? -ne 0 ]  ; then
        log_err "rsync failed, aborting synchronization."
        exit 1
      fi
      rsync -e "ssh $RSYNC_SSH_OPTS $RSYNC_SSH_PROXY_CMD -p $PORT -i $TEMP_ACCESS_KEY" $RSYNC_OPTIONS $RSYNC_DELETE $RSYNC_COMPRESS $RSYNC_CHMOD "$feeduser"plugin_feed_info.inc $NVT_DIR
      if [ $? -ne 0 ]  ; then
        log_err "rsync failed, aborting synchronization."
        exit 1
      fi
      eval "cd \"$NVT_DIR\" ; md5sum -c --status \"$NVT_DIR/md5sums\""
      if [ $? -ne 0 ]  ; then
        if [ -n "$retried" ]
        then
          log_err "Feed integrity check failed twice, aborting synchronization."
          cleanup_temp_access_key
          exit 1
        else
          log_write "The feed integrity check failed. This may be due to a concurrent feed update or other temporary issues."
          log_write "Sleeping 15 seconds before retrying ..."
          sleep 15
          retried=1
        fi
      else
        notsynced=0
      fi
    done
    cleanup_temp_access_key
    log_write "Synchronization with the Greenbone Security Feed successful."
    get_feed_info
    if [ $FEED_PRESENT -eq 1 ] ; then
      FEEDCOUNT=`grep -E "nasl$|inc$" $NVT_DIR/md5sums | wc -l`
      log_write "Current status: Using $FEED_NAME at version $FEED_VERSION ($FEEDCOUNT NVTs)"
    else
      log_write "Current status: No feed installed."
    fi
  else
    log_notice "No Greenbone Security Feed access key found, falling back to Greenbone Community Feed"
    do_rsync_community_feed
  fi
}

do_self_test ()
{
  MD5SUM_AVAIL=`command -v md5sum`
  if [ $? -ne 0 ] ; then
    SELFTEST_FAIL=1
    stderr_write "The md5sum binary could not be found."
  fi

  RSYNC_AVAIL=`command -v rsync`
  if [ $? -ne 0 ] ; then
    SELFTEST_FAIL=1
    stderr_write "The rsync binary could not be found."
  fi
}

do_describe ()
{
  echo "This script synchronizes an NVT collection with the '$FEED_NAME'."
  echo "The '$FEED_NAME' is provided by '$FEED_VENDOR'."
  echo "Online information about this feed: '$FEED_HOME'."
}

do_feedversion () {
  if [ $FEED_PRESENT -eq 1 ] ; then
    echo $FEED_VERSION
  else
    stderr_write "The file containing the feed version could not be found."
    exit 1
  fi
}

do_sync ()
{
  do_self_test
  if [ $SELFTEST_FAIL -ne 0 ] ; then
    exit $SELFTEST_FAIL
  fi

  if [ $FEED_CURRENT -eq 1 ]
  then
    log_write "Feed is already current, skipping synchronization."
  else
    (
      chmod +660 $OPENVAS_FEED_LOCK_PATH
      flock -n 9
      if [ $? -eq 1 ] ; then
          log_warning "Another process related to the feed update is already running"
          exit 1
      fi
      date > $OPENVAS_FEED_LOCK_PATH
      sync_nvts
      echo -n $OPENVAS_FEED_LOCK_PATH
    )9>>$OPENVAS_FEED_LOCK_PATH
  fi
}

do_help () {
  echo "$0: Sync NVT data"
  echo " --describe     display current feed info"
  echo " --feedcurrent  just check if feed is up-to-date"
  echo " --feedversion  display version of this feed"
  echo " --help         display this help"
  echo " --identify     display information"
  echo " --nvtdir dir   set dir as NVT directory"
  echo " --selftest     perform self-test and set exit code"
  echo " --verbose      makes the sync process print details"
  echo " --version      display version"
  echo ""
  echo ""
  echo "Environment variables:"
  echo "NVT_DIR         where to extract plugins (absolute path)"
  echo "PRIVATE_SUBDIR  subdirectory of \$NVT_DIR to exclude from synchronization"
  echo "TMPDIR          temporary directory used to download the files"
  echo "Note that you can use standard ones as well (e.g. RSYNC_PROXY) for rsync"
  echo ""
  exit 0
}

while test $# -gt 0; do
  case "$1" in
    --version)
      echo $VERSION
      exit 0
      ;;
    --identify)
      echo "NVTSYNC|$SCRIPT_NAME|$VERSION|$FEED_NAME|$RESTRICTED|NVTSYNC"
      exit 0
      ;;
    --selftest)
      do_self_test
      exit $SELFTEST_FAIL
      ;;
    --describe)
      do_describe
      exit 0
      ;;
    --feedversion)
      do_feedversion
      exit 0
      ;;
    --help)
      do_help
      exit 0
      ;;
    --nvt-dir)
      NVT_DIR="$2"
      shift
      ;;
    --feedcurrent)
      is_feed_current
      exit $?
      ;;
    --verbose)
      RSYNC_VERBOSE="-v"
      ;;
  esac
  shift
done

do_sync

exit 0