xml-check

#!/usr/bin/env bash

# Check the input files for well-formedness and schema errors, produce a feedback file if appropriate

WORKSPACE=${WORKSPACE:-/work/space}
INPUTDIR=${1:-input}
SRCDIR=${2:-src}
FILEMASK=${3}*
DATASETS=${4:-iati-datasets}
PREFIX="XML check"
API=${API:-none}

VALLOGL=100
VALLOGS=50

mkdir -p \
  $WORKSPACE/tmp/xmltest \
  $WORKSPACE/tmp/xmltestlog \
  $WORKSPACE/tmp/xmlschemalog \
  $WORKSPACE/tmp/iatifeedback \
  $WORKSPACE/tmp/feedback \
  $WORKSPACE/input \
  $WORKSPACE/dest

cd $WORKSPACE/$INPUTDIR
find . -name "${FILEMASK}*" -type f -print | while read FILE; do

  NAME=`basename "$FILE"`
  MD5="${NAME%.xml}"
  
  cd $WORKSPACE/$INPUTDIR
  xmllint --recover --output "$WORKSPACE/tmp/xmltest/$NAME" "$NAME" 2> "$WORKSPACE/tmp/xmltestlog/$NAME"
  LL=`cat "$WORKSPACE/tmp/xmltestlog/$NAME" | wc -l`
  if [[ $LL -gt 100 ]]; then
    echo "$PREFIX $MD5: limiting XML validation log of $LL lines to $VALLOGL lines"
    sed -i "2001,$((LL-$VALLOGS))c...skipped $((LL-$VALLOGL)) lines..." "$WORKSPACE/tmp/xmltestlog/$NAME" 
  fi
  
  if [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
    mv "$WORKSPACE/tmp/xmltest/$NAME" $WORKSPACE/$SRCDIR
  else
    cp --preserve "$WORKSPACE/$INPUTDIR/$NAME" $WORKSPACE/$SRCDIR
    rm "$WORKSPACE/tmp/xmltest/$NAME"
  fi

  NSERROR=''
  if grep -q "namespace error" "$WORKSPACE/tmp/xmltestlog/$NAME"; then
    NSERROR=true
  else
    NSERROR=false
  fi

  ISIATI=''
  cd $WORKSPACE/$SRCDIR
  XMLROOT=`xmllint --xpath "local-name(/*)" "$NAME" 2> /dev/null`
  XMLATTR=`xmllint --xpath "string(/*/@version)" "$NAME" 2> /dev/null`
  if [[ "$XMLROOT." == "iati-activities." || "$XMLROOT." == "iati-organisations." ]]; then
    ISIATI=true
  else  
    ISIATI=false  
  fi

  if [[ $NSERROR == false && $ISIATI == true ]]; then
      if [[ $XMLATTR. =~ 1\.0[1-5]\. || $XMLATTR. =~ 2\.0[1-3]\. ]]; then
        VERSION=$XMLATTR
      elif [[ $XMLATTR. =~ 1\..* ]]; then
        VERSION=1.05
      else
        VERSION=2.03
      fi
      
      if xmllint --noout --schema /home/lib/iati-rulesets/lib/schemata/$VERSION/$XMLROOT-schema.xsd "$NAME" 2> "$WORKSPACE/tmp/xmlschemalog/$NAME"; then
        rm "$WORKSPACE/tmp/xmlschemalog/$NAME"
        if [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
          echo '<recovered-iati-file/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
          echo "$PREFIX $MD5: recovered-iati-file"
          
          if [[ "$API" != "none" ]]; then
            APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"recovered-iati-file\", \"iati-version\": \"$VERSION\"}"

            curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
              -d "$APIDATA" \
              "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
          fi        
        else
          echo '<iati-file/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
          echo "$PREFIX $MD5: iati-file"
          
          if [[ "$API" != "none" ]]; then
            APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"iati-file\", \"iati-version\": \"$VERSION\"}"

            curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
              -d "$APIDATA" \
              "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
          fi        
        fi
      else
        LL=`cat "$WORKSPACE/tmp/xmlschemalog/$NAME" | wc -l`
        if [[ $LL -gt 4000 ]]; then
          echo "$PREFIX $MD5: limiting schema validation log of $LL lines to $VALLOGL lines"
          sed -i "2001,$((LL-$VALLOGS))c...skipped $((LL-$VALLOGL)) lines..." "$WORKSPACE/tmp/xmlschemalog/$NAME" 
        fi

        if [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
          echo '<recovered-iati-file-with-schema-errors/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
          echo "$PREFIX $MD5: recovered-iati-file-with-schema-errors"

          if [[ "$API" != "none" ]]; then
            APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"recovered-iati-file-with-schema-errors\", \"iati-version\": \"$VERSION\"}"

            curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
              -d "$APIDATA" \
              "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
          fi        
        else
          echo '<iati-file-with-schema-errors/>' > "$WORKSPACE/tmp/iatifeedback/$NAME"
          echo "$PREFIX $MD5: iati-file-with-schema-errors"

          if [[ "$API" != "none" ]]; then
            APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"iati-file-with-schema-errors\", \"iati-version\": \"$VERSION\"}"

            curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
              -d "$APIDATA" \
              "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
          fi
        fi       
      fi
  
  elif [ -s "$WORKSPACE/tmp/xmltestlog/$NAME" ]; then
     
    echo '<not-an-xml-file/>' > "$WORKSPACE/tmp/feedback/$NAME"
    echo "$PREFIX $MD5: not-an-xml-file"
    rm "$WORKSPACE/src/$NAME"
    
    if [[ "$API" != "none" ]]; then
      APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"not-an-xml-file\"}"

      curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
        -d "$APIDATA" \
        "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
    fi    
  else
    echo '<not-an-iati-file/>' > "$WORKSPACE/tmp/feedback/$NAME"
    echo "$PREFIX $MD5: not-an-iati-file"
    rm "$WORKSPACE/src/$NAME"

    if [[ "$API" != "none" ]]; then
      APIDATA="{\"md5\": \"$MD5\", \"file-type\": \"not-an-iati-file\"}"

      curl -sS -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
        -d "$APIDATA" \
        "$API/$DATASETS/update?where=%7B%22md5%22%3A%22$MD5%22%7D"
    fi
  fi 

done