#!/bin/sh

# $Name:  $
# $Id: edg-fetch-crl.cin,v 1.5 2006/01/16 08:49:37 pmacvsdg Exp $


###############################################################################
# File:        edg-fetch-crl                                                  #
#                                                                             #
# Version:     2.5                                                            #
#                                                                             #
# Description: this script is useful to download and install a set of         #
#              certificate revocation lists (CRL) published by the            #
#              Certification Authorities supported by the DataGRID project.   #
#              Each CRL file is downloaded, appropiately named and copied to  #
#              the specified directory so that Globus can find it.            #
#                                                                             #
# Usage:       edg-fetch-crl [-h|--help]                                      #
#              edg-fetch-crl [-l|--loc locationDirectory]                     #
#                            [-o|--out outputDirectory] [-q|--quiet]          #
#                            [-a|--agingtolerance hours ]                     #
#                                                                             #
# Author:      Fabio Hernandez                                                #
#              fabio@in2p3.fr                                                 #
#              IN2P3 Computer Center                                          #
#              http://www.in2p3.fr/CC                                         #
#              Lyon (FRANCE)                                                  #
#                                                                             #
# Date:        Aug 2001                                                       #
#              Dec 2002 - fix problem with openssl                            #
#              Apr 2003 - add support for EDG v2.0-style config files         #
#              Feb 2005 - DG - fix security vulnerability related to tmpfiles #
#              Feb 2005 - DG - make it paranoid about overwriting any.r0 file #
#              Feb 2005 - DG - new packaging format for RPM, no cron job left #
#              Aug 2005 - DG - ensure the latest version of OpenSSL gets used #
#              Oct 2005 - RW - fix https handling problem with wget           #
#              Nov 2005 - DG - fix issue with overwriting good CRL by old one #
#              Jan 2006 - DG - allow for entirely untrusted DL source for CRL #
#                                                                             #
###############################################################################

#-----------------------------------------------------------------------------#
#                        I N I T I A L I Z A T I O N                          #
#-----------------------------------------------------------------------------#

#
# Needed commands: it is useful to specify the full path of the needed commands here
# in order to be able to run this script within te context of a user whitout the
# PATH environment varible initialized (e.g. cron, root, ...)
#
openssl=/usr/bin/openssl
lynx=/usr/bin/lynx
wget=/usr/bin/wget
basename=/bin/basename
getopt=/usr/bin/getopt
awk=/bin/awk
cat=/bin/cat
cp=/bin/cp
chmod=/bin/chmod
chown=/bin/chown
chgrp=/bin/chgrp
mv=/bin/mv
rm=/bin/rm
id=/usr/bin/id
ls=/bin/ls
date=/bin/date
sed=/bin/sed
grep=/bin/grep
mktemp=/bin/mktemp
stat=/usr/bin/stat

#
# Global variables
#
programName=`${basename} $0`
tempDir="/tmp"                   # temporary directory
verboseMode=1                    # enable message display
outputDirectory=`pwd`            # default output directory is current directory
crlLocationFileSuffix="crl_url"  # this script will look for files with this extension
cRLAgingThreshold=0              # maximum age of a local CRL before download
                                 # errors are shown to the user
noServerCertCheck=0              # require valid server cert
wgetAdditionalOptions=""         # require valid server cert

# get defaults
WGET_RETRIES=3
WGET_TIMEOUT=30


#-----------------------------------------------------------------------------#
#                               R O U T I N E S                               #
#-----------------------------------------------------------------------------#

#
# RetrieveFileByURL - downloads a file given a URL and writes its contents to the
#                     file pointed to by the ${tempFile} variable. You can use
#                     replace the call to lynx by wget if you prefer.
#                     Returns 0 if the specified URL can be donwloaded and stored
#                     in a file
#
RetrieveFileByURL()
{
   url=$1
   destinationFile=$2

   if [ -s ${destinationFile} ]; then
     PrintError "RetrieveFileByURL: temporary file ${destinationFile} unexpectedly full of data"
     exit 1
   fi
   #
   # If you don't have 'wget' installed on your machine  or you prefer use 'lynx' instead, 
   # uncomment next line and comment the following one.
   #
   # ${lynx} -source ${url} > ${destinationFile}
   #
   wgetOptions="${wgetAdditionalOptions}"
   test "$noServerCertCheck" -eq 1 && wgetOptions="${wgetOptions} --no-check-certificate"
   if [ `${wget} --help | ${grep} -c "ca-directory"` -eq 1 ]; then
     ${wget} $wgetOptions --ca-directory=${locationDirectory} \
             -q -t $WGET_RETRIES -T $WGET_TIMEOUT -O ${destinationFile} ${url}
   else
     ${wget} $wgetOptions -q -t $WGET_RETRIES -T $WGET_TIMEOUT \
             -O ${destinationFile} ${url}
   fi
   return $?
}

#
# ShowUsage - show this program usage
#
ShowUsage()
{
   echo
   echo "Usage:" ${programName} "[-h|--help]" 
   echo "      " ${programName} "[-l|--loc <locationDirectory>]" 
   echo "                     [-o|--out <outputDirectory>] [-q|--quiet]" 
   echo "                     [-a|--agingtolerance <hours>]"
   echo
   echo "   Options:"
   echo
   echo "      -h|--help show this help"
   echo
   echo "      -l|--loc  <locationDirectory>"
   echo "                The script will search this directory for files with the"
   echo "                suffix '.${crlLocationFileSuffix}'. It is supposed that each one of these"
   echo "                files contains the URL of a Certificate Revocation List (CRL)"
   echo "                for a Certification Authority. This URL is of the form "
   echo "                http://www.myhost.com/myCRL."
   echo "                Note: the CRL files to download must be in either PEM or"
   echo "                      DER format."
   echo "                For validity checking of the CA certificates, this script"
   echo "                assumes that the certificates of the CAs are found also"
   echo "                in this directory."
   echo "                Default: output directory (see below)"
   echo
   echo "      -o|--out  <outputDirectory>"
   echo "                directory where to put the downloaded and processed CRLs."
   echo "                The directory to be used as argument for this option"
   echo "                is typically /etc/grid-security/certificates"
   echo "                Default: current working directory"
   echo 
   echo "      -a|--agingtolerance hours"
   echo "              The  maximum  age  of the locally downloaded CRL before download"
   echo "              failures trigger actual error messages. This error message  sup-"
   echo "              pression  mechanism  only  works  if the crl_url files are named"
   echo "              after the hash of the CRL issuer  name,  a  stat(1)  command  is"
   echo "              installed,  and a CRL has already been downloaded at least once."
   echo 
   echo
   echo "      -q|--quiet"
   echo "                Quiet mode (do not print information messages)"
   echo
   echo "      -n|--no-check-certificate"
   echo "                Do not check the server certificate when downloading URLs"
   echo
   echo "   Defaults can be set in the fetch-crl system configuration file"
   echo "   /etc/sysconfig/fetch-crl, see manual for details."
   echo
}

#
# Print information message
#
PrintMessage()
{
   if [ ${verboseMode} -eq 0 ]; then
      return
   fi

   timeStamp=`${date} +%Y/%m/%d-%H:%M:%S`
   echo ${programName}": ["${timeStamp}"]" $*
}

#
# Print error message
#
PrintError()
{
   timeStamp=`${date} +%Y/%m/%d-%H:%M:%S`
   echo ${programName}": ["${timeStamp}"]" $* 1>&2
}

#
# get date of lastUpdate from CRL file in standard format YYYYMMDDhh (dont use mmss, since
# some systems cannot handle numbers larger than MAXINT in test(1) comparisons
#
LastUpdateOfCRL()
{
  crlhashfile=$1

  if [ ! -r ${crlhashfile} ]; then
    lastUpdate=0000000000
    return
  fi

  u2date='
  BEGIN {
   im["Jan"]=1;im["Feb"]=2;im["Mar"]=3;im["Apr"]=4;im["May"]=5;im["Jun"]=6;
   im["Jul"]=7;im["Aug"]=8;im["Sep"]=9;im["Oct"]=10;im["Nov"]=11;im["Dec"]=12;
  }
  /.*Update=/ { 
    m=substr($1,index($1,"=")+1); 
    h=substr($3,0,2); mi=substr($3,4,2); s=substr($3,7,2); 
    printf "%04d%02d%02d%02d\n",$4,im[m],$2,h;
  }
  '

  lastUpdateText=`${openssl} crl -noout -in ${crlhashfile} -lastupdate`
  lastUpdate=`echo $lastUpdateText | ${awk} "$u2date"`
}

#
# ValidateCRLHashFile
#
ValidateCRLHashFile()
{
   crlhashfile=$1

   conversionSucceeded="no"
   supportedFormats="PEM DER"
   for format in ${supportedFormats}; do
      crlHashValue=`${openssl} crl -hash -inform ${format} -in ${crlhashfile} \
                                   -noout 2>/dev/null | ${awk} '{print $1}'`

      if [ "X"${crlHashValue} != "X" ]; then
         conversionSucceeded="yes"
         break
      fi
   done

   fileHashValue=`${basename} ${crlhashfile} ".r0"`
   if [ x"${fileHashValue}" != x"${crlHashValue}" ]; then
      conversionSucceeded="no"
   fi

   crlHashFileIsValid=${conversionSucceeded}

   PrintMessage "File ${crlhashfile} valid: ${crlHashFileIsValid}"
}

#
# ProcessCRLFile
#
ProcessCRLFile()
{
   downloadedFile=$1
   
   #
   # Compute hash value to build the CRL file name
   #
   pemFile=`${mktemp} -q ${tempDir}/crlpem-XXXXXX`
   if [ $? -ne 0 ]; then
       PrintError "can't create temp file in ${tempDir}, exiting..."
       exit 1
   fi
   conversionSucceeded="no"
   supportedFormats="PEM DER"
   for format in ${supportedFormats}; do
      crlHashValue=`${openssl} crl -hash -inform ${format} -in ${downloadedFile} \
                                   -out ${pemFile} -text 2>/dev/null | ${awk} '{print $1}'`

      if [ "X"${crlHashValue} != "X" ]; then
         conversionSucceeded="yes"
         break
      fi
   done

   ${rm} -f ${downloadedFile} 2>/dev/null
   if [ ${conversionSucceeded} = "no" ]; then
      return 1
   fi

   #
   # Rename the converted CRL file
   #
   result=${pemFile}
   resulthash=${crlHashValue}

   #
   # We are done
   # 
   return 0
}


#-----------------------------------------------------------------------------#
#                                 M  A  I  N                                  #
#-----------------------------------------------------------------------------#

# read defaults that used to be set by the cron job
if [ -r /etc/sysconfig/fetch-crl ] ; then
  . /etc/sysconfig/fetch-crl
  if [ "X${CRLDIR}" != "X" ]; then
    locationDirectory="${CRLDIR}"
    outputDirectory="${CRLDIR}"
  fi
  if [ "X${QUIET}" == "Xyes" ]; then
    verboseMode=0
  fi
  if [ "X${SERVERCERTCHECK}" == "Xno" ]; then
    noServerCertCheck=1
  fi
  if [ "X${WGET_OPTS}" != "X" ]; then
    wgetAdditionalOptions="${WGET_OPTS}"
  fi
fi

#
# Parse the command line
#
getoptResult=`${getopt} -o hl:o:qa:n -a -l help,loc:,out:,quiet,agingtolerance,no-check-certificate -n ${programName} -- "$@"`
if [ $? != 0 ] ; then
   ShowUsage
   exit 1
fi

eval set -- "${getoptResult}"
while true ; do
   case "$1" in
      -h|--help)  helpRequested="true" ; shift ;;
      -l|--loc)   locationDirectory=$2; shift 2 ;;
      -n|--no-check-certificate)   noServerCertCheck=1; shift 1 ;;
      -o|--out)   outputDirectory=$2; shift 2 ;;
      -q|--quiet) verboseMode=0; shift ;;
      -a|--agingtolerance) cRLAgingThreshold=$2; shift 2 ;;
      --)         shift; break;;
      *)          echo ${programName}": internal error!" ; exit 1 ;;
   esac
done

#
# Are there extra arguments?
#
if [ $1 ]; then
   echo ${programName}": unexpected argument '"$1"'"
   ShowUsage
   exit 1
fi

#
# Did the user request help?
#
if [ "X${helpRequested}" = "Xtrue" ]; then
   ShowUsage
   exit 0
fi

#
# Make sure that we can write to the specified output directory
#
if [ ! -d ${outputDirectory} -o ! -w ${outputDirectory} ]; then
   PrintError "'"${outputDirectory}"' is not a directory or cannot be written"
   exit 1
fi

#
# Look for the Globus configuration file and extract the root of the Globus installation and the
# path of the configuration file
#
globusSysconfigFile="/etc/sysconfig/globus"
if [ -r ${globusSysconfigFile} ]; then
    globusLocation=`${grep} -i "^[[:space:]]*GLOBUS_LOCATION" ${globusSysconfigFile} | ${sed} "s/^[[:space:]]*//g" | ${awk} -F'=' '{print $2}' | ${sed} "s/[[:space:]]*//g"`
    if [ "X${globusLocation}" != "X" ]; then
        GLOBUS_LOCATION=${globusLocation}
    fi

    globusConfigurationFile=`${grep} -i "^[[:space:]]*GLOBUS_CONFIG" ${globusSysconfigFile} | ${sed} "s/^[[:space:]]*//g" | ${awk} -F'=' '{print $2}' | ${sed} "s/[[:space:]]*//g"`
fi


#
# Make sure the location directory exists
#
if [ "X${locationDirectory}" = "X" ]; then
   #
   # Location directory is not supplied. Let's try to find where it may be.
   # Look into the Globus configuration file for extracting the directory where
   # the certificates are located.
   #
   if [ "X${globusConfigurationFile}" = "X" ]; then
      globusConfigurationFile="/etc/globus.conf"
   fi

   if [ -r ${globusConfigurationFile} ]; then
      certDir=`${grep} "^[ ]*X509_CERT_DIR" ${globusConfigurationFile} | ${sed} "s/^[[:space:]]*//g" | ${awk} -F'=' '{print $2}' | ${sed} "s/[[:space:]]*//g"`
      if [ "X${certDir}" != "X" ]; then
         if [ -d ${certDir} ]; then
            locationDirectory=${certDir}
         fi
      fi
   fi
fi

if [ "X${locationDirectory}" = "X" ]; then
   locationDirectory=${outputDirectory}
fi

if [ ! -d ${locationDirectory} ]; then
   PrintError "'"${locationDirectory}"' is not a directory or cannot be read"
   exit 1
fi

#
# This script needs "openssl", which can be installed within the Globus
# hierarchy or elsewhere. Let's try to find it, but make sure we get the
# latest version
#
if test "x${FETCH_CRL_OPENSSL}" = "x" 
then
  OIFS="$IFS"
  IFS=":"
  openssl_paths=""
  for p in $PATH ; do
    if test "x${openssl_paths}" = "x" 
    then
      openssl_paths="$p/openssl"
    else
      openssl_paths="${openssl_paths}:$p/openssl"
    fi
  done
  openssl_paths="${openssl_paths}:$openssl:/usr/local/bin/openssl:/usr/bin/openssl"
  if [ ! -z "${GLOBUS_LOCATION}" ]; then
     if [ -x ${GLOBUS_LOCATION}/bin/openssl ]; then
        openssl_paths="${GLOBUS_LOCATION}/bin/openssl:$openssl_paths"
     fi
  fi

  oversion="OpenSSL 0"
  for probe in $openssl_paths ; do
    if test -x "$probe" ; then
      pversion=`"$probe" version 2>/dev/null`
      if test `expr "x$pversion" \> "x$oversion"` -eq 1 ; then
        openssl="$probe"
        oversion="$pversion"
      fi
    fi
  done
  IFS="$OIFS"
  PrintMessage "Using OpenSSL version $oversion at $openssl"
else
  openssl="${FETCH_CRL_OPENSSL}"
  PrintMessage "Using prespecified version of OpenSSL at $openssl"
fi

if [ ! -x "${openssl}" ]; then
  PrintError "openssl not found - define GLOBUS_LOCATION or create '${globusConfigFile}'"
  exit 1
fi


#
# Initialize the group name for the 'globus' user
#

#
# Look for CRL location files with the expected suffix
#
locationFiles=`${ls} ${locationDirectory}/*.${crlLocationFileSuffix} 2>/dev/null`
if [ "X${locationFiles}" = "X" ]; then
   PrintError "no files with suffix '."${crlLocationFileSuffix}"' found in '"${locationDirectory}"'"
   exit 1
fi

#
# Process each one of the CRL location files
#
for nextLocationFile in ${locationFiles}; do

   PrintMessage "processing '"${nextLocationFile}"'"

   CRLDownloadError=0
   errorsCRLDownloadError=""

   while true ; do
      #
      # Extract the next URL from this CRL location file
      #
      read nextLine
      if [ $? != 0 ]; then
         break
      fi

      nextURL=`echo ${nextLine} | ${awk} -F'#' '{print $1}'`
      if [ -z ${nextURL} ]; then
         # This is a comment or a blank line, skip it
         continue
      fi

      #
      # Download this CRL
      #
      tempFile=`${mktemp} -q ${tempDir}/crl-dg.XXXXXX`
      if [ $? -ne 0 ]; then
             PrintError "can't create temp file in ${tempDir}, exiting..."
             exit 1
      fi

      RetrieveFileByURL ${nextURL} ${tempFile}
      if [ $? != 0 ]; then
         CRLDownloadError=1
         errorsCRLDownloadError="$errorsCRLDownloadError ${nextURL}"
         PrintMessage "could not download a valid file from '"${nextURL}"'"
         ${rm} -f ${tempFile}
         continue
      else
         CRLDownloadError=0
         errorMessageCRLDownloadError=""
      fi

      #
      # Process and rename the downloaded file
      #
      ProcessCRLFile ${tempFile}
      if [ $? != 0 ]; then
         continue
      fi
      crlFile=${result}
      crlHash=${resulthash}

      #
      # Verify this CRL
      #
      issuer=`${openssl} crl -inform "PEM" -in ${crlFile} -issuer -noout | ${awk} '{print substr($0,index($0,"/CN=")+4)}'`
      issuer="$issuer (${resulthash})"
      verifyResult=`${openssl} crl -CApath ${locationDirectory} -in ${crlFile} -noout 2>&1`
      if [ "X${verifyResult}" != "Xverify OK" ]; then
         PrintError "verify failed for CRL issued by '"${issuer}"' (${verifyResult})"
         ${rm} -f ${crlFile} 2>/dev/null
         continue
      fi

      #
      # Move the temporary file to the output directory and set the appropriate file
      # permissions and ownership
      #
      PrintMessage "updating CRL '"${issuer}"'"
      ${chmod} 0644 ${crlFile}
      finalCrlFileName="${crlHash}.r0"

      if [ -e ${outputDirectory}/${finalCrlFileName} ]; then
        ValidateCRLHashFile ${outputDirectory}/${finalCrlFileName}
        if [ x"${crlHashFileIsValid}" != x"yes" ]; then
          PrintError "Attempt to overwrite" \
		${outputDirectory}/${finalCrlFileName} \
		"failed since the original is not a valid CRL file"
          exit 1
        fi
      fi

      # is the new CRL indeed newer than the current one?
      LastUpdateOfCRL ${outputDirectory}/${finalCrlFileName}
      currentLastUpdate=$lastUpdate
      LastUpdateOfCRL ${crlFile}
      newLastUpdate=$lastUpdate

      if [ $newLastUpdate -lt $currentLastUpdate ]; then
        PrintError "Attempt to install " \
		${finalCrlFileName} \
		"failed since the current CRL is more recent " \
                "than the one that was downloaded."
      else
        ${mv} ${crlFile} ${outputDirectory}/${finalCrlFileName} > /dev/null 2>&1
      fi

      #
      # Check the validity of the CA certificate
      #
      caCertificate=`${basename} ${finalCrlFileName} ".r0"`".0"
      verifyResult=`${openssl} verify -CApath ${locationDirectory} ${locationDirectory}/${caCertificate} 2>&1 | ${awk} '{print $2}'`
      if [ "X${verifyResult}" != "XOK" ]; then
         PrintError "verify failed for CA certificate issued by '"${issuer}"' (${verifyResult})"
      fi
   done < ${nextLocationFile}  # while


   if [ $CRLDownloadError -ne 0 ]; then
      # this may be a cause for errors, but suppress if nextLocationFile
      # name resembles a hash AND the associated hash.r0 file is younger than
      # cRLAgingThreshold
      if [ -x ${stat} ]; then 
      hashFileName=`basename ${nextLocationFile} .${crlLocationFileSuffix}`
      if [ `expr ${hashFileName} : [a-fA-F0-9]\\\\{8\\\\}` -eq 8 ]; then
        if [ -f ${outputDirectory}/${hashFileName}.r0 ]; then
          currentTimeFile=`${mktemp} -q ${tempDir}/fetch-crl-ts.XXXXXX`
          nowAge=`${stat} -t ${currentTimeFile} | ${awk} '{print $13}'`
          ${rm} -f ${currentTimeFile}
          hashFileTime=`${stat} -t ${outputDirectory}/${hashFileName}.r0 | \
                          ${awk} '{print $13}'`
          hashFileAge=`expr \( $nowAge - $hashFileTime \) / 3600`
          if [ ${hashFileAge} -le ${cRLAgingThreshold} ]; then
            CRLDownloadError=0
            PrintMessage "CRL download error for ${hashFileName} suppressed"
          else
            PrintError "Persistent errors (${hashFileAge} hours) for ${hashFileName}:"
          fi
        fi
      fi
      fi

      if [ $CRLDownloadError -ne 0 ]; then
        PrintError "Could not download any CRL from $nextLocationFile:"
        for url in $errorsCRLDownloadError 
        do
          PrintError "download failed from '"${url}"'"
        done
      fi
   fi

done # for

#
# Done
#
exit 0
