Add log_must_{retry,busy} helpers
Add helpers which automatically retry the provided command when the error message matches the provided keyword. This provides an easy way to handle the asynchronous nature of some ZFS commands. For example, the `zfs destroy` command may need to be retried in the case where the block device is unexpected busy. This can be accomplished as follows: log_must_busy $ZFS destroy ... Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov> Issue #5002
This commit is contained in:
parent
2158b165ed
commit
e623aea2ec
@ -69,6 +69,81 @@ function log_must
|
||||
(( $? != 0 )) && log_fail
|
||||
}
|
||||
|
||||
# Execute a positive test but retry the command on failure if the output
|
||||
# matches an expected pattern. Otherwise behave like log_must and exit
|
||||
# $STF_FAIL is test fails.
|
||||
#
|
||||
# $1 - retry keyword
|
||||
# $2 - retry attempts
|
||||
# $3-$@ - command to execute
|
||||
#
|
||||
function log_must_retry
|
||||
{
|
||||
typeset out=""
|
||||
typeset logfile="/tmp/log.$$"
|
||||
typeset status=1
|
||||
typeset expect=$1
|
||||
typeset retry=$2
|
||||
typeset delay=1
|
||||
shift 2
|
||||
|
||||
while [[ -e $logfile ]]; do
|
||||
logfile="$logfile.$$"
|
||||
done
|
||||
|
||||
while (( $retry > 0 )); do
|
||||
"$@" 2>$logfile
|
||||
status=$?
|
||||
out="$CAT $logfile"
|
||||
|
||||
if (( $status == 0 )); then
|
||||
$out | $EGREP -i "internal error|assertion failed" \
|
||||
> /dev/null 2>&1
|
||||
# internal error or assertion failed
|
||||
if [[ $? -eq 0 ]]; then
|
||||
print -u2 $($out)
|
||||
_printerror "$@" "internal error or" \
|
||||
" assertion failure exited $status"
|
||||
status=1
|
||||
else
|
||||
[[ -n $LOGAPI_DEBUG ]] && print $($out)
|
||||
_printsuccess "$@"
|
||||
fi
|
||||
break
|
||||
else
|
||||
$out | $GREP -i "$expect" > /dev/null 2>&1
|
||||
if (( $? == 0 )); then
|
||||
print -u2 $($out)
|
||||
_printerror "$@" "Retry in $delay seconds"
|
||||
$SLEEP $delay
|
||||
|
||||
(( retry=retry - 1 ))
|
||||
(( delay=delay * 2 ))
|
||||
else
|
||||
break;
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if (( $status != 0 )) ; then
|
||||
print -u2 $($out)
|
||||
_printerror "$@" "exited $status"
|
||||
fi
|
||||
|
||||
_recursive_output $logfile "false"
|
||||
return $status
|
||||
}
|
||||
|
||||
# Execute a positive test and exit $STF_FAIL is test fails after being
|
||||
# retried up to 5 times when the command returns the keyword "busy".
|
||||
#
|
||||
# $@ - command to execute
|
||||
function log_must_busy
|
||||
{
|
||||
log_must_retry "busy" 5 "$@"
|
||||
(( $? != 0 )) && log_fail
|
||||
}
|
||||
|
||||
# Execute a negative test and exit $STF_FAIL if test passes
|
||||
#
|
||||
# $@ - command to execute
|
||||
|
Loading…
Reference in New Issue
Block a user