From 631b5197e225d039b75e1a23ead54e2f4c04bad1 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Oct 08 2012 16:27:02 +0000 Subject: fence_sanlock: various fixes and changes - change device arg name to path - add some metadata descriptions - in off, make request after first acquire fails - in off, always success after acquire works - in off, check if victim re-acquired cleanly - in off, check if already off at start - in status, use ^timestamp Signed-off-by: David Teigland --- diff --git a/fence_sanlock/fence_sanlock.in b/fence_sanlock/fence_sanlock.in index 1cea274..1140880 100755 --- a/fence_sanlock/fence_sanlock.in +++ b/fence_sanlock/fence_sanlock.in @@ -30,10 +30,10 @@ # # # -# +# # setup (one time only): -# fence_sanlock sanlock_init $device +# fence_sanlock -o sanlock_init -p $path # # startup: # service wdmd start @@ -44,7 +44,7 @@ max_hosts=128 opts= action= -device= +path= host_id= offset= @@ -56,14 +56,14 @@ help() { echo "Options:" echo " -o Action: off (default), on, status or metadata" echo " (sanlock specific actions: sanlock_init)" - echo " -d sanlock shared storage for leases" + echo " -p sanlock shared storage for leases" echo " -i sanlock host_id of node to operate on" echo " -h Print this help, then exit" echo " -V Print program version information, then exit" echo "" echo "stdin options:" echo " action=" - echo " device=" + echo " path=" echo " host_id=" } @@ -74,8 +74,8 @@ cli_options() { action=$2 shift ;; - -d) - device=$2 + -p) + path=$2 shift ;; -i) @@ -103,8 +103,8 @@ stdin_options() { action) action=$val ;; - device) - device=$val + path) + path=$val ;; host_id) host_id=$val @@ -117,7 +117,7 @@ stdin_options() { if [ $# -eq 0 ]; then stdin_options else - opts=$(getopt n:o:d:i:hV $@) + opts=$(getopt n:o:p:i:hV $@) if [ "$?" != 0 ]; then help exit 1 @@ -125,13 +125,15 @@ else cli_options $opts fi -# FIXME: Dave to review descriptions metadata() { cat << EOF - + -fence_sanlock is ... add long description here. +fence_sanlock is an i/o fencing agent that uses the watchdog device to +reset nodes. Shared storage (block or file) is used by sanlock to ensure +that fenced nodes are reset, and to notify partitioned nodes that they +need to be reset. http://www.redhat.com/ @@ -140,15 +142,15 @@ fence_sanlock is ... add long description here. Fencing Action - - + + - Path to sanlock shared storage device + Path to sanlock shared storage - Path to sanlock shared storage device + Host id for sanlock (1-128) @@ -163,31 +165,31 @@ EOF return 0 } -verify_device() { - # verify device has been initialized +verify_path() { + # verify storage has been initialized - leader=$(sanlock direct read_leader -r fence:h$host_id:$device:$offset 2>&1) + leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1) [ "$?" != 0 ] && { - echo "Unable to read $device information" + echo "Unable to read $path" return 1 } magic="$(echo "$leader" | grep magic | awk '{print $NF}')" [ -z "$magic" ] && { - echo "Unable to determine $device sanlock magic" + echo "Unable to determine $path sanlock magic" return 1 } [ "$magic" != "0x6152010" ] && { - echo "Error: $device magic does not match sanlock magic" + echo "Error: $path magic $magic does not match sanlock magic 0x6152010" return 1 } return 0 } action_on() { - verify_device || return 1 + verify_path || return 1 [ -z "$(pidof fence_sanlockd)" ] && { - daemonerr="$(fence_sanlockd -d $device -i $host_id 2>&1)" + daemonerr="$(fence_sanlockd -p $path -i $host_id 2>&1)" [ "$?" != 0 ] && { echo "Unable to execute fence_sanlockd. Error:" echo "$daemonerr" @@ -199,7 +201,7 @@ action_on() { # it can take minutes, and we can't allow fence_tool join # until this is complete - while ! sanlock client status | grep -q fence:h$host_id:$device:$offset; do + while ! sanlock client status | grep -q fence:h$host_id:$path:$offset; do # FIXME: check that r is really done being acquired? # just appearing in output may not be enough @@ -210,60 +212,97 @@ action_on() { } action_off() { - verify_device || return 1 + verify_path || return 1 owner_id="$(echo "$leader" | grep owner_id | awk '{print $NF}')" owner_gen="$(echo "$leader" | grep owner_gen | awk '{print $NF}')" ver="$(echo "$leader" | grep lver | awk '{print $NF}')" + timestamp="$(echo "$leader" | grep ^timestamp | awk '{print $NF}')" - # owner_id should equal host_id - [ "$owner_id" != "$host_id" ] && { - echo "Error: owner_id $owner_id does not match host_id $host_id" + [ -z "$timestamp" ] && { + echo "Unable to determine timestamp" return 1 } - sanlock client request -r fence:h$host_id:$device:$offset:$((ver + 1)) -f 2 > /dev/null 2>&1 - [ "$?" != 0 ] && { - echo "Unable to send sanlock client request" + # lease is released, so host is off + [ "$timestamp" = 0 ] && { + return 0 + } + + # owner_id should equal host_id + [ "$owner_id" != "$host_id" ] && { + echo "victim lease $host_id owned by $owner_id:$owner_gen" return 1 } pid="$(pidof fence_sanlockd)" [ -z "$pid" ] && { - echo "Unable to determine fence_sanlockd pin" + echo "Unable to determine fence_sanlockd pid" return 1 } + loop=0 + # FIXME: should this loop have a retry limit? while : do - sanlock client acquire -r fence:h$host_id:$device:$offset -p $pid + loop=$(($loop+1)) + + sanlock client acquire -r fence:h$host_id:$path:$offset -p $pid [ "$?" = 0 ] && { # fence success - sanlock client release -r fence:h$host_id:$device:$offset -p $pid + sanlock client release -r fence:h$host_id:$path:$offset -p $pid [ "$?" != 0 ] && { - # FIXME: what should we do here? - echo "Unable to release lock?" - #return 1 + echo "release $host_id error $?" } - break; + return 0 } - sleep 5 + if [ "$loop" = 1 ]; then + + # acquire probably failed because the victim is + # still alive and renewing its lease, (we could + # verify that by checking the error code, but the + # error codes are currently messed up due to + # negation). use a request on the victim's lease + # to tell it that it's being fenced and needs to + # reset. the -f 2 causes SIGUSR1 to be sent to + # fence_sanlockd on the victim. + + sanlock client request -r fence:h$host_id:$path:$offset:$((ver + 1)) -f 2 > /dev/null 2>&1 + [ "$?" != 0 ] && { + echo "request $host_id error $?" + } + fi + + sleep 10 + + # Reread the leader; if the victim's lease has been + # reacquired cleanly by the victim host (same host_id, new + # generation), we can quit with success - # FIXME: there probably some errors here where we should just fail + leader=$(sanlock direct read_leader -r fence:h$host_id:$path:$offset 2>&1) + tmp_id="$(echo "$leader" | grep owner_id | awk '{print $NF}')" + tmp_gen="$(echo "$leader" | grep owner_gen | awk '{print $NF}')" - # FIXME: we should probably reread the leader, and if it's been - # reacquired cleanly by the host, I think we can quit with success + if [ "$owner_id" -eq "$tmp_id" ] && [ "$owner_gen" -lt "$tmp_gen" ]; then + echo "victim $owner_id:$owner_gen reacquired lease gen $tmp_gen" + return 0 + fi + + if [ "$owner_id" -ne "$tmp_id" ]; then + echo "victim $owner_id:$owner_gen acquired by $tmp_id:$tmp_gen" + return 1 + fi done return 0 } action_status() { - verify_device || return 1 + verify_path || return 1 - timestamp="$(echo "$leader" | grep timestamp | awk '{print $NF}')" + timestamp="$(echo "$leader" | grep ^timestamp | awk '{print $NF}')" [ -z "$timestamp" ] && { echo "Unable to determine timestamp" @@ -282,17 +321,17 @@ action_status() { } sanlock_init() { - # initialize lease device - echo -n "Initializing lease device $device: " - sanlock direct init -s fence:0:$device:0 \ + # initialize lease path + echo -n "Initializing fence sanlock lockspace on $path: " + sanlock direct init -s fence:0:$path:0 \ > /dev/null 2>/dev/null || \ { echo "error $?" && return 1; } echo "ok" - echo -n "Initializing host leases: " + echo -n "Initializing $max_hosts sanlock host leases on $path: " for host_id in $(seq 1 $max_hosts); do offset=$((host_id * 1048576)) - sanlock direct init -r fence:h$host_id:$device:$offset \ + sanlock direct init -r fence:h$host_id:$path:$offset \ > /dev/null 2>/dev/null || \ { echo "error $? for host $host_id" && return 1; } done @@ -304,10 +343,10 @@ sanlock_init() { [ -z "$action" ] && action=off # check actions and options compatibility -# all actions beside metadata needs device +# all actions beside metadata needs storage [ "$action" != "metadata" ] && { - [ -z "$device" ] && { - echo "device argument required" + [ -z "$path" ] && { + echo "storage path argument required" exit 1 } # all actions beside sanlock_init needs host_id diff --git a/fence_sanlock/fence_sanlockd.c b/fence_sanlock/fence_sanlockd.c index 6a0de61..ea7ffdc 100644 --- a/fence_sanlock/fence_sanlockd.c +++ b/fence_sanlock/fence_sanlockd.c @@ -42,7 +42,7 @@ * start before cman and stop after cman. In future * we could have the init script start the daemon, * but that would also require a new config scheme - * since we wouldn't be getting the device and host_id + * since we wouldn't be getting the path and host_id * from cluster.conf via fence_node. * * simplest would be to have cman init just send us @@ -64,7 +64,7 @@ static int shutdown; static int we_are_victim; static int daemon_debug; static int our_host_id; -static char device[PATH_MAX]; +static char path[PATH_MAX]; static struct sanlk_lockspace ls; static struct sanlk_resource *r; static char rdbuf[sizeof(struct sanlk_resource) + sizeof(struct sanlk_disk)]; @@ -260,11 +260,11 @@ static int setup_signals(void) static void print_usage(void) { printf("Usage:\n"); - printf("fence_sanlockd -d -i \n"); + printf("fence_sanlockd -p -i \n"); printf("\n"); printf("Options:\n"); printf(" -D Enable debugging to stderr and don't fork\n"); - printf(" -d Path to shared storage with sanlock leases\n"); + printf(" -p Path to shared storage with sanlock leases\n"); printf(" -i Local sanlock host_id\n"); printf(" -h Print this help, then exit\n"); printf(" -V Print program version information, then exit\n"); @@ -282,14 +282,14 @@ int main(int argc, char *argv[]) int sock, con, rv, i; while (cont) { - optchar = getopt(argc, argv, "Dd:i:hV"); + optchar = getopt(argc, argv, "Dp:i:hV"); switch (optchar) { case 'D': daemon_debug = 1; break; - case 'd': - strcpy(device, optarg); + case 'p': + strcpy(path, optarg); break; case 'i': our_host_id = atoi(optarg); @@ -316,9 +316,9 @@ int main(int argc, char *argv[]) exit(1); } - if (!device[0]) { + if (!path[0]) { daemon_debug = 1; - log_error("device arg required"); + log_error("path arg required"); exit(1); } @@ -370,7 +370,7 @@ int main(int argc, char *argv[]) } memset(&ls, 0, sizeof(ls)); - sprintf(ls.host_id_disk.path, "%s", device); + sprintf(ls.host_id_disk.path, "%s", path); strcpy(ls.name, "fence"); ls.host_id = our_host_id; @@ -388,7 +388,7 @@ int main(int argc, char *argv[]) r = (struct sanlk_resource *)&rdbuf; strcpy(r->lockspace_name, "fence"); sprintf(r->name, "h%d", our_host_id); - sprintf(r->disks[0].path, "%s", device); + sprintf(r->disks[0].path, "%s", path); r->disks[0].offset = our_host_id * 1048576; r->num_disks = 1;