backstore

#!/bin/sh
################################################################################
#
# OCF resource agent to manage a Linux-IO backing LUN
# Copyright (c) 2014 Tiger Computing Ltd.
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
################################################################################

: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs

TCM_SYSFS=/sys/kernel/config/target/core
HBA_MAX=255 # arbitrary high limit
MULTISTATE=0 # set from backstore_validate_all()
ALUA_PTGP=default_tg_pt_gp
ALUA_PTGP_ID=0
LOCKFILE="${HA_RSCTMP}/${OCF_RESOURCE_TYPE}.lock"

################################################################################

backstore_meta_data() {
	cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="backstore">
<version>1.0</version>

<longdesc lang="en">
The backstore resource manages a Linux-IO (LIO) backing LUN. This LUN can then
be exported to an initiator via a number of transports, including iSCSI, FCoE,
Fibre Channel, etc...

This resource can be run a a single primitive or as a multistate (master/slave)
resource. When used in multistate mode, the resource agent manages ALUA
attributes for multipathing.
</longdesc>

<shortdesc lang="en">
Linux-IO LUN
</shortdesc>

<parameters>

<parameter name="hba_type" unique="0" required="1">
<longdesc lang="en">
The backing store HBA type, for example 'iblock' or 'fileio'.
</longdesc>
<shortdesc lang="en">
Backing store type
</shortdesc>
<content type="string" />
</parameter>

<parameter name="name" unique="0" required="1">
<longdesc lang="en">
The name of the LUN. Will be used when exporting the LUN via a transport, and
may optionally be exposed to the initiator.
</longdesc>
<shortdesc lang="en">
LUN name
</shortdesc>
<content type="string" />
</parameter>

<parameter name="device" unique="1" required="1">
<longdesc lang="en">
The backing device or file for the LUN.
</longdesc>
<shortdesc lang="en">
Backing device or file
</shortdesc>
<content type="string" />
</parameter>

<parameter name="unit_serial" unique="1" required="1">
<longdesc lang="en">
The T10 serial number to use for the LUN. Should be a UUID. This is exposed to
the initiator as the vendor-specific unit serial number, and is used to generate
the NAA WWN.
</longdesc>
<shortdesc lang="en">
The unit serial number.
</shortdesc>
<content type="string" />
</parameter>

<parameter name="attrib" unique="0" required="0">
<longdesc lang="en">
Backing store attributes to set, in key=value form. Attributes not listed here
will use default values set in the kernel.
</longdesc>
<shortdesc lang="en">
Backing store attributes
</shortdesc>
<content type="string" />
</parameter>

<parameter name="alua_hosts" unique="0" required="0">
<longdesc lang="en">
This attribute is required when running in multistate mode and ignored
otherwise. It is a space separated list of hostnames that this resource might
run on, which is used to generate consistent ALUA port group IDs.
</longdesc>
<shortdesc lang="en">
List of hosts this resource may run on
</shortdesc>
<content type="string" />
</parameter>

</parameters>

<actions>
<action name="start"        timeout="40" />
<action name="promote"      timeout="90" />
<action name="demote"       timeout="90" />
<action name="notify"       timeout="90" />
<action name="stop"         timeout="120" />
<action name="monitor"      depth="0" timeout="20" interval="10" />
<action name="monitor"      depth="0" timeout="20" interval="20" role="Slave" />
<action name="monitor"      depth="0" timeout="20" interval="10" role="Master" />
<action name="meta-data"    timeout="5" />
<action name="validate-all" timeout="30" />
</actions>
</resource-agent>
END

return $OCF_SUCCESS
}

backstore_methods() {
	cat <<-!
	start
	promote
	demote
	notify
	stop
	status
	monitor
	validate-all
	meta-data
	usage
	!
}

backstore_usage() {
	cat <<END
		usage: $0 {start|stop|monitor|status|validate-all|promote|demote|notify|meta-data}
END
}

find_hbadev()
{
	local hba_list
	local hba
	local hbadev

	# list all HBAs of the configured type
	hba_list=$(ls -1 $TCM_SYSFS | egrep "^${OCF_RESKEY_hba_type}_[0-9]+$")

	for hba in $hba_list; do
		hbadev="${hba}/${OCF_RESKEY_name}"
		if [ -d "${TCM_SYSFS}/${hbadev}" ]; then
			echo "$hbadev"
			return 0
		fi
	done

	return 1
}

set_alua()
{
	local hbadev="$1"
	local prop="$2"
	local value="$3"

	echo "$value" > "${TCM_SYSFS}/${hbadev}/alua/${ALUA_PTGP}/${prop}"
}

get_alua()
{
	local hbadev="$1"
	local prop="$2"

	cat "${TCM_SYSFS}/${hbadev}/alua/${ALUA_PTGP}/${prop}"
}

update_master_score()
{
	local hbadev="$1"

	# Try to find an HBA containing this device
	if [ -z "$hbadev" ]; then
		hbadev=$(find_hbadev)
		if [ -z "$hbadev" ]; then
			remove_master_score
			return 1
		fi
	fi

	# Count how many targets refer to us
	local num_tgt="$(get_alua "$hbadev" members | wc -l)"

	# Calculate our score
	local score=$(( $num_tgt * 1000 ))

	ocf_log info "setting master score to $score"
	if [ $score -gt 0 ]; then
		set_master_score $score
	else
		remove_master_score
	fi
}

set_master_score()
{
	# Use quiet mode (-Q) to quench logging. Actual score updates
	# will get logged by attrd anyway
	ocf_run ${HA_SBIN_DIR}/crm_master -Q -l reboot -v $1
}

remove_master_score()
{
	ocf_run ${HA_SBIN_DIR}/crm_master -l reboot -D
}

# A shell-like implementation of option parsing as found in the Linux kernel
# input is expected to be a comma-separated string of key=value pairs
match_opts()
{
	local optstr="$1"
	local match="$2"
	local opt

	local IFS=','
	for opt in $optstr; do
		if [ "${opt%%=*}" = "$match" ]; then
			echo "${opt#*=}"
			return 0
		fi
	done

	return 1
}

backstore_status()
{
	local hbadev
	local alua_state
	local alua_pref

	# If target_core_mod isn't loaded, we can't be running
	[ -e /sys/kernel/config/target/version ] || return $OCF_NOT_RUNNING

	# Try to find an HBA containing this device
	hbadev=$(find_hbadev)
	if [ -z "$hbadev" ]; then
		return $OCF_NOT_RUNNING
	fi

	# From this point the LUN is "running", though perhaps not fully
	# configured. We shouldn't return OCF_NOT_RUNNING but an error instead.

	# Check whether the device is enabled
	if [ "$(cat "${TCM_SYSFS}/${hbadev}/enable")" -ne 1 ]; then
		return $OCF_ERR_GENERIC
	fi

	# If this isn't a multistate resource, we're done
	[ $MULTISTATE -eq 0 ] && return $OCF_SUCCESS

	alua_state="$(get_alua "$hbadev" alua_access_state)"
	alua_pref="$(get_alua "$hbadev" preferred)"

	if [ "$alua_state" -eq 0 ] && [ "$alua_pref" -eq 1 ]; then
		return $OCF_RUNNING_MASTER
	elif [ "$alua_state" -eq 2 ] && [ "$alua_pref" -eq 0 ]; then
		return $OCF_SUCCESS # slave
	else
		return $OCF_FAILED_MASTER
	fi
}

backstore_monitor()
{
	local rc

	backstore_status; rc=$?

	case $rc in
	$OCF_SUCCESS|$OCF_RUNNING_MASTER)
		update_master_score
		;;
	$OCF_NOT_RUNNING)
		remove_master_score
		;;
	esac

	return $rc
}

backstore_start()
{
	local rc
	local hba_idx
	local hba
	local hbadev
	local attr

	# Check if we have anything to do
	backstore_monitor; rc=$?
	if [ $rc -eq $OCF_SUCCESS ]; then
		ocf_log info "Resource is already running"
		return $OCF_SUCCESS
	fi

	# Load target_core_mod if it isn't already
	if ! [ -e /sys/kernel/config/target/version ]; then
		ocf_run modprobe target_core_mod; rc=$?
		if [ $rc -ne 0 ]; then
			return $OCF_ERR_GENERIC
		fi
	fi

	# Take a lock so other starting backstores don't race with us and end up
	# with the same HBA instance
	ocf_take_lock $LOCKFILE
	ocf_release_lock_on_exit $LOCKFILE

	# Find a free HBA number
	for idx in $(seq 0 $HBA_MAX); do
		hba="${OCF_RESKEY_hba_type}_${idx}"
		[ ! -d "${TCM_SYSFS}/${hba}" ] && break
		hba=""
	done

	if [ -z "$hba" ]; then
		ocf_log err "Could not allocate a new HBA: no free indexes"
		return $OCF_ERR_GENERIC
	fi

	hbadev="${hba}/${OCF_RESKEY_name}"
	ocf_log debug "Using hbadev=${hbadev}"

	# Create the HBA and LUN inside it
	ocf_run tcm_node --establishdev "$hbadev" "$OCF_RESKEY_device"; rc=$?
	[ $rc -eq 0 ] || return $OCF_ERR_GENERIC

	# At this point we no longer need the lock
	rm -f $LOCKFILE

	# Set the unit serial number
	ocf_run tcm_node --setunitserialwithmd \
		"$hbadev" "$OCF_RESKEY_unit_serial"; rc=$?
	[ $rc -eq 0 ] || return $OCF_ERR_GENERIC

	# Establish ALUA Target Port Group
	ocf_run tcm_node --addaluatpgwithmd \
		"$hbadev" "$ALUA_PTGP" "$ALUA_PTGP_ID"; rc=$?
	[ $rc -eq 0 ] || return $OCF_ERR_GENERIC

	if [ $MULTISTATE -eq 0 ]; then
		ocf_log debug "Setting ALUA type to none"

		# Single-state LUN, no ALUA needed
		ocf_run tcm_node --setaluatype="$hbadev" "$ALUA_PTGP" none \
			|| return $OCF_ERR_GENERIC
		ocf_run tcm_node --clearaluapref="$hbadev" "$ALUA_PTGP" \
			|| return $OCF_ERR_GENERIC
		set_alua "$hbadev" nonop_delay_msecs 100 \
			|| return $OCF_ERR_GENERIC
		set_alua "$hbadev" trans_delay_msecs 0 \
			|| return $OCF_ERR_GENERIC
	else
		# Multi-state LUN, use ALUA. The RA should always start in
		# "slave" mode and be promoted to master by Pacemaker.

		ocf_log debug "Setting ALUA type to implicit"

		# Set Implicit only access type (cannot be set by initiator)
		ocf_run tcm_node --setaluatype="$hbadev" "$ALUA_PTGP" implict \
			|| return $OCF_ERR_GENERIC
		ocf_run tcm_node --clearaluapref="$hbadev" "$ALUA_PTGP" \
			|| return $OCF_ERR_GENERIC
		set_alua "$hbadev" nonop_delay_msecs 100 \
			|| return $OCF_ERR_GENERIC
		set_alua "$hbadev" trans_delay_msecs 0 \
			|| return $OCF_ERR_GENERIC

		# Standby # XXX: custom slave state?
		ocf_run tcm_node --setaluastate="$hbadev" "$ALUA_PTGP" s \
			|| return $OCF_ERR_GENERIC
	fi

	# Set attributes
	for attrib in ${OCF_RESKEY_attrib}; do
		local var=${attrib%%=*}
		local val=${attrib#*=}
		echo "$val" > "${TCM_SYSFS}/${hbadev}/attrib/${var}"; rc=$?
		[ $rc -eq 0 ] || return $OCF_ERR_GENERIC
	done

	[ $MULTISTATE -eq 1 ] && update_master_score

	return $OCF_SUCCESS
}

backstore_stop()
{
	local hbadev
	local hba
	local rc

	# If target_core_mod isn't loaded, we can't be running
	if ! [ -e /sys/kernel/config/target/version ]; then
		remove_master_score
		return $OCF_SUCCESS
	fi

	# Try to find an HBA containing this device
	hbadev=$(find_hbadev)
	if [ -z "$hbadev" ]; then
		ocf_log info "Resource is already stopped"
		remove_master_score
		return $OCF_SUCCESS
	fi

	# Extract the HBA name from the hba/dev combo
	hba=${hbadev%/*}

	# Delete the HBA
	# Using --delhba removes the entire HBA and any LUN(s) inside, but
	# leaves state data in /var/target intact. Using --freedev removes an
	# individual LUN and all its state information, but leaves the HBA
	# device in place.
	ocf_run tcm_node --delhba $hba; rc=$?
	if [ $rc -ne 0 ]; then
		remove_master_score
		return $OCF_ERR_GENERIC
	fi

	# FIXME: if we are the last to go, clean up and unload modules?

	remove_master_score
	return $OCF_SUCCESS
}

backstore_promote()
{
	local hbadev
	local status
	local rc=$OCF_ERR_GENERIC
	local first_try=true

	# Keep trying to promote the resource;
	# wait for the CRM to time us out if this fails
	while :; do
		backstore_monitor; status=$?

		case $status in
		$OCF_SUCCESS)
			ocf_log debug "attempting to promote"
			# We are slave, let's make ourselves master

			hbadev=$(find_hbadev)
			[ -n "$hbadev" ] || break

			# Active/Optimized
			ocf_run tcm_node --setaluastate="$hbadev" \
				"$ALUA_PTGP" o || break
			# Set preferred bit
			ocf_run tcm_node --setaluapref="$hbadev" \
				"$ALUA_PTGP" || break
			;;
		$OCF_NOT_RUNNING)
			ocf_log error "Trying to promote a resource that was not started"
			break
			;;
		$OCF_RUNNING_MASTER)
			ocf_log debug "promotion successful"
			rc=$OCF_SUCCESS
			break
			;;
		esac

		$first_try || sleep 1
		first_try=false
	done

	# avoid too tight pacemaker driven "recovery" loop,
	# if promotion keeps failing for some reason
	if [ $rc != 0 ]; then
		ocf_log warn "promotion failed; sleeping 15s to prevent tight recovery loop"
		sleep 15
	fi
	return $rc
}

backstore_demote()
{
	local hbadev
	local status
	local rc=$OCF_ERR_GENERIC
	local first_try=true

	# Keep trying to promote the resource;
	# wait for the CRM to time us out if this fails
	while :; do
		backstore_monitor; status=$?

		case $status in
		$OCF_SUCCESS)
			ocf_log debug "demotion successful"
			rc=$OCF_SUCCESS
			break
			;;
		$OCF_NOT_RUNNING)
			ocf_log error "Trying to demote a resource that was not started"
			break
			;;
		$OCF_RUNNING_MASTER)
			ocf_log debug "attempting to demote"
			# We are master, let's make ourselves slave

			hbadev=$(find_hbadev)
			[ -n "$hbadev" ] || break

			# Standby
			ocf_run tcm_node --setaluastate="$hbadev" \
				"$ALUA_PTGP" s || break
			# Clear preferred bit
			ocf_run tcm_node --clearaluapref="$hbadev" \
				"$ALUA_PTGP" || break
			;;
		esac

		$first_try || sleep 1
		first_try=false
	done

	# avoid too tight pacemaker driven "recovery" loop,
	# if demotion keeps failing for some reason
	if [ $rc != 0 ]; then
		ocf_log warn "demotion failed; sleeping 15s to prevent tight recovery loop"
		sleep 15
	fi
	return $rc
}

backstore_notify()
{
	local n_type=$OCF_RESKEY_CRM_meta_notify_type
	local n_op=$OCF_RESKEY_CRM_meta_notify_operation

	: "== DEBUG == slave   = $OCF_RESKEY_CRM_meta_notify_slave_uname"
	: "== DEBUG == master  = $OCF_RESKEY_CRM_meta_notify_master_uname"
	: "== DEBUG == start   = $OCF_RESKEY_CRM_meta_notify_start_uname"
	: "== DEBUG == promote = $OCF_RESKEY_CRM_meta_notify_promote_uname"
	: "== DEBUG == stop    = $OCF_RESKEY_CRM_meta_notify_stop_uname"
	: "== DEBUG == demote  = $OCF_RESKEY_CRM_meta_notify_demote_uname"

	backstore_monitor

	return $OCF_SUCCESS
}

backstore_validate_all()
{
	local rc

	if ocf_is_clone; then
		if ! ocf_is_ms; then
			ocf_log err "may only be a primitive or master/slave RA"
			return $OCF_ERR_CONFIGURED
		fi

		if [ "$OCF_RESKEY_CRM_meta_clone_max" -ne 2 ] ||
		   [ "$OCF_RESKEY_CRM_meta_clone_node_max" -ne 1 ] ||
		   [ "$OCF_RESKEY_CRM_meta_master_node_max" -ne 1 ] ||
		   [ "$OCF_RESKEY_CRM_meta_master_max" -ne 1 ]; then
			ocf_log err "Clone options misconfigured. (expect: clone_max=2,clone_node_max=1,master_node_max=1,master_max=1)"
			return $OCF_ERR_CONFIGURED
		fi

		if [ -z "$OCF_RESKEY_alua_hosts" ]; then
			ocf_log err "alua_hosts parameter required for multistate resources"
			return $OCF_ERR_CONFIGURED
		fi

		# Concoct an ALUA port group name and ID from our hostname and
		# resource instance number
		ALUA_PTGP="$(hostname -s)"

		local id=16
		local host
		for host in $OCF_RESKEY_alua_hosts; do
			if [ "$host" = "$ALUA_PTGP" ]; then
				ALUA_PTGP_ID=$id
				break
			fi
			id=$(($id + 1))
		done

		if [ $ALUA_PTGP_ID -eq 0 ]; then
			ocf_log err "alua_hosts does not include $ALUA_PTGP"
			return $OCF_ERR_CONFIGURED
		fi

		ocf_log debug "Running as a multi-state resource"
		MULTISTATE=1
	fi

	# ensure ALUA and PR state directories exist
	for DIR in /var/target/alua /var/target/pr; do
		[ -d "$DIR" ] || ocf_run mkdir -p "$DIR"; rc=$?
		if [ $rc -ne 0 ]; then
			ocf_log err "Could not create $DIR"
			return $OCF_ERR_INSTALLED
		fi
	done

	# ensure configfs is mounted
	if [ ! -d /sys/kernel/config ]; then
		ocf_run modprobe configfs; rc=$?
		if [ $rc -ne 0 ]; then
			ocf_log err "Unable to load the configfs module"
			return $OCF_ERR_INSTALLED
		fi
	fi
	if ! grep -q '/sys/kernel/config configfs' /proc/mounts; then
		ocf_run mount -t configfs configfs /sys/kernel/config; rc=$?
		if [ $rc -ne 0 ]; then
			ocf_log err "Unable to mount configfs on /sys/kernel/config"
			return $OCF_ERR_INSTALLED
		fi
	fi

	# ensure the hba_type is one we know about
	case "$OCF_RESKEY_hba_type" in
	iblock)
		# device is simply a (symlink to a) block device
		if [ ! -b "$OCF_RESKEY_device" ]; then
			ocf_log err "${OCF_RESKEY_device}: is not a block device"
			return $OCF_ERR_CONFIGURED
		fi
		;;
	fileio)
		# device is a parameter string that can contain multiple options:
		#   fd_dev_name     path to file on disk
		#   fd_dev_size     size of file on disk - required for files
		#                     but ignored/optional for block devices
		#   fd_buffered_io  whether IO should be buffered - default is
		#                     unbuffered/synchronous
		local dev_name="$(match_opts "$OCF_RESKEY_device" fd_dev_name)"
		local dev_size="$(match_opts "$OCF_RESKEY_device" fd_dev_size)"
		local buff_io="$(match_opts "$OCF_RESKEY_device" fd_buffered_io)"

		if [ -n "$buff_io" ] && [ "$buff_io" != "1" ]; then
			ocf_log err "fd_buffered_io must be '1' or not set"
			return $OCF_ERR_CONFIGURED
		fi
		if [ ! -b "$dev_name" ] && [ -z "$dev_size" ]; then
			ocf_log err "fd_dev_size must be given unless fd_dev_name is a block device"
			return $OCF_ERR_CONFIGURED
		fi
		;;
	*)
		ocf_log err "Unknown hba_type: ${OCF_RESKEY_hba_type}"
		return $OCF_ERR_CONFIGURED
		;;
	esac

	return $OCF_SUCCESS
}

OCF_REQUIRED_PARAMS="hba_type name device unit_serial"
OCF_REQUIRED_BINARIES="tcm_node"
ocf_rarun $*