#!/bin/bash
#
# Copyright (C) Microsoft Corporation.
#
# SQL Server Always-On Availability Groups resource agent
#
# Valid actions are:
#    start
#    stop
#    promote
#    demote
#    monitor
#    validate-all
#    meta-data
#

# ----------------------------------------------------------------------------------------------------------
# Location of other files used by this resource agent
#
: "${AG_HELPER_BIN=${OCF_ROOT}/lib/mssql/ag-helper}"
: "${METADATA_FILE=${OCF_ROOT}/lib/mssql/agv2_metadata}"
: "${USAGE_FILE=${OCF_ROOT}/lib/mssql/agv2_usage}"
: "${AGV2_SELFCERT=${OCF_ROOT}/lib/mssql/agv2_selfcert.sh}"

# ----------------------------------------------------------------------------------------------------------
# Defaults values for optional parameters
#
: "${MONITOR_LEVEL_DEFAULT=3}"
: "${MONITOR_INTERVAL_DEFAULT=10}"
: "${MONITOR_TIMEOUT_DEFAULT=30}"
: "${CONNECTION_TIMEOUT_DEFAULT=15}"
: "${DISABLE_PRIMARY_ON_QUORUM_TIMEOUT_DEFAULT=30}"
: "${MONITORING_CREDENTIALS_FILE_DEFAULT=/var/opt/mssql/secrets/passwd}"
: "${SELF_SIGNED_CERTIFICATE_FILE_DEFAULT=/var/opt/mssql/secrets/pcsaghost.pfx}"
: "${PCSAG_SERVICE_NAME_DEFAULT=mssql-pcsag}"
: "${PCSAG_PROCESS_NAME_DEFAULT=PcsAgAgent}"
: "${PORT_DEFAULT=1433}"
: "${RESTFUL_API_PORT=57151}"
: "${PROCESS_NAME_DEFAULT=sqlservr}"
: "${PRIMARY_LEASE_DURATION_DEFAULT=$[MONITOR_INTERVAL_DEFAULT+MONITOR_TIMEOUT_DEFAULT+2]}"
: "${REQUIRED_SYNCHRONIZED_SECONDARIES_TO_COMMIT_DEFAULT=-1}" # -1 is a sentinel that ag-helper interprets as "unset"

# Subtract this from the exit code of ag-helper to get the OCF exit code.
# This is the inverse of the operation performed by `mssqlcommon.OcfExit` used by ag-helper.
OCF_EXIT_DIFFERENCE=10

# ----------------------------------------------------------------------------------------------------------
# Pacemaker libraries
#
: "${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}"
. "$OCF_FUNCTIONS"
: "${__OCF_ACTION=$1}"

# ----------------------------------------------------------------------------------------------------------
# function: mssql_meta_data
#
# Description:
#    Implements the OCF "meta-data" action.
#
mssql_meta_data() {
	cat "$METADATA_FILE"
}

# ----------------------------------------------------------------------------------------------------------
# function: usage
#
mssql_usage() {
	cat "$USAGE_FILE"
}

mssql_conf="/var/opt/mssql/mssql.conf"

if [ -f $mssql_conf ]; then
	hostname=$(sed -n -e '/^\s*\[network]\s*/I,/\s*ipaddress\s*=./I {s/^\s*ipaddress\s*=\s*\(.*\)/\1/I p}' $mssql_conf)
fi

# ----------------------------------------------------------------------------------------------------------
# function: mssql_start
#
# Description:
#    Implements the OCF "start" action.
#
mssql_start() {
	ocf_log info 'mssql_start'

	# Fetch sequence numbers of all replicas
	#
	local sequence_numbers="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-sequence-number" -QA)"
	local escaped_sns=$(jq -Rs . <<< "$sequence_numbers")
	ocf_log info "mssql_start: escaped_sn: $escaped_sns"

	local command_output
	local rc

	if ! pidof "$OCF_RESKEY_process_name"; then
		# SQL Server crashed or isn't running at all.
		# Set promotion score to -INFINITY so pacemaker will not promote this node by mistake. 
		# Once SQL process is recovered, promotion score will be reset to 10 by start() -> monitor().
		set_promotion_score "PROMOTION_SCORE: 0"
		ocf_exit_reason "SQL Server isn't running."
		return "$OCF_NOT_RUNNING"
	fi

	# RHEL9 introduces a breaking change about crm_resource cmd return value from "Master" to "Promoted". 
	# We need to change grep pip condition accordingly so our agent code supports both old and new pacemaker version.
	local current_master="$(crm_resource -r "$OCF_RESOURCE_INSTANCE" --locate | grep -Po 'resource [^ ]+ is running on: \K(.+)(?= (Master|Promoted)$)')"
	ocf_log info "mssql_start current_master: $current_master"

	json_payload=$(cat <<EOF 
{
	"AgName": "$OCF_RESKEY_ag_name",
	"ApplicationName": "monitor-$OCF_RESOURCE_INSTANCE-start",
	"Action": "Start",
	"SequenceNumbers": $escaped_sns,
	"CurrentMaster": "$current_master",
	"RequiredSynchronizedSecondariesToCommit":$OCF_RESKEY_required_synchronized_secondaries_to_commit,
	"ActionTimeout": $OCF_RESKEY_meta_timeout_sec,
	"PrimaryWriteLeaseDuration": $OCF_RESKEY_primary_write_lease_duration,
	"Description": "This is for RestFul API Test"
}
EOF
	)

	command_output=$(
		curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/start/$OCF_RESKEY_ag_name
	)
	ocf_log info "mssql_start: command_output: $command_output"
	rc=$(jq -r '.ocfReturnCode' <<<"$command_output")
	ocf_log info "mssql_start :rc: $rc"

	set_exit_reason "$command_output"

	$promotion_score= $(jq -r '.promotionScore' <<<"$command_output")
	ocf_log info "mssql_start : set_promotion_score $promotion_score"
	set_promotion_score "PROMOTION_SCORE: $promotion_score"

	if [ -z "$rc" ]; then
		ocf_exit_reason 'Could not find ocfReturnCode output.'
		return "$OCF_ERR_GENERIC"
	fi
	return "$rc"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_stop
#
# Description:
#    Implements the OCF "stop" action.
#
mssql_stop() {
	ocf_log info 'mssql_stop'

	local command_output
	local rc

	if ! pidof "$OCF_RESKEY_process_name"; then
		# SQL Server crashed. This is as good as stopped.
		# Even if it was a PRIMARY before it crashed and automatically restarts, it will come back as RESOLVING.
		ocf_exit_reason "SQL Server isn't running."
		return "$OCF_NOT_RUNNING"
	fi

	# Reserve 5s for killing the SQL Server process if ag-helper fails
	# and use the rest to run ag-helper
	local stop_timeout="$(( OCF_RESKEY_CRM_meta_timeout / 1000 - 5 ))"
	if (( stop_timeout < 5 )); then
		# The stop timeout should be atleast 10 seconds so that there's enough time to change the AG replica's role,
		# and kill the SQL Server process if that fails. If the user set it shorter, there doesn't seem to be any way to
		# tell them that while still having the `stop` action succeed (failure would cause the node to be fenced).
		#
		# So print an error to cluster log and then pretend the change role failed without actually trying to change the role.
		# This will kill the process and hopefully trigger the user to read the cluster log and discover this error.
		#
		# Note that Pacemaker documentation recommends both that the default timeout specified in a resource's metadata (10s in this case)
		# is the minimum that the user should set, and further recommends not setting *any* timeout less than 10 seconds anyway.
		# So the user who sets this timeout to less than 10 seconds is going out of their way to do it.
		ocf_log error 'The stop action timeout should be at least 10 seconds'
		rc="$OCF_ERR_GENERIC"
	else
			json_payload=$(cat <<EOF
{
	"AgName": "$OCF_RESKEY_ag_name",
	"ApplicationName": "monitor-$OCF_RESOURCE_INSTANCE-stop",
	"Action": "Stop",
	"ActionTimeout": $OCF_RESKEY_meta_timeout_sec,
	"Description": "This is for RestFul API Test"
}
EOF
			)
		command_output=$(
			curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/stop/$OCF_RESKEY_ag_name
		)

		ocf_log info "mssql_stop: command_output: $command_output"

		rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
		ocf_log info "mssql_stop :rc: $rc"

		set_exit_reason "$command_output"
	fi

	case "$rc" in
		"$OCF_SUCCESS")
			# ag-helper succeeded. Nothing else to do.
			#
			;;
		"$OCF_ERR_GENERIC")
			# ag-helper failed to set the AG replica to SECONDARY Role.
			# Kill the instance to ensure that it stops being in PRIMARY role.
			#
			ocf_log info 'Skipping killing SQL Server process...'

			local mssql_pid="$(pidof "$OCF_RESKEY_process_name")"
			ocf_log info "SQL Server process id before kill: $mssql_pid"
			;;
	esac

	if [ -z "$rc" ]; then
		ocf_exit_reason 'Could not find ocfReturnCode output.'
		return "$OCF_ERR_GENERIC"
	fi
	return "$rc"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_monitor
#
# Description:
#    Implements the OCF "monitor" action.
#
mssql_monitor() {
	ocf_log info 'mssql_monitor'

	local command_output
	local rc

	if ! pidof "$OCF_RESKEY_process_name"; then
		# SQL Server crashed or isn't running at all.
		# Set promotion score to -INFINITY so pacemaker will not promote this node by mistake. 
		# Once SQL process is recovered, promotion score will be reset to 10 by start() -> monitor().
		set_promotion_score "PROMOTION_SCORE: 0"
		ocf_exit_reason "SQL Server isn't running."
		return "$OCF_NOT_RUNNING"
	fi
	
	if (( OCF_RESKEY_meta_timeout_sec <= OCF_RESKEY_connection_timeout )); then
		ocf_log info "WARNING: Monitor timeout is lower than connection timeout. Connection will not retried if connection timeout occurs"
	fi	

	# RHEL9 introduces a breaking change about crm_resource cmd return value from "Master" to "Promoted". 
	# We need to change grep pip condition accordingly so our agent code supports both old and new pacemaker version.
	local current_master="$(crm_resource -r "$OCF_RESOURCE_INSTANCE" --locate | grep -Po 'resource [^ ]+ is running on: \K(.+)(?= (Master|Promoted)$)')"
	ocf_log info "mssql_monitor current_master: $current_master"

	json_payload=$(cat <<EOF
{
	"AgName": "$OCF_RESKEY_ag_name",
	"ApplicationName": "monitor-$OCF_RESOURCE_INSTANCE-monitor",
	"Action": "Monitor",
	"CurrentMaster": "$current_master",
	"RequiredSynchronizedSecondariesToCommit":$OCF_RESKEY_required_synchronized_secondaries_to_commit,
	"DisablePrimaryOnQuorumTimeoutAfter": $OCF_RESKEY_disable_primary_on_quorum_timeout_after,
	"PrimaryWriteLeaseDuration": $OCF_RESKEY_primary_write_lease_duration,
	"ActionTimeout": $OCF_RESKEY_meta_timeout_sec,
	"Description": "This is for RestFul API Test"
}
EOF
	)
	command_output=$(
		curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/monitor/$OCF_RESKEY_ag_name
	)
	ocf_log info "mssql_monitor: command_output: $command_output"

	rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
	ocf_log info "mssql_monitor return $rc"

	set_exit_reason "$command_output"
	
	local lease_expiry=$(jq -r '.leaseExpiry' <<<"$command_output")

	attrd_updater -n "$OCF_RESOURCE_INSTANCE-lease-expiry" -U "$lease_expiry" -p

	local lease="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-lease-expiry" -QA)"
	
	ocf_log info "lease_expiry after monitor update: $lease_expiry"
	ocf_log info "lease information from all replicas after monitor update: $lease"

	echo "$lease" | systemd-cat

	local promotion_score=$(jq -r '.promotionScore' <<<"$command_output")
	ocf_log info "mssql_monitor set_promotion_score to $promotion_score"
	set_promotion_score "PROMOTION_SCORE: $promotion_score"

	if [ -z "$rc" ]; then
		ocf_exit_reason 'Could not find ocfReturnCode output.'
		return "$OCF_ERR_GENERIC"
	fi
	return "$rc"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_promote
#
# Description:
#    Implements the OCF "promote" action.
#
mssql_promote() {
	ocf_log info 'mssql_promote'

	if ! pidof "$OCF_RESKEY_process_name"; then
		# SQL Server crashed.
		ocf_exit_reason "SQL Server isn't running."
		return "$OCF_NOT_RUNNING"
	fi

	local command_output
	local rc

	# Fetch sequence numbers of all replicas
	#
	local sequence_numbers="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-sequence-number" -QA)"
	local escaped_sns=$(jq -Rs . <<< "$sequence_numbers")
	ocf_log info "mssql_promote: escaped_sn: $escaped_sns"
	json_payload=$(cat <<EOF
{
	"AgName": "$OCF_RESKEY_ag_name",
	"ApplicationName": "monitor-$OCF_RESOURCE_INSTANCE-promote",
	"Action": "Promote",
	"SequenceNumbers": $escaped_sns,
	"CurrentMaster": "$OCF_RESKEY_CRM_meta_notify_promote_uname",
	"RequiredSynchronizedSecondariesToCommit":$OCF_RESKEY_required_synchronized_secondaries_to_commit,
	"DisablePrimaryOnQuorumTimeoutAfter": $OCF_RESKEY_disable_primary_on_quorum_timeout_after,
	"PrimaryWriteLeaseDuration": $OCF_RESKEY_primary_write_lease_duration,
	"ActionTimeout": $OCF_RESKEY_meta_timeout_sec,
	"Description": "This is for RestFul API Test"
}
EOF
	)
	command_output=$(
		curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/promote/$OCF_RESKEY_ag_name
	)
	ocf_log info "mssql_promote: command_output: $command_output"

	rc=$(jq -r '.ocfReturnCode' <<<"$command_output")
	ocf_log info "mssql_promote return $rc"
	
	local lease_expiry=$(jq -r '.leaseExpiry' <<<"$command_output")

	attrd_updater -n "$OCF_RESOURCE_INSTANCE-lease-expiry" -U "$lease_expiry" -p

	local lease="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-lease-expiry" -QA)"

	echo "$lease" | systemd-cat

	set_exit_reason "$command_output"

	local promotion_score=$(jq -r '.promotionScore' <<<"$command_output")
	ocf_log info "mssql_monitor set_promotion_score to $promotion_score"
	set_promotion_score "PROMOTION_SCORE: $promotion_score"
	
	if [ -z "$rc" ]; then
		ocf_exit_reason 'Could not find ocfReturnCode output.'
		return "$OCF_ERR_GENERIC"
	fi
	return "$rc"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_demote
#
# Description:
#    Implements the OCF "demote" action.
#
mssql_demote() {
	ocf_log info 'mssql_demote'

	if ! pidof "$OCF_RESKEY_process_name"; then
		# SQL Server crashed. This is as good as demoted.
		# Even if it was a PRIMARY before it crashed and automatically restarts, it will come back as RESOLVING.
		ocf_exit_reason "SQL Server isn't running."
		return "$OCF_SUCCESS"
	fi

	local command_output
	local rc
	local potential_primarys="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-potential-primary" -QA)"
	local escaped_pps=$(jq -Rs . <<< "$potential_primarys")
	ocf_log info "mssql_demote: escaped_sn: $escaped_pps"

	json_payload=$(cat <<EOF
{
	"AgName": "$OCF_RESKEY_ag_name",
	"ApplicationName": "monitor-$OCF_RESOURCE_INSTANCE-demote",
	"Action": "Demote",
	"SequenceNumbers": $escaped_pps,
	"ActionTimeout": $OCF_RESKEY_meta_timeout_sec,
	"Description": "This is for RestFul API Test"
}
EOF
	)
	command_output=$(
		curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/demote/$OCF_RESKEY_ag_name
	)
	ocf_log info "mssql_demote: command_output: $command_output"

	rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
	ocf_log info "mssql_demote return $rc"
	
	local lease_expiry=$(jq -r '.leaseExpiry' <<<"$command_output")

	attrd_updater -n "$OCF_RESOURCE_INSTANCE-lease-expiry" -U "$lease_expiry" -p

	local lease="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-lease-expiry" -QA)"

	echo "$lease" | systemd-cat

	set_exit_reason "$command_output"
	
	if [ -z "$rc" ]; then
		ocf_exit_reason 'Could not find ocfReturnCode output.'
		return "$OCF_ERR_GENERIC"
	fi
	return "$rc"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_notify
#
# Description:
#    Implements the OCF "notify" action.
#
mssql_notify() {
	ocf_log info "mssql_notify $OCF_RESKEY_CRM_meta_notify_type-$OCF_RESKEY_CRM_meta_notify_operation"
	json_payload=$(cat <<EOF
{
	"AgName": "$OCF_RESKEY_ag_name",
	"ApplicationName": "monitor-$OCF_RESOURCE_INSTANCE-notify",
	"Action": "Notify",
	"SequenceNumbers": "",
	"RequiredSynchronizedSecondariesToCommit":$OCF_RESKEY_required_synchronized_secondaries_to_commit,
	"ActionTimeout": $OCF_RESKEY_meta_timeout_sec,
	"Description": "This is for RestFul API Test"
}
EOF
	)

	local command_output
	local rc

	case "$OCF_RESKEY_CRM_meta_notify_type-$OCF_RESKEY_CRM_meta_notify_operation" in
		'pre-start')
			command_output=$(
				curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/PreStart/$OCF_RESKEY_ag_name
			)
			ocf_log info "pre-start: command_output: $command_output"

			rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
			ocf_log info "pre-start return $rc"
			;;

		'post-promote')
			# Reset sequence number attribute so that it doesn't retain old values for subsequent starts or promotes
			attrd_updater -n "$OCF_RESOURCE_INSTANCE-sequence-number" -D
			command_output=$(
				curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/PostPromote/$OCF_RESKEY_ag_name
			)
			ocf_log info "post-promote: command_output: $command_output"
			rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
			ocf_log info "post-promote return $rc"
			;;
			
		'post-start')
			# Reset sequence number attribute so that it doesn't retain old values for subsequent starts or promotes
			attrd_updater -n "$OCF_RESOURCE_INSTANCE-sequence-number" -D 
			return "$OCF_SUCCESS"
			;;	
		'pre-demote')
			command_output=$(
				curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/PreDemote/$OCF_RESKEY_ag_name
			)
			ocf_log info "pre-demote: command_output: $command_output"
			rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
			ocf_log info "pre-demote return $rc"
			set_exit_reason "$command_output"

			if (( rc == OCF_SUCCESS )); then
				attrd_updater -n "$OCF_RESOURCE_INSTANCE-potential-primary" -U "1" -p

				# Work around attrd bug https://bugzilla.redhat.com/show_bug.cgi?id=1463033
				# attrd_updater can receive ack from attrd for the update before attrd has propagated the value to other nodes
				# or even committed it locally
				local attribute_value=''
				while :; do
					attribute_value="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-potential-primary" -Q)"
					if [ -n "$attribute_value" ]; then
						break
					fi

					sleep 5
				done
				ocf_log info "attrd_updater $OCF_RESOURCE_INSTANCE-potential-primary to 1"
			fi
			return "$OCF_SUCCESS"
			;;
		'post-demote')
			attrd_updater -n "$OCF_RESOURCE_INSTANCE-potential-primary" -D
			return "$OCF_SUCCESS"
			;;

		'post-stop')
			command_output=$(
				curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/PostStop/$OCF_RESKEY_ag_name
			)
			ocf_log info "post-stop: command_output: $command_output"

			rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
			ocf_log info "post-stop return $rc"
			;;

		'pre-promote')
			command_output=$(
				curl -X POST -H "Content-Type: application/json" -H "Authorization: Basic $basicAuth" -d "$json_payload" https://localhost:$OCF_RESKEY_restful_api_port/api/PcsAgAgent/PrePromote/$OCF_RESKEY_ag_name
			)
			ocf_log info "pre-promote: command_output: $command_output"

			rc=$(jq -r '.ocfReturnCode' <<<"$command_output")		
			ocf_log info "pre-promote return $rc"
			;;

		*)
			return "$OCF_SUCCESS"
			;;
	esac

	set_exit_reason "$command_output"

	case "$OCF_RESKEY_CRM_meta_notify_type-$OCF_RESKEY_CRM_meta_notify_operation" in
		'pre-start' | 'pre-promote')
			if (( rc == OCF_SUCCESS )); then
				# Find sequence number in ag-helper's output
				#
				local sequence_number=$(jq -r '.sequenceNumber' <<<"$command_output")

				if [ -z "$sequence_number" ]; then
					ocf_exit_reason 'Could not find sequence number in ag-helper output.'
					return "$OCF_SUCCESS"
				fi

				attrd_updater -n "$OCF_RESOURCE_INSTANCE-sequence-number" -U "$sequence_number" -p

				# Work around attrd bug https://bugzilla.redhat.com/show_bug.cgi?id=1463033
				# attrd_updater can receive ack from attrd for the update before attrd has propagated the value to other nodes
				# or even committed it locally
				local attribute_value=''
				while :; do
					attribute_value="$(attrd_updater -n "$OCF_RESOURCE_INSTANCE-sequence-number" -Q)"
					if [ -n "$attribute_value" ]; then
						break
					fi

					sleep 5
				done

				# -Q returns the value when it's committed locally but not necessarily propagated to other nodes,
				# so sleep some more to let the update propagate
				# sleep 5

				ocf_log info "attrd_updater $OCF_RESOURCE_INSTANCE-sequence-number to $sequence_number"
			fi
			;;
	esac

	ocf_log info "mssql_notify $OCF_RESKEY_CRM_meta_notify_type-$OCF_RESKEY_CRM_meta_notify_operation return $rc"
	if [ -z "$rc" ]; then
		ocf_exit_reason 'Could not find ocfReturnCode output.'
		return "$OCF_ERR_GENERIC"
	fi
	return "$OCF_SUCCESS"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_validate
#
# Description:
#    Implements the OCF "validate-all" action.
#
# Returns:
#    OCF_SUCCESS: The credentials file exists and the ag-helper binary is present.
#    OCF_ERR_ARGS: The credentials file does not exist.
#    OCF_ERR_CONFIGURED: The ag-helper binary is not present.
#
mssql_validate() {
	ocf_log info 'mssql_validate'
	local restartService=false

	# Set default parameters
	#
	: "${OCF_RESKEY_disable_primary_on_quorum_timeout_after=$DISABLE_PRIMARY_ON_QUORUM_TIMEOUT_DEFAULT}"
	: "${OCF_RESKEY_monitoring_credentials_file=$MONITORING_CREDENTIALS_FILE_DEFAULT}"
	: "${OCF_RESKEY_SELF_SIGNED_CERTIFICATE_FILE_DEFAULT=$SELF_SIGNED_CERTIFICATE_FILE_DEFAULT}"
	: "${OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT=$PCSAG_SERVICE_NAME_DEFAULT}"
	: "${OCF_RESKEY_PCSAG_PROCESS_NAME_DEFAULT=$PCSAG_PROCESS_NAME_DEFAULT}"
	: "${OCF_RESKEY_monitor_policy=$MONITOR_LEVEL_DEFAULT}"
	: "${OCF_RESKEY_port=$PORT_DEFAULT}"
	: "${OCF_RESKEY_restful_api_port=$RESTFUL_API_PORT}"
	
	ocf_log info "OCF_RESKEY_CRM_meta_interval value: $OCF_RESKEY_CRM_meta_interval"
	ocf_log info "OCF_RESKEY_CRM_meta_timeout value: $OCF_RESKEY_CRM_meta_timeout"
	# try to set write_lease_duration value based on monitor meta
	: "${OCF_RESKEY_primary_write_lease_duration=$(( OCF_RESKEY_CRM_meta_interval / 1000 + OCF_RESKEY_CRM_meta_timeout / 1000 + 2 ))}"
	: "${OCF_RESKEY_process_name=$PROCESS_NAME_DEFAULT}"

	# monitor_timeout is an old alias for connection_timeout
	: "${OCF_RESKEY_connection_timeout:=$OCF_RESKEY_monitor_timeout}"
	: "${OCF_RESKEY_connection_timeout:=$CONNECTION_TIMEOUT_DEFAULT}"

	# required_copies_to_commit is an old alias for required_synchronized_secondaries_to_commit
	: "${OCF_RESKEY_required_synchronized_secondaries_to_commit:=$OCF_RESKEY_required_copies_to_commit}"
	: "${OCF_RESKEY_required_synchronized_secondaries_to_commit:=$REQUIRED_SYNCHRONIZED_SECONDARIES_TO_COMMIT_DEFAULT}"
	
	# timeout value in seconds	
	: "${OCF_RESKEY_meta_timeout_sec=$(( OCF_RESKEY_CRM_meta_timeout / 1000 ))}"

	# Check binaries necessary for the resource agent to run
	#
	# check_binary "$AG_HELPER_BIN"

	# Check credentials file
	#
	if [[ ! -f "$OCF_RESKEY_monitoring_credentials_file" ]]; then
		ocf_exit_reason "Credentials file at $OCF_RESKEY_monitoring_credentials_file does not exist"
		return "$OCF_ERR_ARGS"
	fi

	# check the self-signed certificate.
	if [[ ! -f "$OCF_RESKEY_SELF_SIGNED_CERTIFICATE_FILE_DEFAULT" ]]; then
		ocf_log info "Certificate file at $OCF_RESKEY_SELF_SIGNED_CERTIFICATE_FILE_DEFAULT does not exist, run selfcert.sh"
		source "$AGV2_SELFCERT"
		restartService=true
	fi

	if [[ ! -f "$OCF_RESKEY_SELF_SIGNED_CERTIFICATE_FILE_DEFAULT" ]]; then
		ocf_exit_reason "Certificate file at $OCF_RESKEY_SELF_SIGNED_CERTIFICATE_FILE_DEFAULT does not exist after running selfcert.sh"
		return "$OCF_ERR_ARGS"
	fi

	if pidof "$OCF_RESKEY_PCSAG_PROCESS_NAME_DEFAULT" > /dev/null; then
		ocf_log info "process $OCF_RESKEY_PCSAG_PROCESS_NAME_DEFAULT is running."
	else
		ocf_log info "process $OCF_RESKEY_PCSAG_PROCESS_NAME_DEFAULT is not running. restart it."
		restartService=true
	fi

	# restart the pcsag service if needed.
	if [ "$restartService" = true ]; then
		ocf_log info "service $OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT is not running. restart it."
		systemctl enable "$OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT"
		systemctl restart "$OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT"

		#verify restart.
		if systemctl is-active --quiet "$OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT"; then
			ocf_log info "service $OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT restarted successfully."
		else
			ocf_exit_reason "failed to restart service $OCF_RESKEY_PCSAG_SERVICE_NAME_DEFAULT"
			return "$OCF_ERR_ARGS"
		fi
	fi

	# Check notify=true
	if [[ "$OCF_RESKEY_CRM_meta_notify" != 'true' ]]; then
		ocf_exit_reason 'Resource must be configured with notify=true'
		return "$OCF_ERR_CONFIGURED"
	fi

	return "$OCF_SUCCESS"
}

# ----------------------------------------------------------------------------------------------------------
# function: mssql_export_ocf_exit_codes
#
# Description:
#    Exports the OCF exit code variables as environment variables for sub-processes.
#
mssql_export_ocf_exit_codes() {
	export \
        OCF_ERR_ARGS OCF_ERR_CONFIGURED OCF_ERR_GENERIC OCF_ERR_PERM OCF_ERR_UNIMPLEMENTED \
        OCF_FAILED_MASTER OCF_NOT_RUNNING \
        OCF_RUNNING_MASTER OCF_SUCCESS
}

# ----------------------------------------------------------------------------------------------------------
# function: set_exit_reason
#
# Description:
#    Extracts the exit reason from the given command output if it exists, and sets it.
#
set_exit_reason() {
	local exit_reason="$(echo "$1" | grep -Po '^ERROR: \K.*' | head -n1)"
	if [[ -n "$exit_reason" ]]; then
		ocf_exit_reason "$exit_reason"
	fi
}

# ----------------------------------------------------------------------------------------------------------
# function: set_promotion_score
#
# Description:
#    Extracts the promotion score value from the given command output and sets it.
#    If no output is found, sets the promotion score to `-INFINITY`
#
set_promotion_score() {
	local promotion_score="$(echo "$1" | grep -Po '^PROMOTION_SCORE: \K.*')"
	if [ -z "$promotion_score" ]; then
		crm_master -v '0' -l reboot
	else
		crm_master -v "$promotion_score" -l reboot
	fi
}

# ----------------------------------------------------------------------------------------------------------
#
if [[ "$__OCF_ACTION" = 'meta-data' ]]; then
	mssql_meta_data
	exit "$OCF_SUCCESS"
fi

mssql_validate
validate_result="$?"

ocf_log info "Resource agent invoked with: $__OCF_ACTION"

# Everything else must pass validation
if (( validate_result != 0 )); then
	exit "$validate_result"
fi

mssql_export_ocf_exit_codes

# read user id & pw from credential file.
read -r userId < "$OCF_RESKEY_monitoring_credentials_file"
read -r userPw < <(tail -n +2 "$OCF_RESKEY_monitoring_credentials_file")
basicAuth=$(base64 <<< "$userId:$userPw")

case "$__OCF_ACTION" in
	'start')
		mssql_start
		;;
	'stop')
		mssql_stop
		;;
	'monitor')
		mssql_monitor
		;;
	'promote')
		mssql_promote
		;;
	'demote')
		mssql_demote
		;;
	'notify')
		mssql_notify
		;;
	'validate-all')
		exit "$validate_result"
		;;
	'usage' | 'help')
		mssql_usage
		exit "$OCF_SUCCESS"
		;;
	*)
		mssql_usage
		exit "$OCF_ERR_UNIMPLEMENTED"
		;;
esac
rc="$?"

ocf_log info "$OCF_RESOURCE_INSTANCE $__OCF_ACTION : $rc"
exit "$rc"
