Blame SOURCES/bz1816205-fence_aws-fix-race-condition.patch

97bc7e
--- a/agents/aws/fence_aws.py	2020-03-26 10:31:03.653171381 +0100
97bc7e
+++ b/agents/aws/fence_aws.py	2020-03-24 16:21:16.942155519 +0100
97bc7e
@@ -3,14 +3,33 @@
97bc7e
 import sys, re
97bc7e
 import logging
97bc7e
 import atexit
97bc7e
+import requests
97bc7e
 sys.path.append("@FENCEAGENTSLIBDIR@")
97bc7e
 from fencing import *
97bc7e
-from fencing import fail, fail_usage, EC_TIMED_OUT, run_delay
97bc7e
+from fencing import fail, fail_usage, run_delay, EC_STATUS, SyslogLibHandler
97bc7e
 
97bc7e
 import boto3
97bc7e
 from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError
97bc7e
 
97bc7e
+logger = logging.getLogger("fence_aws")
97bc7e
+logger.propagate = False
97bc7e
+logger.setLevel(logging.INFO)
97bc7e
+logger.addHandler(SyslogLibHandler())
97bc7e
+logging.getLogger('botocore.vendored').propagate = False
97bc7e
+	
97bc7e
+def get_instance_id():
97bc7e
+	try:
97bc7e
+		r = requests.get('http://169.254.169.254/latest/meta-data/instance-id')
97bc7e
+		return r.content.decode("UTF-8")
97bc7e
+	except HTTPError as http_err:
97bc7e
+		logger.error('HTTP error occurred while trying to access EC2 metadata server: %s', http_err)
97bc7e
+	except Exception as err:
97bc7e
+		logger.error('A fatal error occurred while trying to access EC2 metadata server: %s', err)
97bc7e
+	return None
97bc7e
+	
97bc7e
+
97bc7e
 def get_nodes_list(conn, options):
97bc7e
+	logger.info("Starting monitor operation")
97bc7e
 	result = {}
97bc7e
 	try:
97bc7e
 		for instance in conn.instances.all():
97bc7e
@@ -19,13 +38,17 @@
97bc7e
 		fail_usage("Failed: Incorrect Access Key or Secret Key.")
97bc7e
 	except EndpointConnectionError:
97bc7e
 		fail_usage("Failed: Incorrect Region.")
97bc7e
-
97bc7e
+	except Exception as e:
97bc7e
+		logger.error("Failed to get node list: %s", e)
97bc7e
+	logger.debug("Monitor operation OK: %s",result)
97bc7e
 	return result
97bc7e
 
97bc7e
 def get_power_status(conn, options):
97bc7e
+	logger.debug("Starting status operation")
97bc7e
 	try:
97bc7e
 		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [options["--plug"]]}])
97bc7e
 		state = list(instance)[0].state["Name"]
97bc7e
+		logger.info("Status operation for EC2 instance %s returned state: %s",options["--plug"],state.upper())
97bc7e
 		if state == "running":
97bc7e
 			return "on"
97bc7e
 		elif state == "stopped":
97bc7e
@@ -38,20 +61,49 @@
97bc7e
 	except EndpointConnectionError:
97bc7e
 		fail_usage("Failed: Incorrect Region.")
97bc7e
 	except IndexError:
97bc7e
+		fail(EC_STATUS)
97bc7e
+	except Exception as e:
97bc7e
+		logging.error("Failed to get power status: %s", e)
97bc7e
+		fail(EC_STATUS)
97bc7e
+
97bc7e
+def get_self_power_status(conn, instance_id):
97bc7e
+	try:
97bc7e
+		instance = conn.instances.filter(Filters=[{"Name": "instance-id", "Values": [instance_id]}])
97bc7e
+		state = list(instance)[0].state["Name"]
97bc7e
+		if state == "running":
97bc7e
+			logging.debug("Captured my (%s) state and it %s - returning OK - Proceeding with fencing",instance_id,state.upper())
97bc7e
+			return "ok"
97bc7e
+		else:
97bc7e
+			logging.debug("Captured my (%s) state it is %s - returning Alert - Unable to fence other nodes",instance_id,state.upper())
97bc7e
+			return "alert"
97bc7e
+	
97bc7e
+	except ClientError:
97bc7e
+		fail_usage("Failed: Incorrect Access Key or Secret Key.")
97bc7e
+	except EndpointConnectionError:
97bc7e
+		fail_usage("Failed: Incorrect Region.")
97bc7e
+	except IndexError:
97bc7e
 		return "fail"
97bc7e
 
97bc7e
 def set_power_status(conn, options):
97bc7e
-	if (options["--action"]=="off"):
97bc7e
-		conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
97bc7e
-	elif (options["--action"]=="on"):
97bc7e
-		conn.instances.filter(InstanceIds=[options["--plug"]]).start()
97bc7e
-
97bc7e
+	my_instance = get_instance_id()
97bc7e
+	try:
97bc7e
+		if (options["--action"]=="off"):
97bc7e
+			if (get_self_power_status(conn,my_instance) == "ok"):
97bc7e
+				conn.instances.filter(InstanceIds=[options["--plug"]]).stop(Force=True)
97bc7e
+				logger.info("Called StopInstance API call for %s", options["--plug"])
97bc7e
+			else:
97bc7e
+				logger.info("Skipping fencing as instance is not in running status")
97bc7e
+		elif (options["--action"]=="on"):
97bc7e
+			conn.instances.filter(InstanceIds=[options["--plug"]]).start()
97bc7e
+	except Exception as e:
97bc7e
+		logger.error("Failed to power %s %s: %s", \
97bc7e
+				options["--action"], options["--plug"], e)
97bc7e
 
97bc7e
 def define_new_opts():
97bc7e
 	all_opt["region"] = {
97bc7e
 		"getopt" : "r:",
97bc7e
 		"longopt" : "region",
97bc7e
-		"help" : "-r, --region=[name]            Region, e.g. us-east-1",
97bc7e
+		"help" : "-r, --region=[region]           Region, e.g. us-east-1",
97bc7e
 		"shortdesc" : "Region.",
97bc7e
 		"required" : "0",
97bc7e
 		"order" : 2
97bc7e
@@ -59,7 +111,7 @@
97bc7e
 	all_opt["access_key"] = {
97bc7e
 		"getopt" : "a:",
97bc7e
 		"longopt" : "access-key",
97bc7e
-		"help" : "-a, --access-key=[name]         Access Key",
97bc7e
+		"help" : "-a, --access-key=[key]         Access Key",
97bc7e
 		"shortdesc" : "Access Key.",
97bc7e
 		"required" : "0",
97bc7e
 		"order" : 3
97bc7e
@@ -67,23 +119,32 @@
97bc7e
 	all_opt["secret_key"] = {
97bc7e
 		"getopt" : "s:",
97bc7e
 		"longopt" : "secret-key",
97bc7e
-		"help" : "-s, --secret-key=[name]         Secret Key",
97bc7e
+		"help" : "-s, --secret-key=[key]         Secret Key",
97bc7e
 		"shortdesc" : "Secret Key.",
97bc7e
 		"required" : "0",
97bc7e
 		"order" : 4
97bc7e
 	}
97bc7e
+	all_opt["boto3_debug"] = {
97bc7e
+		"getopt" : "b:",
97bc7e
+		"longopt" : "boto3_debug",
97bc7e
+		"help" : "-b, --boto3_debug=[option]      Boto3 and Botocore library debug logging",
97bc7e
+		"shortdesc": "Boto Lib debug",
97bc7e
+		"required": "0",
97bc7e
+		"order": 5
97bc7e
+	}
97bc7e
 
97bc7e
 # Main agent method
97bc7e
 def main():
97bc7e
 	conn = None
97bc7e
 
97bc7e
-	device_opt = ["port", "no_password", "region", "access_key", "secret_key"]
97bc7e
+	device_opt = ["port", "no_password", "region", "access_key", "secret_key", "boto3_debug"]
97bc7e
 
97bc7e
 	atexit.register(atexit_handler)
97bc7e
 
97bc7e
 	define_new_opts()
97bc7e
 
97bc7e
 	all_opt["power_timeout"]["default"] = "60"
97bc7e
+	all_opt["boto3_debug"]["default"] = "off"
97bc7e
 
97bc7e
 	options = check_input(device_opt, process_input(device_opt))
97bc7e
 
97bc7e
@@ -99,6 +160,28 @@
97bc7e
 
97bc7e
 	run_delay(options)
97bc7e
 
97bc7e
+	if options.get("--verbose") is not None:
97bc7e
+		lh = logging.FileHandler('/var/log/fence_aws_debug.log')
97bc7e
+		logger.addHandler(lh)
97bc7e
+		lhf = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
97bc7e
+		lh.setFormatter(lhf)
97bc7e
+		logger.setLevel(logging.DEBUG)
97bc7e
+	
97bc7e
+	if options["--boto3_debug"] != "on":
97bc7e
+		boto3.set_stream_logger('boto3',logging.INFO)
97bc7e
+		boto3.set_stream_logger('botocore',logging.INFO)
97bc7e
+		logging.getLogger('botocore').propagate = False
97bc7e
+		logging.getLogger('boto3').propagate = False
97bc7e
+	else:
97bc7e
+		log_format = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
97bc7e
+		logging.getLogger('botocore').propagate = False
97bc7e
+		logging.getLogger('boto3').propagate = False
97bc7e
+		fdh = logging.FileHandler('/var/log/fence_aws_boto3.log')
97bc7e
+		fdh.setFormatter(log_format)
97bc7e
+		logging.getLogger('boto3').addHandler(fdh)
97bc7e
+		logging.getLogger('botocore').addHandler(fdh)
97bc7e
+		logging.debug("Boto debug level is %s and sending debug info to /var/log/fence_aws_boto3.log", options["--boto3_debug"])
97bc7e
+
97bc7e
 	region = options.get("--region")
97bc7e
 	access_key = options.get("--access-key")
97bc7e
 	secret_key = options.get("--secret-key")
97bc7e
@@ -106,12 +189,12 @@
97bc7e
 		conn = boto3.resource('ec2', region_name=region,
97bc7e
 				      aws_access_key_id=access_key,
97bc7e
 				      aws_secret_access_key=secret_key)
97bc7e
-	except:
97bc7e
-		fail_usage("Failed: Unable to connect to AWS. Check your configuration.")
97bc7e
+	except Exception as e:
97bc7e
+		fail_usage("Failed: Unable to connect to AWS: " + str(e))
97bc7e
 
97bc7e
 	# Operate the fencing device
97bc7e
 	result = fence_action(conn, options, set_power_status, get_power_status, get_nodes_list)
97bc7e
 	sys.exit(result)
97bc7e
 
97bc7e
 if __name__ == "__main__":
97bc7e
-	main()
97bc7e
+	main()
97bc7e
\ No newline at end of file