From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Benjamin Marzinski <bmarzins@redhat.com>
Date: Fri, 7 Oct 2022 12:35:40 -0500
Subject: [PATCH] libmultipath: fix queue_mode feature handling

device-mapper is not able to change the queue_mode on a table reload.
Make sure that when multipath sets up the map, both on regular reloads
and reconfigures, it keeps the queue_mode the same.

Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
Reviewed-by: Martin Wilck <mwilck@suse.com>
---
 libmultipath/configure.c   |  4 +++
 libmultipath/dmparser.c    |  2 ++
 libmultipath/propsel.c     | 55 ++++++++++++++++++++++++++++++++++++++
 libmultipath/structs.h     |  7 +++++
 libmultipath/util.c        | 10 +++++++
 libmultipath/util.h        |  1 +
 multipath/multipath.conf.5 |  7 +++--
 7 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/libmultipath/configure.c b/libmultipath/configure.c
index 6cad0468..287289f7 100644
--- a/libmultipath/configure.c
+++ b/libmultipath/configure.c
@@ -1102,6 +1102,7 @@ int coalesce_paths (struct vectors * vecs, vector newmp, char * refwwid,
 	uint64_t *size_mismatch_seen;
 	bool map_processed = false;
 	bool no_daemon = false;
+	struct multipath * cmpp;
 
 	/* ignore refwwid if it's empty */
 	if (refwwid && !strlen(refwwid))
@@ -1197,6 +1198,9 @@ int coalesce_paths (struct vectors * vecs, vector newmp, char * refwwid,
 		}
 		verify_paths(mpp, vecs);
 
+		cmpp = find_mp_by_wwid(curmp, mpp->wwid);
+		if (cmpp)
+			mpp->queue_mode = cmpp->queue_mode;
 		params[0] = '\0';
 		if (setup_map(mpp, params, PARAMS_SIZE, vecs)) {
 			remove_map(mpp, vecs, 0);
diff --git a/libmultipath/dmparser.c b/libmultipath/dmparser.c
index b856a07f..b9c4dabc 100644
--- a/libmultipath/dmparser.c
+++ b/libmultipath/dmparser.c
@@ -164,6 +164,8 @@ int disassemble_map(vector pathvec, char *params, struct multipath *mpp,
 
 		FREE(word);
 	}
+	mpp->queue_mode = strstr(mpp->features, "queue_mode bio") ?
+			  QUEUE_MODE_BIO : QUEUE_MODE_RQ;
 
 	/*
 	 * hwhandler
diff --git a/libmultipath/propsel.c b/libmultipath/propsel.c
index be79902f..3f119dd9 100644
--- a/libmultipath/propsel.c
+++ b/libmultipath/propsel.c
@@ -26,6 +26,7 @@
 #include "propsel.h"
 #include <inttypes.h>
 #include <libudev.h>
+#include <ctype.h>
 
 pgpolicyfn *pgpolicies[] = {
 	NULL,
@@ -413,6 +414,59 @@ void reconcile_features_with_options(const char *id, char **features, int* no_pa
 	}
 }
 
+static void reconcile_features_with_queue_mode(struct multipath *mp)
+{
+	char *space = NULL, *val = NULL, *mode_str = NULL, *feat;
+	int features_mode = QUEUE_MODE_UNDEF;
+
+	if (!mp->features)
+		return;
+
+	pthread_cleanup_push(cleanup_free_ptr, &space);
+	pthread_cleanup_push(cleanup_free_ptr, &val);
+	pthread_cleanup_push(cleanup_free_ptr, &mode_str);
+
+	if (!(feat = strstr(mp->features, "queue_mode")) ||
+	    feat == mp->features || !isspace(*(feat - 1)) ||
+	    sscanf(feat, "queue_mode%m[ \f\n\r\t\v]%ms", &space, &val) != 2)
+		goto sync_mode;
+	if (asprintf(&mode_str, "queue_mode%s%s", space, val) < 0) {
+		condlog(1, "failed to allocate space for queue_mode feature string");
+		mode_str = NULL; /* value undefined on failure */
+		goto exit;
+	}
+
+	if (!strcmp(val, "rq") || !strcmp(val, "mq"))
+		features_mode = QUEUE_MODE_RQ;
+	else if (!strcmp(val, "bio"))
+		features_mode = QUEUE_MODE_BIO;
+	if (features_mode == QUEUE_MODE_UNDEF) {
+		condlog(2, "%s: ignoring invalid feature '%s'",
+			mp->alias, mode_str);
+		goto sync_mode;
+	}
+
+	if (mp->queue_mode == QUEUE_MODE_UNDEF)
+		mp->queue_mode = features_mode;
+	if (mp->queue_mode == features_mode)
+		goto exit;
+
+	condlog(2,
+		"%s: ignoring feature '%s' because queue_mode is set to '%s'",
+		mp->alias, mode_str,
+		(mp->queue_mode == QUEUE_MODE_RQ)? "rq" : "bio");
+
+sync_mode:
+	if (mode_str)
+		remove_feature(&mp->features, mode_str);
+	if (mp->queue_mode == QUEUE_MODE_BIO)
+		add_feature(&mp->features, "queue_mode bio");
+exit:
+	pthread_cleanup_pop(1);
+	pthread_cleanup_pop(1);
+	pthread_cleanup_pop(1);
+}
+
 int select_features(struct config *conf, struct multipath *mp)
 {
 	const char *origin;
@@ -428,6 +482,7 @@ out:
 	reconcile_features_with_options(mp->alias, &mp->features,
 					&mp->no_path_retry,
 					&mp->retain_hwhandler);
+	reconcile_features_with_queue_mode(mp);
 	condlog(3, "%s: features = \"%s\" %s", mp->alias, mp->features, origin);
 	return 0;
 }
diff --git a/libmultipath/structs.h b/libmultipath/structs.h
index 3ed5cfc1..9a404da7 100644
--- a/libmultipath/structs.h
+++ b/libmultipath/structs.h
@@ -187,6 +187,12 @@ enum max_sectors_kb_states {
 	MAX_SECTORS_KB_MIN = 4,  /* can't be smaller than page size */
 };
 
+enum queue_mode_states {
+	QUEUE_MODE_UNDEF = 0,
+	QUEUE_MODE_BIO,
+	QUEUE_MODE_RQ,
+};
+
 enum scsi_protocol {
 	SCSI_PROTOCOL_FCP = 0,	/* Fibre Channel */
 	SCSI_PROTOCOL_SPI = 1,	/* parallel SCSI */
@@ -397,6 +403,7 @@ struct multipath {
 	int needs_paths_uevent;
 	int ghost_delay;
 	int ghost_delay_tick;
+	int queue_mode;
 	uid_t uid;
 	gid_t gid;
 	mode_t mode;
diff --git a/libmultipath/util.c b/libmultipath/util.c
index dd30a46e..e04d20ab 100644
--- a/libmultipath/util.c
+++ b/libmultipath/util.c
@@ -465,6 +465,16 @@ void free_scandir_result(struct scandir_result *res)
 	FREE(res->di);
 }
 
+void cleanup_free_ptr(void *arg)
+{
+	void **p = arg;
+
+	if (p && *p) {
+		free(*p);
+		*p = NULL;
+	}
+}
+
 void close_fd(void *arg)
 {
 	close((long)arg);
diff --git a/libmultipath/util.h b/libmultipath/util.h
index ce277680..f898c829 100644
--- a/libmultipath/util.h
+++ b/libmultipath/util.h
@@ -44,6 +44,7 @@ void set_max_fds(rlim_t max_fds);
 	pthread_cleanup_push(((void (*)(void *))&f), (arg))
 
 void close_fd(void *arg);
+void cleanup_free_ptr(void *arg);
 void cleanup_mutex(void *arg);
 
 struct scandir_result {
diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5
index 8e418372..61d2712b 100644
--- a/multipath/multipath.conf.5
+++ b/multipath/multipath.conf.5
@@ -458,8 +458,11 @@ precedence. See KNOWN ISSUES.
 <mode> can be \fIbio\fR, \fIrq\fR or \fImq\fR, which corresponds to
 bio-based, request-based, and block-multiqueue (blk-mq) request-based,
 respectively.
-The default depends on the kernel parameter \fBdm_mod.use_blk_mq\fR. It is
-\fImq\fR if the latter is set, and \fIrq\fR otherwise.
+Before kernel 4.20 The default depends on the kernel parameter
+\fBdm_mod.use_blk_mq\fR. It is \fImq\fR if the latter is set, and \fIrq\fR
+otherwise. Since kernel 4.20, \fIrq\fR and \fImq\fR both correspond to
+block-multiqueue. Once a multipath device has been created, its queue_mode
+cannot be changed.
 .TP
 The default is: \fB<unset>\fR
 .RE