bdev/rbd: integrate SpdkContextWQ for reactor thread execution

mk/spdk.common.mk

@@ -321,7 +321,7 @@ LDFLAGS += -fsanitize=fuzzer-no-link
 SYS_LIBS += $(CONFIG_FUZZER_LIB)
 endif
 
-SPDK_GIT_COMMIT := 0efce164f90d98de445e756559933f981650dc77
+SPDK_GIT_COMMIT := dce40dd4f63bfa77ebf8397768cd9131fee56505
 ifneq (, $(SPDK_GIT_COMMIT))
 COMMON_CFLAGS += -DSPDK_GIT_COMMIT=$(SPDK_GIT_COMMIT)
 endif

mk/spdk.modules.mk

@@ -74,7 +74,7 @@ endif
 
 ifeq ($(CONFIG_RBD),y)
 BLOCKDEV_MODULES_LIST += bdev_rbd
-BLOCKDEV_MODULES_PRIVATE_LIBS += -lrados -lrbd
+BLOCKDEV_MODULES_PRIVATE_LIBS += -lrados -lrbd -lstdc++
 endif
 
 ifeq ($(CONFIG_DAOS),y)

module/bdev/rbd/Makefile

@@ -10,6 +10,7 @@ SO_VER := 8
 SO_MINOR := 0
 
 C_SRCS = bdev_rbd.c bdev_rbd_rpc.c
+CXX_SRCS = bdev_rbd_spdk_context_wq.cpp
 LIBNAME = bdev_rbd
 
 SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map

module/bdev/rbd/bdev_rbd.c

@@ -6,6 +6,7 @@
 #include "spdk/stdinc.h"
 
 #include "bdev_rbd.h"
+#include "bdev_rbd_spdk_context_wq.h"
 
 #include <rbd/librbd.h>
 #include <rados/librados.h>
@@ -28,6 +29,7 @@ static int bdev_rbd_count = 0;
  * global parameter to control CRC32C usage in RBD write operations.
  */
 static bool g_rbd_with_crc32c = false;
+static bool g_rbd_with_spdk_wq = false;
 
 struct bdev_rbd_pool_ctx {
 	rados_t *cluster_p;
@@ -75,6 +77,9 @@ struct bdev_rbd {
 	int (*reservation_fn_cbk)(void *ns);
 	char cluster_fsid[37];
 
+	/* SPDK ContextWQ for this bdev */
+	struct bdev_rbd_spdk_context_wq *spdk_context_wq;
+
 };
 
 struct bdev_rbd_io_channel {
@@ -224,6 +229,16 @@ bdev_rbd_free(struct bdev_rbd *rbd)
 		rbd_close(rbd->image);
 	}
 
+	/* Clean up SPDK ContextWQ after RBD image is closed.
+	 * This ensures no new I/O completions can occur after ContextWQ is destroyed.
+	 * The drain() function in the destructor will wait for any pending messages
+	 * to complete before the ContextWQ is actually destroyed.
+	 */
+	if (rbd->spdk_context_wq != NULL) {
+		bdev_rbd_spdk_context_wq_destroy(rbd->spdk_context_wq);
+		rbd->spdk_context_wq = NULL;
+	}
+
 	free(rbd->disk.name);
 	free(rbd->rbd_name);
 	free(rbd->user_id);
@@ -488,11 +503,21 @@ bdev_rbd_init_context(void *arg)
 	}
 
 	assert(io_ctx != NULL);
+	if (g_rbd_with_spdk_wq) {
+		/* Find reactor thread, create SpdkContextWQ if available, then open with context_wq (NULL uses default AsioContextWQ) */
+		struct spdk_thread *reactor_thread = bdev_rbd_find_reactor_thread();
+		rbd->spdk_context_wq = bdev_rbd_spdk_context_wq_create_from_ioctx(
+			*io_ctx, reactor_thread);
+	} else {
+		rbd->spdk_context_wq = NULL;
+		SPDK_NOTICELOG("rbd_with_spdk_wq is disabled, using AsioContextWQ for RBD image %s/%s\n",
+			       rbd->pool_name, rbd->rbd_name);
+	}
 	if (rbd->rbd_read_only) {
 		SPDK_DEBUGLOG(bdev_rbd, "Will open RBD image %s/%s as read-only\n", rbd->pool_name, rbd->rbd_name);
-		rc = rbd_open_read_only(*io_ctx, rbd->rbd_name, &rbd->image, NULL);
+		rc = rbd_open_read_only_with_context_wq(*io_ctx, rbd->rbd_name, &rbd->image, NULL, rbd->spdk_context_wq);
 	} else {
-		rc = rbd_open(*io_ctx, rbd->rbd_name, &rbd->image, NULL);
+		rc = rbd_open_with_context_wq(*io_ctx, rbd->rbd_name, &rbd->image, NULL, rbd->spdk_context_wq);
 	}
 	if (rc < 0) {
 		SPDK_ERRLOG("Failed to open specified rbd device\n");
@@ -1948,3 +1973,16 @@ bdev_rbd_set_with_crc32c(bool enable)
 {
 	g_rbd_with_crc32c = enable;
 }
+
+bool
+bdev_rbd_get_with_spdk_wq(void)
+{
+	return g_rbd_with_spdk_wq;
+}
+
+/** enable or disable SPDK ContextWQ for RBD operations */
+void
+bdev_rbd_set_with_spdk_wq(bool enable)
+{
+	g_rbd_with_spdk_wq = enable;
+}

module/bdev/rbd/bdev_rbd.h

@@ -96,4 +96,19 @@ bool bdev_rbd_get_with_crc32c(void);
  */
 void bdev_rbd_set_with_crc32c(bool enable);
 
+/**
+ * Get the current rbd_with_spdk_wq setting.
+ *
+ * \return true if SPDK ContextWQ is enabled, false otherwise
+ */
+bool bdev_rbd_get_with_spdk_wq(void);
+
+/**
+ * Set the rbd_with_spdk_wq parameter to enable/disable SPDK ContextWQ
+ * for RBD operations.
+ *
+ * \param enable true to enable SPDK ContextWQ, false to disable (uses AsioContextWQ)
+ */
+void bdev_rbd_set_with_spdk_wq(bool enable);
+
 #endif /* SPDK_BDEV_RBD_H */

module/bdev/rbd/bdev_rbd_rpc.c

@@ -439,3 +439,53 @@ SPDK_RPC_REGISTER("bdev_rbd_wait_for_latest_osdmap", rpc_bdev_rbd_wait_for_lates
 
 SPDK_RPC_REGISTER("bdev_rbd_get_with_crc32c", rpc_bdev_rbd_get_with_crc32c, SPDK_RPC_RUNTIME)
 SPDK_RPC_REGISTER("bdev_rbd_set_with_crc32c", rpc_bdev_rbd_set_with_crc32c, SPDK_RPC_STARTUP)
+
+/**
+ * RPC function to get the current rbd_with_spdk_wq setting
+ */
+static void
+rpc_bdev_rbd_get_with_spdk_wq(struct spdk_jsonrpc_request *request,
+			      const struct spdk_json_val *params)
+{
+	struct spdk_json_write_ctx *w;
+
+	w = spdk_jsonrpc_begin_result(request);
+	spdk_json_write_bool(w, bdev_rbd_get_with_spdk_wq());
+	spdk_jsonrpc_end_result(request, w);
+}
+
+/**
+ * RPC function to set the rbd_with_spdk_wq parameter
+ */
+struct rpc_bdev_rbd_set_with_spdk_wq {
+	bool enable;
+};
+
+static const struct spdk_json_object_decoder rpc_bdev_rbd_set_with_spdk_wq_decoders[] = {
+	{"enable", offsetof(struct rpc_bdev_rbd_set_with_spdk_wq, enable), spdk_json_decode_bool},
+};
+
+static void
+rpc_bdev_rbd_set_with_spdk_wq(struct spdk_jsonrpc_request *request,
+			     const struct spdk_json_val *params)
+{
+	struct rpc_bdev_rbd_set_with_spdk_wq req = {};
+	struct spdk_json_write_ctx *w;
+
+	if (spdk_json_decode_object(params, rpc_bdev_rbd_set_with_spdk_wq_decoders,
+				    SPDK_COUNTOF(rpc_bdev_rbd_set_with_spdk_wq_decoders),
+				    &req)) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						"Missing or invalid enable parameter");
+		return;
+	}
+
+	bdev_rbd_set_with_spdk_wq(req.enable);
+
+	w = spdk_jsonrpc_begin_result(request);
+	spdk_json_write_bool(w, bdev_rbd_get_with_spdk_wq());
+	spdk_jsonrpc_end_result(request, w);
+}
+
+SPDK_RPC_REGISTER("bdev_rbd_get_with_spdk_wq", rpc_bdev_rbd_get_with_spdk_wq, SPDK_RPC_RUNTIME)
+SPDK_RPC_REGISTER("bdev_rbd_set_with_spdk_wq", rpc_bdev_rbd_set_with_spdk_wq, SPDK_RPC_STARTUP)