nvme: add lock_depth for ctrlr_lock

This supplements checking pthread_mutex_destroy() return code for
detecting if the ctrlr lock is destroy while it is held. At least
on some systems, pthread_mutex_destroy() returns success even if
the lock is currently held.

Note: this is part of patch series that reproduced issue spdk#3401,
and this patch reliably catches that the lock is held when
destroyed after a controller loss timeout from the bdev nvme module.

Signed-off-by: Jim Harris <[email protected]>
Change-Id: Id2a4e3dd9f5d52b3038143ff9c0dceb779b55b56
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23735
Community-CI: Mellanox Build Bot
Reviewed-by: Konrad Sztyber <[email protected]>
Reviewed-by: Shuhei Matsumoto <[email protected]>
Tested-by: SPDK CI Jenkins <[email protected]>

lib/nvme/nvme_ctrlr.c

@@ -4234,6 +4234,11 @@ nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr)
 {
 	int rc;
 
+	if (ctrlr->lock_depth > 0) {
+		SPDK_ERRLOG("lock currently held (depth=%d)!\n", ctrlr->lock_depth);
+		assert(false);
+	}
+
 	rc = pthread_mutex_destroy(&ctrlr->ctrlr_lock);
 	if (rc) {
 		SPDK_ERRLOG("could not destroy ctrlr_lock: %s\n", spdk_strerror(rc));

lib/nvme/nvme_internal.h

@@ -1061,7 +1061,7 @@ struct spdk_nvme_ctrlr {
 
 	nvme_request_stailq_t		queued_aborts;
 	uint32_t			outstanding_aborts;
-
+	uint32_t			lock_depth;
 	/* CB to notify the user when the ctrlr is constructed. */
 	spdk_nvme_construct_cb			construct_cb;
 	bool					lazy_fabric_connect;
@@ -1215,7 +1215,11 @@ nvme_robust_mutex_lock(pthread_mutex_t *mtx)
 static inline int
 nvme_ctrlr_lock(struct spdk_nvme_ctrlr *ctrlr)
 {
-	return nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	int rc;
+
+	rc = nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);
+	ctrlr->lock_depth++;
+	return rc;
 }
 
 static inline int
@@ -1227,6 +1231,7 @@ nvme_robust_mutex_unlock(pthread_mutex_t *mtx)
 static inline int
 nvme_ctrlr_unlock(struct spdk_nvme_ctrlr *ctrlr)
 {
+	ctrlr->lock_depth--;
 	return nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);
 }