blk-mq: move the cache friendly bitmap type of out blk-mq-tag

We will use it for the pending list in blk-mq core as well.

Signed-off-by: Jens Axboe <axboe@fb.com>

3 files changed, 17 insertions, 11 deletions

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 8d526a3e02f..03ce6a11ba7 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -21,7 +21,7 @@ static bool bt_has_free_tags(struct blk_mq_bitmap_tags *bt)
 	int i;
 
 	for (i = 0; i < bt->map_nr; i++) {
-		struct blk_mq_bitmap *bm = &bt->map[i];
+		struct blk_align_bitmap *bm = &bt->map[i];
 		int ret;
 
 		ret = find_first_zero_bit(&bm->word, bm->depth);
@@ -40,7 +40,7 @@ bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
 	return bt_has_free_tags(&tags->bitmap_tags);
 }
 
-static int __bt_get_word(struct blk_mq_bitmap *bm, unsigned int last_tag)
+static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
 {
 	int tag, org_last_tag, end;
 
@@ -283,7 +283,7 @@ static void bt_for_each_free(struct blk_mq_bitmap_tags *bt,
 	int i;
 
 	for (i = 0; i < bt->map_nr; i++) {
-		struct blk_mq_bitmap *bm = &bt->map[i];
+		struct blk_align_bitmap *bm = &bt->map[i];
 		int bit = 0;
 
 		do {
@@ -323,7 +323,7 @@ static unsigned int bt_unused_tags(struct blk_mq_bitmap_tags *bt)
 	unsigned int i, used;
 
 	for (i = 0, used = 0; i < bt->map_nr; i++) {
-		struct blk_mq_bitmap *bm = &bt->map[i];
+		struct blk_align_bitmap *bm = &bt->map[i];
 
 		used += bitmap_weight(&bm->word, bm->depth);
 	}
@@ -361,7 +361,7 @@ static int bt_alloc(struct blk_mq_bitmap_tags *bt, unsigned int depth,
 		}
 
 		nr = ALIGN(depth, tags_per_word) / tags_per_word;
-		bt->map = kzalloc_node(nr * sizeof(struct blk_mq_bitmap),
+		bt->map = kzalloc_node(nr * sizeof(struct blk_align_bitmap),
 						GFP_KERNEL, node);
 		if (!bt->map)
 			return -ENOMEM;
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 7aa9f066548..9014269f391 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -1,6 +1,8 @@
 #ifndef INT_BLK_MQ_TAG_H
 #define INT_BLK_MQ_TAG_H
 
+#include "blk-mq.h"
+
 enum {
 	BT_WAIT_QUEUES	= 8,
 	BT_WAIT_BATCH	= 8,
@@ -14,18 +16,13 @@ struct bt_wait_state {
 #define TAG_TO_INDEX(bt, tag)	((tag) >> (bt)->bits_per_word)
 #define TAG_TO_BIT(bt, tag)	((tag) & ((1 << (bt)->bits_per_word) - 1))
 
-struct blk_mq_bitmap {
-	unsigned long word;
-	unsigned long depth;
-} ____cacheline_aligned_in_smp;
-
 struct blk_mq_bitmap_tags {
 	unsigned int depth;
 	unsigned int wake_cnt;
 	unsigned int bits_per_word;
 
 	unsigned int map_nr;
-	struct blk_mq_bitmap *map;
+	struct blk_align_bitmap *map;
 
 	unsigned int wake_index;
 	struct bt_wait_state *bs;
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 97cfab9c092..5e5a378962b 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -52,4 +52,13 @@ void blk_mq_disable_hotplug(void);
 extern unsigned int *blk_mq_make_queue_map(struct blk_mq_tag_set *set);
 extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues);
 
+/*
+ * Basic implementation of sparser bitmap, allowing the user to spread
+ * the bits over more cachelines.
+ */
+struct blk_align_bitmap {
+	unsigned long word;
+	unsigned long depth;
+} ____cacheline_aligned_in_smp;
+
 #endif