diff options
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r-- | block/blk-mq.c | 70 |
1 files changed, 59 insertions, 11 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c index 883f7208901..b1bcc619d0e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -320,7 +320,7 @@ void __blk_mq_complete_request(struct request *rq) rq->csd.func = __blk_mq_complete_request_remote; rq->csd.info = rq; rq->csd.flags = 0; - __smp_call_function_single(ctx->cpu, &rq->csd, 0); + smp_call_function_single_async(ctx->cpu, &rq->csd); } else { rq->q->softirq_done_fn(rq); } @@ -514,7 +514,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) LIST_HEAD(rq_list); int bit, queued; - if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->flags))) + if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state))) return; hctx->run++; @@ -603,7 +603,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) { - if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->flags))) + if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state))) return; if (!async) @@ -623,7 +623,7 @@ void blk_mq_run_queues(struct request_queue *q, bool async) queue_for_each_hw_ctx(q, hctx, i) { if ((!blk_mq_hctx_has_pending(hctx) && list_empty_careful(&hctx->dispatch)) || - test_bit(BLK_MQ_S_STOPPED, &hctx->flags)) + test_bit(BLK_MQ_S_STOPPED, &hctx->state)) continue; blk_mq_run_hw_queue(hctx, async); @@ -994,8 +994,46 @@ static void blk_mq_hctx_notify(void *data, unsigned long action, blk_mq_put_ctx(ctx); } -static void blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx, - void (*init)(void *, struct blk_mq_hw_ctx *, +static int blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx, + int (*init)(void *, struct blk_mq_hw_ctx *, + struct request *, unsigned int), + void *data) +{ + unsigned int i; + int ret = 0; + + for (i = 0; i < hctx->queue_depth; i++) { + struct request *rq = hctx->rqs[i]; + + ret = init(data, hctx, rq, i); + if (ret) + break; + } + + return ret; +} + +int blk_mq_init_commands(struct request_queue *q, + int (*init)(void *, struct blk_mq_hw_ctx *, + struct request *, unsigned int), + void *data) +{ + struct blk_mq_hw_ctx *hctx; + unsigned int i; + int ret = 0; + + queue_for_each_hw_ctx(q, hctx, i) { + ret = blk_mq_init_hw_commands(hctx, init, data); + if (ret) + break; + } + + return ret; +} +EXPORT_SYMBOL(blk_mq_init_commands); + +static void blk_mq_free_hw_commands(struct blk_mq_hw_ctx *hctx, + void (*free)(void *, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data) { @@ -1004,12 +1042,12 @@ static void blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx, for (i = 0; i < hctx->queue_depth; i++) { struct request *rq = hctx->rqs[i]; - init(data, hctx, rq, i); + free(data, hctx, rq, i); } } -void blk_mq_init_commands(struct request_queue *q, - void (*init)(void *, struct blk_mq_hw_ctx *, +void blk_mq_free_commands(struct request_queue *q, + void (*free)(void *, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data) { @@ -1017,9 +1055,9 @@ void blk_mq_init_commands(struct request_queue *q, unsigned int i; queue_for_each_hw_ctx(q, hctx, i) - blk_mq_init_hw_commands(hctx, init, data); + blk_mq_free_hw_commands(hctx, free, data); } -EXPORT_SYMBOL(blk_mq_init_commands); +EXPORT_SYMBOL(blk_mq_free_commands); static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx) { @@ -1430,6 +1468,16 @@ static int blk_mq_queue_reinit_notify(struct notifier_block *nb, return NOTIFY_OK; } +void blk_mq_disable_hotplug(void) +{ + mutex_lock(&all_q_mutex); +} + +void blk_mq_enable_hotplug(void) +{ + mutex_unlock(&all_q_mutex); +} + static int __init blk_mq_init(void) { blk_mq_cpu_init(); |