diff options
| -rw-r--r-- | block/blk-core.c | 57 | ||||
| -rw-r--r-- | block/blk-sysfs.c | 1 | ||||
| -rw-r--r-- | block/blk-throttle.c | 50 | ||||
| -rw-r--r-- | block/blk.h | 6 | ||||
| -rw-r--r-- | block/elevator.c | 2 |
5 files changed, 87 insertions, 29 deletions
diff --git a/block/blk-core.c b/block/blk-core.c index 034cbb2024f..7e1523521c7 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -349,11 +349,13 @@ EXPORT_SYMBOL(blk_put_queue); /** * blk_drain_queue - drain requests from request_queue * @q: queue to drain + * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV * - * Drain ELV_PRIV requests from @q. The caller is responsible for ensuring - * that no new requests which need to be drained are queued. + * Drain requests from @q. If @drain_all is set, all requests are drained. + * If not, only ELVPRIV requests are drained. The caller is responsible + * for ensuring that no new requests which need to be drained are queued. */ -void blk_drain_queue(struct request_queue *q) +void blk_drain_queue(struct request_queue *q, bool drain_all) { while (true) { int nr_rqs; @@ -361,9 +363,15 @@ void blk_drain_queue(struct request_queue *q) spin_lock_irq(q->queue_lock); elv_drain_elevator(q); + if (drain_all) + blk_throtl_drain(q); __blk_run_queue(q); - nr_rqs = q->rq.elvpriv; + + if (drain_all) + nr_rqs = q->rq.count[0] + q->rq.count[1]; + else + nr_rqs = q->rq.elvpriv; spin_unlock_irq(q->queue_lock); @@ -373,30 +381,40 @@ void blk_drain_queue(struct request_queue *q) } } -/* - * Note: If a driver supplied the queue lock, it is disconnected - * by this function. The actual state of the lock doesn't matter - * here as the request_queue isn't accessible after this point - * (QUEUE_FLAG_DEAD is set) and no other requests will be queued. +/** + * blk_cleanup_queue - shutdown a request queue + * @q: request queue to shutdown + * + * Mark @q DEAD, drain all pending requests, destroy and put it. All + * future requests will be failed immediately with -ENODEV. */ void blk_cleanup_queue(struct request_queue *q) { - /* - * We know we have process context here, so we can be a little - * cautious and ensure that pending block actions on this device - * are done before moving on. Going into this function, we should - * not have processes doing IO to this device. - */ - blk_sync_queue(q); + spinlock_t *lock = q->queue_lock; - del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); + /* mark @q DEAD, no new request or merges will be allowed afterwards */ mutex_lock(&q->sysfs_lock); queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q); - mutex_unlock(&q->sysfs_lock); + + spin_lock_irq(lock); + queue_flag_set(QUEUE_FLAG_NOMERGES, q); + queue_flag_set(QUEUE_FLAG_NOXMERGES, q); + queue_flag_set(QUEUE_FLAG_DEAD, q); if (q->queue_lock != &q->__queue_lock) q->queue_lock = &q->__queue_lock; + spin_unlock_irq(lock); + mutex_unlock(&q->sysfs_lock); + + /* drain all requests queued before DEAD marking */ + blk_drain_queue(q, true); + + /* @q won't process any more request, flush async actions */ + del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); + blk_sync_queue(q); + + /* @q is and will stay empty, shutdown and put */ blk_put_queue(q); } EXPORT_SYMBOL(blk_cleanup_queue); @@ -1509,9 +1527,6 @@ generic_make_request_checks(struct bio *bio) goto end_io; } - if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) - goto end_io; - part = bio->bi_bdev->bd_part; if (should_fail_request(part, bio->bi_size) || should_fail_request(&part_to_disk(part)->part0, diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index a8eff5f8b9c..e7f9f657f10 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -490,6 +490,7 @@ static void blk_release_queue(struct kobject *kobj) if (q->queue_tags) __blk_queue_free_tags(q); + blk_throtl_release(q); blk_trace_shutdown(q); bdi_destroy(&q->backing_dev_info); diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 900a0c98745..8edb9499b50 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -309,6 +309,10 @@ static struct throtl_grp * throtl_get_tg(struct throtl_data *td) struct blkio_cgroup *blkcg; struct request_queue *q = td->queue; + /* no throttling for dead queue */ + if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) + return NULL; + rcu_read_lock(); blkcg = task_blkio_cgroup(current); tg = throtl_find_tg(td, blkcg); @@ -1001,11 +1005,6 @@ static void throtl_release_tgs(struct throtl_data *td) } } -static void throtl_td_free(struct throtl_data *td) -{ - kfree(td); -} - /* * Blk cgroup controller notification saying that blkio_group object is being * delinked as associated cgroup object is going away. That also means that @@ -1204,6 +1203,41 @@ out: return throttled; } +/** + * blk_throtl_drain - drain throttled bios + * @q: request_queue to drain throttled bios for + * + * Dispatch all currently throttled bios on @q through ->make_request_fn(). + */ +void blk_throtl_drain(struct request_queue *q) + __releases(q->queue_lock) __acquires(q->queue_lock) +{ + struct throtl_data *td = q->td; + struct throtl_rb_root *st = &td->tg_service_tree; + struct throtl_grp *tg; + struct bio_list bl; + struct bio *bio; + + lockdep_is_held(q->queue_lock); + + bio_list_init(&bl); + + while ((tg = throtl_rb_first(st))) { + throtl_dequeue_tg(td, tg); + + while ((bio = bio_list_peek(&tg->bio_lists[READ]))) + tg_dispatch_one_bio(td, tg, bio_data_dir(bio), &bl); + while ((bio = bio_list_peek(&tg->bio_lists[WRITE]))) + tg_dispatch_one_bio(td, tg, bio_data_dir(bio), &bl); + } + spin_unlock_irq(q->queue_lock); + + while ((bio = bio_list_pop(&bl))) + generic_make_request(bio); + + spin_lock_irq(q->queue_lock); +} + int blk_throtl_init(struct request_queue *q) { struct throtl_data *td; @@ -1276,7 +1310,11 @@ void blk_throtl_exit(struct request_queue *q) * it. */ throtl_shutdown_wq(q); - throtl_td_free(td); +} + +void blk_throtl_release(struct request_queue *q) +{ + kfree(q->td); } static int __init throtl_init(void) diff --git a/block/blk.h b/block/blk.h index c018dba4e33..3f6551b3c92 100644 --- a/block/blk.h +++ b/block/blk.h @@ -15,7 +15,7 @@ void blk_rq_bio_prep(struct request_queue *q, struct request *rq, struct bio *bio); int blk_rq_append_bio(struct request_queue *q, struct request *rq, struct bio *bio); -void blk_drain_queue(struct request_queue *q); +void blk_drain_queue(struct request_queue *q, bool drain_all); void blk_dequeue_request(struct request *rq); void __blk_queue_free_tags(struct request_queue *q); bool __blk_end_bidi_request(struct request *rq, int error, @@ -191,15 +191,19 @@ static inline int blk_do_io_stat(struct request *rq) #ifdef CONFIG_BLK_DEV_THROTTLING extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio); +extern void blk_throtl_drain(struct request_queue *q); extern int blk_throtl_init(struct request_queue *q); extern void blk_throtl_exit(struct request_queue *q); +extern void blk_throtl_release(struct request_queue *q); #else /* CONFIG_BLK_DEV_THROTTLING */ static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio) { return false; } +static inline void blk_throtl_drain(struct request_queue *q) { } static inline int blk_throtl_init(struct request_queue *q) { return 0; } static inline void blk_throtl_exit(struct request_queue *q) { } +static inline void blk_throtl_release(struct request_queue *q) { } #endif /* CONFIG_BLK_DEV_THROTTLING */ #endif /* BLK_INTERNAL_H */ diff --git a/block/elevator.c b/block/elevator.c index 74a277ffed3..66343d6917d 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -626,7 +626,7 @@ void elv_quiesce_start(struct request_queue *q) queue_flag_set(QUEUE_FLAG_ELVSWITCH, q); spin_unlock_irq(q->queue_lock); - blk_drain_queue(q); + blk_drain_queue(q, false); } void elv_quiesce_end(struct request_queue *q) |