From 8fa72d234da9b6b473bbb1f74d533663e4996e6b Mon Sep 17 00:00:00 2001 From: Jeff Moyer Date: Wed, 5 Dec 2012 20:17:21 +0100 Subject: bdi: add a user-tunable cpu_list for the bdi flusher threads In realtime environments, it may be desirable to keep the per-bdi flusher threads from running on certain cpus. This patch adds a cpu_list file to /sys/class/bdi/* to enable this. The default is to tie the flusher threads to the same numa node as the backing device (though I could be convinced to make it a mask of all cpus to avoid a change in behaviour). Thanks to Jeremy Eder for the original idea. Signed-off-by: Jeff Moyer Signed-off-by: Jens Axboe --- include/linux/backing-dev.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include') diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index 2a9a9abc912..238521a1984 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -18,6 +18,7 @@ #include #include #include +#include struct page; struct device; @@ -105,6 +106,9 @@ struct backing_dev_info { struct timer_list laptop_mode_wb_timer; + cpumask_t *flusher_cpumask; /* used for writeback thread scheduling */ + struct mutex flusher_cpumask_lock; + #ifdef CONFIG_DEBUG_FS struct dentry *debug_dir; struct dentry *debug_stats; -- cgit v1.2.3-70-g09d2 From 3f3299d5c0268d6cc3f47b446e8aca436e4a5651 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 28 Nov 2012 13:42:38 +0100 Subject: block: Rename queue dead flag QUEUE_FLAG_DEAD is used to indicate that queuing new requests must stop. After this flag has been set queue draining starts. However, during the queue draining phase it is still safe to invoke the queue's request_fn, so QUEUE_FLAG_DYING is a better name for this flag. This patch has been generated by running the following command over the kernel source tree: git grep -lEw 'blk_queue_dead|QUEUE_FLAG_DEAD' | xargs sed -i.tmp -e 's/blk_queue_dead/blk_queue_dying/g' \ -e 's/QUEUE_FLAG_DEAD/QUEUE_FLAG_DYING/g'; \ sed -i.tmp -e "s/QUEUE_FLAG_DYING$(printf \\t)*5/QUEUE_FLAG_DYING$(printf \\t)5/g" \ include/linux/blkdev.h; \ sed -i.tmp -e 's/ DEAD/ DYING/g' -e 's/dead queue/a dying queue/' \ -e 's/Dead queue/A dying queue/' block/blk-core.c Signed-off-by: Bart Van Assche Acked-by: Tejun Heo Cc: James Bottomley Cc: Mike Christie Cc: Jens Axboe Cc: Chanho Min Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 2 +- block/blk-core.c | 26 +++++++++++++------------- block/blk-exec.c | 2 +- block/blk-sysfs.c | 4 ++-- block/blk-throttle.c | 2 +- block/blk.h | 2 +- drivers/scsi/scsi_lib.c | 2 +- include/linux/blkdev.h | 4 ++-- 8 files changed, 22 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d0b770391ad..5dea4e8dbc5 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -231,7 +231,7 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, * we shouldn't allow anything to go through for a bypassing queue. */ if (unlikely(blk_queue_bypass(q))) - return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY); + return ERR_PTR(blk_queue_dying(q) ? -EINVAL : -EBUSY); return __blkg_lookup_create(blkcg, q, NULL); } EXPORT_SYMBOL_GPL(blkg_lookup_create); diff --git a/block/blk-core.c b/block/blk-core.c index ee0e5cafa85..1a95272cca5 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -473,20 +473,20 @@ EXPORT_SYMBOL_GPL(blk_queue_bypass_end); * blk_cleanup_queue - shutdown a request queue * @q: request queue to shutdown * - * Mark @q DEAD, drain all pending requests, destroy and put it. All + * Mark @q DYING, drain all pending requests, destroy and put it. All * future requests will be failed immediately with -ENODEV. */ void blk_cleanup_queue(struct request_queue *q) { spinlock_t *lock = q->queue_lock; - /* mark @q DEAD, no new request or merges will be allowed afterwards */ + /* mark @q DYING, no new request or merges will be allowed afterwards */ mutex_lock(&q->sysfs_lock); - queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q); + queue_flag_set_unlocked(QUEUE_FLAG_DYING, q); spin_lock_irq(lock); /* - * Dead queue is permanently in bypass mode till released. Note + * A dying queue is permanently in bypass mode till released. Note * that, unlike blk_queue_bypass_start(), we aren't performing * synchronize_rcu() after entering bypass mode to avoid the delay * as some drivers create and destroy a lot of queues while @@ -499,11 +499,11 @@ void blk_cleanup_queue(struct request_queue *q) queue_flag_set(QUEUE_FLAG_NOMERGES, q); queue_flag_set(QUEUE_FLAG_NOXMERGES, q); - queue_flag_set(QUEUE_FLAG_DEAD, q); + queue_flag_set(QUEUE_FLAG_DYING, q); spin_unlock_irq(lock); mutex_unlock(&q->sysfs_lock); - /* drain all requests queued before DEAD marking */ + /* drain all requests queued before DYING marking */ blk_drain_queue(q, true); /* @q won't process any more request, flush async actions */ @@ -716,7 +716,7 @@ EXPORT_SYMBOL(blk_init_allocated_queue); bool blk_get_queue(struct request_queue *q) { - if (likely(!blk_queue_dead(q))) { + if (likely(!blk_queue_dying(q))) { __blk_get_queue(q); return true; } @@ -870,7 +870,7 @@ static struct request *__get_request(struct request_list *rl, int rw_flags, const bool is_sync = rw_is_sync(rw_flags) != 0; int may_queue; - if (unlikely(blk_queue_dead(q))) + if (unlikely(blk_queue_dying(q))) return NULL; may_queue = elv_may_queue(q, rw_flags); @@ -1050,7 +1050,7 @@ retry: if (rq) return rq; - if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dead(q))) { + if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) { blk_put_rl(rl); return NULL; } @@ -1910,7 +1910,7 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq) return -EIO; spin_lock_irqsave(q->queue_lock, flags); - if (unlikely(blk_queue_dead(q))) { + if (unlikely(blk_queue_dying(q))) { spin_unlock_irqrestore(q->queue_lock, flags); return -ENODEV; } @@ -2885,9 +2885,9 @@ static void queue_unplugged(struct request_queue *q, unsigned int depth, trace_block_unplug(q, depth, !from_schedule); /* - * Don't mess with dead queue. + * Don't mess with a dying queue. */ - if (unlikely(blk_queue_dead(q))) { + if (unlikely(blk_queue_dying(q))) { spin_unlock(q->queue_lock); return; } @@ -2996,7 +2996,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) /* * Short-circuit if @q is dead */ - if (unlikely(blk_queue_dead(q))) { + if (unlikely(blk_queue_dying(q))) { __blk_end_request_all(rq, -ENODEV); continue; } diff --git a/block/blk-exec.c b/block/blk-exec.c index 8b6dc5bd4dd..4aec98df7ba 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -60,7 +60,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, spin_lock_irq(q->queue_lock); - if (unlikely(blk_queue_dead(q))) { + if (unlikely(blk_queue_dying(q))) { rq->errors = -ENXIO; if (rq->end_io) rq->end_io(rq, rq->errors); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index ce620460882..788147797a7 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -466,7 +466,7 @@ queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page) if (!entry->show) return -EIO; mutex_lock(&q->sysfs_lock); - if (blk_queue_dead(q)) { + if (blk_queue_dying(q)) { mutex_unlock(&q->sysfs_lock); return -ENOENT; } @@ -488,7 +488,7 @@ queue_attr_store(struct kobject *kobj, struct attribute *attr, q = container_of(kobj, struct request_queue, kobj); mutex_lock(&q->sysfs_lock); - if (blk_queue_dead(q)) { + if (blk_queue_dying(q)) { mutex_unlock(&q->sysfs_lock); return -ENOENT; } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index a9664fa0b60..31146225f3d 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -302,7 +302,7 @@ static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td, /* if %NULL and @q is alive, fall back to root_tg */ if (!IS_ERR(blkg)) tg = blkg_to_tg(blkg); - else if (!blk_queue_dead(q)) + else if (!blk_queue_dying(q)) tg = td_root_tg(td); } diff --git a/block/blk.h b/block/blk.h index ca51543b248..2218a8a7829 100644 --- a/block/blk.h +++ b/block/blk.h @@ -96,7 +96,7 @@ static inline struct request *__elv_next_request(struct request_queue *q) q->flush_queue_delayed = 1; return NULL; } - if (unlikely(blk_queue_dead(q)) || + if (unlikely(blk_queue_dying(q)) || !q->elevator->type->ops.elevator_dispatch_fn(q, 0)) return NULL; } diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index da36a3a81a9..f29a1a9b54d 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1406,7 +1406,7 @@ static int scsi_lld_busy(struct request_queue *q) struct scsi_device *sdev = q->queuedata; struct Scsi_Host *shost; - if (blk_queue_dead(q)) + if (blk_queue_dying(q)) return 0; shost = sdev->host; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 1756001210d..aba8246afe7 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -437,7 +437,7 @@ struct request_queue { #define QUEUE_FLAG_STOPPED 2 /* queue is stopped */ #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */ #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ -#define QUEUE_FLAG_DEAD 5 /* queue being torn down */ +#define QUEUE_FLAG_DYING 5 /* queue being torn down */ #define QUEUE_FLAG_BYPASS 6 /* act as dumb FIFO queue */ #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */ #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */ @@ -521,7 +521,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) -#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) +#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) #define blk_queue_noxmerges(q) \ -- cgit v1.2.3-70-g09d2 From c246e80d86736312933646896c4157daf511dadc Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 6 Dec 2012 14:32:01 +0100 Subject: block: Avoid that request_fn is invoked on a dead queue A block driver may start cleaning up resources needed by its request_fn as soon as blk_cleanup_queue() finished, so request_fn must not be invoked after draining finished. This is important when blk_run_queue() is invoked without any requests in progress. As an example, if blk_drain_queue() and scsi_run_queue() run in parallel, blk_drain_queue() may have finished all requests after scsi_run_queue() has taken a SCSI device off the starved list but before that last function has had a chance to run the queue. Signed-off-by: Bart Van Assche Cc: James Bottomley Cc: Mike Christie Cc: Chanho Min Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-core.c | 31 +++++++++++++++++++++++++++---- block/blk-exec.c | 2 +- block/blk.h | 2 ++ include/linux/blkdev.h | 2 ++ 4 files changed, 32 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/block/blk-core.c b/block/blk-core.c index a182b586b06..f52d05ff5d2 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -292,6 +292,25 @@ void blk_sync_queue(struct request_queue *q) } EXPORT_SYMBOL(blk_sync_queue); +/** + * __blk_run_queue_uncond - run a queue whether or not it has been stopped + * @q: The queue to run + * + * Description: + * Invoke request handling on a queue if there are any pending requests. + * May be used to restart request handling after a request has completed. + * This variant runs the queue whether or not the queue has been + * stopped. Must be called with the queue lock held and interrupts + * disabled. See also @blk_run_queue. + */ +inline void __blk_run_queue_uncond(struct request_queue *q) +{ + if (unlikely(blk_queue_dead(q))) + return; + + q->request_fn(q); +} + /** * __blk_run_queue - run a single device queue * @q: The queue to run @@ -305,7 +324,7 @@ void __blk_run_queue(struct request_queue *q) if (unlikely(blk_queue_stopped(q))) return; - q->request_fn(q); + __blk_run_queue_uncond(q); } EXPORT_SYMBOL(__blk_run_queue); @@ -477,8 +496,8 @@ EXPORT_SYMBOL_GPL(blk_queue_bypass_end); * blk_cleanup_queue - shutdown a request queue * @q: request queue to shutdown * - * Mark @q DYING, drain all pending requests, destroy and put it. All - * future requests will be failed immediately with -ENODEV. + * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and + * put it. All future requests will be failed immediately with -ENODEV. */ void blk_cleanup_queue(struct request_queue *q) { @@ -507,9 +526,13 @@ void blk_cleanup_queue(struct request_queue *q) spin_unlock_irq(lock); mutex_unlock(&q->sysfs_lock); - /* drain all requests queued before DYING marking */ + /* + * Drain all requests queued before DYING marking. Set DEAD flag to + * prevent that q->request_fn() gets invoked after draining finished. + */ spin_lock_irq(lock); __blk_drain_queue(q, true); + queue_flag_set(QUEUE_FLAG_DEAD, q); spin_unlock_irq(lock); /* @q won't process any more request, flush async actions */ diff --git a/block/blk-exec.c b/block/blk-exec.c index 4aec98df7ba..1320e74d79b 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -72,7 +72,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, __blk_run_queue(q); /* the queue is stopped so it won't be run */ if (rq->cmd_type == REQ_TYPE_PM_RESUME) - q->request_fn(q); + __blk_run_queue_uncond(q); spin_unlock_irq(q->queue_lock); } EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); diff --git a/block/blk.h b/block/blk.h index 2218a8a7829..47fdfdd4152 100644 --- a/block/blk.h +++ b/block/blk.h @@ -145,6 +145,8 @@ int blk_try_merge(struct request *rq, struct bio *bio); void blk_queue_congestion_threshold(struct request_queue *q); +void __blk_run_queue_uncond(struct request_queue *q); + int blk_dev_init(void); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index aba8246afe7..8bc46c250ca 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -452,6 +452,7 @@ struct request_queue { #define QUEUE_FLAG_ADD_RANDOM 16 /* Contributes to random pool */ #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */ #define QUEUE_FLAG_SAME_FORCE 18 /* force complete on same CPU */ +#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */ #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_STACKABLE) | \ @@ -522,6 +523,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) #define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags) +#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) #define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) #define blk_queue_noxmerges(q) \ -- cgit v1.2.3-70-g09d2 From 24faf6f604efe18236bded4303009fc252913bf0 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 28 Nov 2012 13:46:45 +0100 Subject: block: Make blk_cleanup_queue() wait until request_fn finished Some request_fn implementations, e.g. scsi_request_fn(), unlock the queue lock internally. This may result in multiple threads executing request_fn for the same queue simultaneously. Keep track of the number of active request_fn calls and make sure that blk_cleanup_queue() waits until all active request_fn invocations have finished. A block driver may start cleaning up resources needed by its request_fn as soon as blk_cleanup_queue() finished, so blk_cleanup_queue() must wait for all outstanding request_fn invocations to finish. Signed-off-by: Bart Van Assche Reported-by: Chanho Min Cc: James Bottomley Cc: Mike Christie Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-core.c | 10 ++++++++++ include/linux/blkdev.h | 6 ++++++ 2 files changed, 16 insertions(+) (limited to 'include') diff --git a/block/blk-core.c b/block/blk-core.c index 9fb23537c7a..473015ee28a 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -309,7 +309,16 @@ inline void __blk_run_queue_uncond(struct request_queue *q) if (unlikely(blk_queue_dead(q))) return; + /* + * Some request_fn implementations, e.g. scsi_request_fn(), unlock + * the queue lock internally. As a result multiple threads may be + * running such a request function concurrently. Keep track of the + * number of active request_fn invocations such that blk_drain_queue() + * can wait until all these request_fn calls have finished. + */ + q->request_fn_active++; q->request_fn(q); + q->request_fn_active--; } /** @@ -408,6 +417,7 @@ static void __blk_drain_queue(struct request_queue *q, bool drain_all) __blk_run_queue(q); drain |= q->nr_rqs_elvpriv; + drain |= q->request_fn_active; /* * Unfortunately, requests are queued at and tracked from diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 8bc46c250ca..c9d233e727f 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -378,6 +378,12 @@ struct request_queue { unsigned int nr_sorted; unsigned int in_flight[2]; + /* + * Number of active block driver functions for which blk_drain_queue() + * must wait. Must be incremented around functions that unlock the + * queue_lock internally, e.g. scsi_request_fn(). + */ + unsigned int request_fn_active; unsigned int rq_timeout; struct timer_list timeout; -- cgit v1.2.3-70-g09d2 From 80729beb3326fd682543f4f4ea534df47ab48967 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 28 Nov 2012 13:47:36 +0100 Subject: bsg: Remove unused function bsg_goose_queue() The function bsg_goose_queue() does not have any in-tree callers, so let's remove it. Signed-off-by: Bart Van Assche Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/bsg-lib.c | 13 ------------- include/linux/bsg-lib.h | 1 - 2 files changed, 14 deletions(-) (limited to 'include') diff --git a/block/bsg-lib.c b/block/bsg-lib.c index deee61fbb74..650f427d915 100644 --- a/block/bsg-lib.c +++ b/block/bsg-lib.c @@ -151,19 +151,6 @@ failjob_rls_job: return -ENOMEM; } -/* - * bsg_goose_queue - restart queue in case it was stopped - * @q: request q to be restarted - */ -void bsg_goose_queue(struct request_queue *q) -{ - if (!q) - return; - - blk_run_queue_async(q); -} -EXPORT_SYMBOL_GPL(bsg_goose_queue); - /** * bsg_request_fn - generic handler for bsg requests * @q: request queue to manage diff --git a/include/linux/bsg-lib.h b/include/linux/bsg-lib.h index 4d0fb3df2f4..a226652a5a6 100644 --- a/include/linux/bsg-lib.h +++ b/include/linux/bsg-lib.h @@ -67,6 +67,5 @@ void bsg_job_done(struct bsg_job *job, int result, int bsg_setup_queue(struct device *dev, struct request_queue *q, char *name, bsg_job_fn *job_fn, int dd_job_size); void bsg_request_fn(struct request_queue *q); -void bsg_goose_queue(struct request_queue *q); #endif -- cgit v1.2.3-70-g09d2 From 8dd2cb7e880d2f77fba53b523c99133ad5054cfd Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Fri, 14 Dec 2012 11:15:36 +0800 Subject: block: discard granularity might not be power of 2 In MD raid case, discard granularity might not be power of 2, for example, a 4-disk raid5 has 3*chunk_size discard granularity. Correct the calculation for such cases. Reported-by: Neil Brown Signed-off-by: Shaohua Li Signed-off-by: Jens Axboe --- block/blk-lib.c | 23 +++++++++++++---------- block/blk-settings.c | 6 +++--- include/linux/blkdev.h | 7 ++++--- 3 files changed, 20 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/block/blk-lib.c b/block/blk-lib.c index 9373b58dfab..5677fd33d7d 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -43,8 +43,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, DECLARE_COMPLETION_ONSTACK(wait); struct request_queue *q = bdev_get_queue(bdev); int type = REQ_WRITE | REQ_DISCARD; - unsigned int max_discard_sectors; - unsigned int granularity, alignment, mask; + sector_t max_discard_sectors; + sector_t granularity, alignment; struct bio_batch bb; struct bio *bio; int ret = 0; @@ -57,15 +57,16 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, /* Zero-sector (unknown) and one-sector granularities are the same. */ granularity = max(q->limits.discard_granularity >> 9, 1U); - mask = granularity - 1; - alignment = (bdev_discard_alignment(bdev) >> 9) & mask; + alignment = bdev_discard_alignment(bdev) >> 9; + alignment = sector_div(alignment, granularity); /* * Ensure that max_discard_sectors is of the proper * granularity, so that requests stay aligned after a split. */ max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9); - max_discard_sectors = round_down(max_discard_sectors, granularity); + sector_div(max_discard_sectors, granularity); + max_discard_sectors *= granularity; if (unlikely(!max_discard_sectors)) { /* Avoid infinite loop below. Being cautious never hurts. */ return -EOPNOTSUPP; @@ -83,7 +84,7 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, while (nr_sects) { unsigned int req_sects; - sector_t end_sect; + sector_t end_sect, tmp; bio = bio_alloc(gfp_mask, 1); if (!bio) { @@ -98,10 +99,12 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, * misaligned, stop the discard at the previous aligned sector. */ end_sect = sector + req_sects; - if (req_sects < nr_sects && (end_sect & mask) != alignment) { - end_sect = - round_down(end_sect - alignment, granularity) - + alignment; + tmp = end_sect; + if (req_sects < nr_sects && + sector_div(tmp, granularity) != alignment) { + end_sect = end_sect - alignment; + sector_div(end_sect, granularity); + end_sect = end_sect * granularity + alignment; req_sects = end_sect - sector; } diff --git a/block/blk-settings.c b/block/blk-settings.c index 779bb7646bc..c50ecf0ea3b 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -611,7 +611,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, bottom = b->discard_granularity + alignment; /* Verify that top and bottom intervals line up */ - if (max(top, bottom) & (min(top, bottom) - 1)) + if ((max(top, bottom) % min(top, bottom)) != 0) t->discard_misaligned = 1; } @@ -619,8 +619,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, b->max_discard_sectors); t->discard_granularity = max(t->discard_granularity, b->discard_granularity); - t->discard_alignment = lcm(t->discard_alignment, alignment) & - (t->discard_granularity - 1); + t->discard_alignment = lcm(t->discard_alignment, alignment) % + t->discard_granularity; } return ret; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c9d233e727f..acb4f7bbbd3 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1188,13 +1188,14 @@ static inline int queue_discard_alignment(struct request_queue *q) static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector) { - unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1); + sector_t alignment = sector << 9; + alignment = sector_div(alignment, lim->discard_granularity); if (!lim->max_discard_sectors) return 0; - return (lim->discard_granularity + lim->discard_alignment - alignment) - & (lim->discard_granularity - 1); + alignment = lim->discard_granularity + lim->discard_alignment - alignment; + return sector_div(alignment, lim->discard_granularity); } static inline int bdev_discard_alignment(struct block_device *bdev) -- cgit v1.2.3-70-g09d2