From 01c5f85aebaaddfd7e6051fb2ec80c1d4b463554 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 11 Sep 2018 10:59:53 -0600 Subject: blk-cgroup: increase number of supported policies After merging the iolatency policy, we potentially now have 4 policies being registered, but only support 3. This causes one of them to fail loading. Takashi reports that BFQ no longer works for him, because it fails to load due to policy registration failure. Bump to 5 policies, and also add a warning for when we have exceeded the global amount. If we have to touch this again, we should switch to a dynamic scheme instead. Reported-by: Takashi Iwai Reviewed-by: Jeff Moyer Tested-by: Takashi Iwai Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d6869e0e2b64..6980014357d4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -54,7 +54,7 @@ struct blk_stat_callback; * Maximum number of blkcg policies allowed to be registered concurrently. * Defined here to simplify include dependency. */ -#define BLKCG_MAX_POLS 3 +#define BLKCG_MAX_POLS 5 typedef void (rq_end_io_fn)(struct request *, blk_status_t); -- cgit v1.2.3-71-gd317 From 43b729bfe9cf30ad11499a66e3b7bd300c716d44 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 24 Sep 2018 09:43:47 +0200 Subject: block: move integrity_req_gap_{back,front}_merge to blk.h No need to expose these to drivers. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk.h | 35 +++++++++++++++++++++++++++++++++-- include/linux/blkdev.h | 31 ------------------------------- 2 files changed, 33 insertions(+), 33 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk.h b/block/blk.h index 9db4e389582c..441c2de1d4b9 100644 --- a/block/blk.h +++ b/block/blk.h @@ -158,7 +158,38 @@ static inline bool bio_integrity_endio(struct bio *bio) return __bio_integrity_endio(bio); return true; } -#else + +static inline bool integrity_req_gap_back_merge(struct request *req, + struct bio *next) +{ + struct bio_integrity_payload *bip = bio_integrity(req->bio); + struct bio_integrity_payload *bip_next = bio_integrity(next); + + return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], + bip_next->bip_vec[0].bv_offset); +} + +static inline bool integrity_req_gap_front_merge(struct request *req, + struct bio *bio) +{ + struct bio_integrity_payload *bip = bio_integrity(bio); + struct bio_integrity_payload *bip_next = bio_integrity(req->bio); + + return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], + bip_next->bip_vec[0].bv_offset); +} +#else /* CONFIG_BLK_DEV_INTEGRITY */ +static inline bool integrity_req_gap_back_merge(struct request *req, + struct bio *next) +{ + return false; +} +static inline bool integrity_req_gap_front_merge(struct request *req, + struct bio *bio) +{ + return false; +} + static inline void blk_flush_integrity(void) { } @@ -166,7 +197,7 @@ static inline bool bio_integrity_endio(struct bio *bio) { return true; } -#endif +#endif /* CONFIG_BLK_DEV_INTEGRITY */ void blk_timeout_work(struct work_struct *work); unsigned long blk_rq_timeout(unsigned long timeout); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d6869e0e2b64..bc534c857344 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1843,26 +1843,6 @@ queue_max_integrity_segments(struct request_queue *q) return q->limits.max_integrity_segments; } -static inline bool integrity_req_gap_back_merge(struct request *req, - struct bio *next) -{ - struct bio_integrity_payload *bip = bio_integrity(req->bio); - struct bio_integrity_payload *bip_next = bio_integrity(next); - - return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], - bip_next->bip_vec[0].bv_offset); -} - -static inline bool integrity_req_gap_front_merge(struct request *req, - struct bio *bio) -{ - struct bio_integrity_payload *bip = bio_integrity(bio); - struct bio_integrity_payload *bip_next = bio_integrity(req->bio); - - return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], - bip_next->bip_vec[0].bv_offset); -} - /** * bio_integrity_intervals - Return number of integrity intervals for a bio * @bi: blk_integrity profile for device @@ -1947,17 +1927,6 @@ static inline bool blk_integrity_merge_bio(struct request_queue *rq, return true; } -static inline bool integrity_req_gap_back_merge(struct request *req, - struct bio *next) -{ - return false; -} -static inline bool integrity_req_gap_front_merge(struct request *req, - struct bio *bio) -{ - return false; -} - static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, unsigned int sectors) { -- cgit v1.2.3-71-gd317 From e9907009cbfc0c93d987d5a8fdf3d6c3c7b89717 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 24 Sep 2018 09:43:48 +0200 Subject: block: move req_gap_{back,front}_merge to blk-merge.c Keep it close to the actual users instead of exposing the function to all drivers. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-merge.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/blkdev.h | 69 -------------------------------------------------- 2 files changed, 65 insertions(+), 69 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-merge.c b/block/blk-merge.c index aaec38cc37b8..ad8a226347a6 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -12,6 +12,71 @@ #include "blk.h" +/* + * Check if the two bvecs from two bios can be merged to one segment. If yes, + * no need to check gap between the two bios since the 1st bio and the 1st bvec + * in the 2nd bio can be handled in one segment. + */ +static inline bool bios_segs_mergeable(struct request_queue *q, + struct bio *prev, struct bio_vec *prev_last_bv, + struct bio_vec *next_first_bv) +{ + if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv)) + return false; + if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv)) + return false; + if (prev->bi_seg_back_size + next_first_bv->bv_len > + queue_max_segment_size(q)) + return false; + return true; +} + +static inline bool bio_will_gap(struct request_queue *q, + struct request *prev_rq, struct bio *prev, struct bio *next) +{ + struct bio_vec pb, nb; + + if (!bio_has_data(prev) || !queue_virt_boundary(q)) + return false; + + /* + * Don't merge if the 1st bio starts with non-zero offset, otherwise it + * is quite difficult to respect the sg gap limit. We work hard to + * merge a huge number of small single bios in case of mkfs. + */ + if (prev_rq) + bio_get_first_bvec(prev_rq->bio, &pb); + else + bio_get_first_bvec(prev, &pb); + if (pb.bv_offset) + return true; + + /* + * We don't need to worry about the situation that the merged segment + * ends in unaligned virt boundary: + * + * - if 'pb' ends aligned, the merged segment ends aligned + * - if 'pb' ends unaligned, the next bio must include + * one single bvec of 'nb', otherwise the 'nb' can't + * merge with 'pb' + */ + bio_get_last_bvec(prev, &pb); + bio_get_first_bvec(next, &nb); + if (bios_segs_mergeable(q, prev, &pb, &nb)) + return false; + return __bvec_gap_to_prev(q, &pb, nb.bv_offset); +} + +static inline bool req_gap_back_merge(struct request *req, struct bio *bio) +{ + return bio_will_gap(req->q, req, req->biotail, bio); +} + +static inline bool req_gap_front_merge(struct request *req, struct bio *bio) +{ + return bio_will_gap(req->q, NULL, bio, req->bio); +} + static struct bio *blk_bio_discard_split(struct request_queue *q, struct bio *bio, struct bio_set *bs, diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index bc534c857344..b7e676bb01bc 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1695,75 +1695,6 @@ static inline bool bvec_gap_to_prev(struct request_queue *q, return __bvec_gap_to_prev(q, bprv, offset); } -/* - * Check if the two bvecs from two bios can be merged to one segment. - * If yes, no need to check gap between the two bios since the 1st bio - * and the 1st bvec in the 2nd bio can be handled in one segment. - */ -static inline bool bios_segs_mergeable(struct request_queue *q, - struct bio *prev, struct bio_vec *prev_last_bv, - struct bio_vec *next_first_bv) -{ - if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv)) - return false; - if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv)) - return false; - if (prev->bi_seg_back_size + next_first_bv->bv_len > - queue_max_segment_size(q)) - return false; - return true; -} - -static inline bool bio_will_gap(struct request_queue *q, - struct request *prev_rq, - struct bio *prev, - struct bio *next) -{ - if (bio_has_data(prev) && queue_virt_boundary(q)) { - struct bio_vec pb, nb; - - /* - * don't merge if the 1st bio starts with non-zero - * offset, otherwise it is quite difficult to respect - * sg gap limit. We work hard to merge a huge number of small - * single bios in case of mkfs. - */ - if (prev_rq) - bio_get_first_bvec(prev_rq->bio, &pb); - else - bio_get_first_bvec(prev, &pb); - if (pb.bv_offset) - return true; - - /* - * We don't need to worry about the situation that the - * merged segment ends in unaligned virt boundary: - * - * - if 'pb' ends aligned, the merged segment ends aligned - * - if 'pb' ends unaligned, the next bio must include - * one single bvec of 'nb', otherwise the 'nb' can't - * merge with 'pb' - */ - bio_get_last_bvec(prev, &pb); - bio_get_first_bvec(next, &nb); - - if (!bios_segs_mergeable(q, prev, &pb, &nb)) - return __bvec_gap_to_prev(q, &pb, nb.bv_offset); - } - - return false; -} - -static inline bool req_gap_back_merge(struct request *req, struct bio *bio) -{ - return bio_will_gap(req->q, req, req->biotail, bio); -} - -static inline bool req_gap_front_merge(struct request *req, struct bio *bio) -{ - return bio_will_gap(req->q, NULL, bio, req->bio); -} - int kblockd_schedule_work(struct work_struct *work); int kblockd_schedule_work_on(int cpu, struct work_struct *work); int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); -- cgit v1.2.3-71-gd317 From 27ca1d4ed04ea29dc77b47190a3cc82697023e76 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 24 Sep 2018 09:43:49 +0200 Subject: block: move req_gap_back_merge to blk.h No need to expose these helpers outside the block layer. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk.h | 19 +++++++++++++++++++ include/linux/blkdev.h | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk.h b/block/blk.h index 441c2de1d4b9..63035c95689c 100644 --- a/block/blk.h +++ b/block/blk.h @@ -149,6 +149,25 @@ static inline void blk_queue_enter_live(struct request_queue *q) percpu_ref_get(&q->q_usage_counter); } +static inline bool __bvec_gap_to_prev(struct request_queue *q, + struct bio_vec *bprv, unsigned int offset) +{ + return offset || + ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); +} + +/* + * Check if adding a bio_vec after bprv with offset would create a gap in + * the SG list. Most drivers don't care about this, but some do. + */ +static inline bool bvec_gap_to_prev(struct request_queue *q, + struct bio_vec *bprv, unsigned int offset) +{ + if (!queue_virt_boundary(q)) + return false; + return __bvec_gap_to_prev(q, bprv, offset); +} + #ifdef CONFIG_BLK_DEV_INTEGRITY void blk_flush_integrity(void); bool __bio_integrity_endio(struct bio *); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b7e676bb01bc..1d5e14139795 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1676,25 +1676,6 @@ static inline void put_dev_sector(Sector p) put_page(p.v); } -static inline bool __bvec_gap_to_prev(struct request_queue *q, - struct bio_vec *bprv, unsigned int offset) -{ - return offset || - ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); -} - -/* - * Check if adding a bio_vec after bprv with offset would create a gap in - * the SG list. Most drivers don't care about this, but some do. - */ -static inline bool bvec_gap_to_prev(struct request_queue *q, - struct bio_vec *bprv, unsigned int offset) -{ - if (!queue_virt_boundary(q)) - return false; - return __bvec_gap_to_prev(q, bprv, offset); -} - int kblockd_schedule_work(struct work_struct *work); int kblockd_schedule_work_on(int cpu, struct work_struct *work); int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); -- cgit v1.2.3-71-gd317 From bca6b067b0b269a7b8ba129e2a918309ca8b4a55 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 26 Sep 2018 14:01:03 -0700 Subject: block: Move power management code into a new source file Move the code for runtime power management from blk-core.c into the new source file blk-pm.c. Move the corresponding declarations from into . For CONFIG_PM=n, leave out the declarations of the functions that are not used in that mode. This patch not only reduces the number of #ifdefs in the block layer core code but also reduces the size of header file and hence should help to reduce the build time of the Linux kernel if CONFIG_PM is not defined. Signed-off-by: Bart Van Assche Reviewed-by: Ming Lei Reviewed-by: Christoph Hellwig Cc: Jianchao Wang Cc: Hannes Reinecke Cc: Johannes Thumshirn Cc: Alan Stern Signed-off-by: Jens Axboe --- block/Kconfig | 3 + block/Makefile | 1 + block/blk-core.c | 196 +------------------------------------------------ block/blk-pm.c | 188 +++++++++++++++++++++++++++++++++++++++++++++++ block/blk-pm.h | 43 +++++++++++ block/elevator.c | 22 +----- drivers/scsi/scsi_pm.c | 1 + drivers/scsi/sd.c | 1 + drivers/scsi/sr.c | 1 + include/linux/blk-pm.h | 24 ++++++ include/linux/blkdev.h | 23 ------ 11 files changed, 264 insertions(+), 239 deletions(-) create mode 100644 block/blk-pm.c create mode 100644 block/blk-pm.h create mode 100644 include/linux/blk-pm.h (limited to 'include/linux/blkdev.h') diff --git a/block/Kconfig b/block/Kconfig index 1f2469a0123c..85263e7bded6 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -228,4 +228,7 @@ config BLK_MQ_RDMA depends on BLOCK && INFINIBAND default y +config BLK_PM + def_bool BLOCK && PM + source block/Kconfig.iosched diff --git a/block/Makefile b/block/Makefile index 572b33f32c07..27eac600474f 100644 --- a/block/Makefile +++ b/block/Makefile @@ -37,3 +37,4 @@ obj-$(CONFIG_BLK_WBT) += blk-wbt.o obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o obj-$(CONFIG_BLK_DEBUG_FS_ZONED)+= blk-mq-debugfs-zoned.o obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o +obj-$(CONFIG_BLK_PM) += blk-pm.o diff --git a/block/blk-core.c b/block/blk-core.c index 4dbc93f43b38..6d4dd176bd9d 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -42,6 +42,7 @@ #include "blk.h" #include "blk-mq.h" #include "blk-mq-sched.h" +#include "blk-pm.h" #include "blk-rq-qos.h" #ifdef CONFIG_DEBUG_FS @@ -1726,16 +1727,6 @@ void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part) } EXPORT_SYMBOL_GPL(part_round_stats); -#ifdef CONFIG_PM -static void blk_pm_put_request(struct request *rq) -{ - if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending) - pm_runtime_mark_last_busy(rq->q->dev); -} -#else -static inline void blk_pm_put_request(struct request *rq) {} -#endif - void __blk_put_request(struct request_queue *q, struct request *req) { req_flags_t rq_flags = req->rq_flags; @@ -3757,191 +3748,6 @@ void blk_finish_plug(struct blk_plug *plug) } EXPORT_SYMBOL(blk_finish_plug); -#ifdef CONFIG_PM -/** - * blk_pm_runtime_init - Block layer runtime PM initialization routine - * @q: the queue of the device - * @dev: the device the queue belongs to - * - * Description: - * Initialize runtime-PM-related fields for @q and start auto suspend for - * @dev. Drivers that want to take advantage of request-based runtime PM - * should call this function after @dev has been initialized, and its - * request queue @q has been allocated, and runtime PM for it can not happen - * yet(either due to disabled/forbidden or its usage_count > 0). In most - * cases, driver should call this function before any I/O has taken place. - * - * This function takes care of setting up using auto suspend for the device, - * the autosuspend delay is set to -1 to make runtime suspend impossible - * until an updated value is either set by user or by driver. Drivers do - * not need to touch other autosuspend settings. - * - * The block layer runtime PM is request based, so only works for drivers - * that use request as their IO unit instead of those directly use bio's. - */ -void blk_pm_runtime_init(struct request_queue *q, struct device *dev) -{ - /* Don't enable runtime PM for blk-mq until it is ready */ - if (q->mq_ops) { - pm_runtime_disable(dev); - return; - } - - q->dev = dev; - q->rpm_status = RPM_ACTIVE; - pm_runtime_set_autosuspend_delay(q->dev, -1); - pm_runtime_use_autosuspend(q->dev); -} -EXPORT_SYMBOL(blk_pm_runtime_init); - -/** - * blk_pre_runtime_suspend - Pre runtime suspend check - * @q: the queue of the device - * - * Description: - * This function will check if runtime suspend is allowed for the device - * by examining if there are any requests pending in the queue. If there - * are requests pending, the device can not be runtime suspended; otherwise, - * the queue's status will be updated to SUSPENDING and the driver can - * proceed to suspend the device. - * - * For the not allowed case, we mark last busy for the device so that - * runtime PM core will try to autosuspend it some time later. - * - * This function should be called near the start of the device's - * runtime_suspend callback. - * - * Return: - * 0 - OK to runtime suspend the device - * -EBUSY - Device should not be runtime suspended - */ -int blk_pre_runtime_suspend(struct request_queue *q) -{ - int ret = 0; - - if (!q->dev) - return ret; - - spin_lock_irq(q->queue_lock); - if (q->nr_pending) { - ret = -EBUSY; - pm_runtime_mark_last_busy(q->dev); - } else { - q->rpm_status = RPM_SUSPENDING; - } - spin_unlock_irq(q->queue_lock); - return ret; -} -EXPORT_SYMBOL(blk_pre_runtime_suspend); - -/** - * blk_post_runtime_suspend - Post runtime suspend processing - * @q: the queue of the device - * @err: return value of the device's runtime_suspend function - * - * Description: - * Update the queue's runtime status according to the return value of the - * device's runtime suspend function and mark last busy for the device so - * that PM core will try to auto suspend the device at a later time. - * - * This function should be called near the end of the device's - * runtime_suspend callback. - */ -void blk_post_runtime_suspend(struct request_queue *q, int err) -{ - if (!q->dev) - return; - - spin_lock_irq(q->queue_lock); - if (!err) { - q->rpm_status = RPM_SUSPENDED; - } else { - q->rpm_status = RPM_ACTIVE; - pm_runtime_mark_last_busy(q->dev); - } - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_post_runtime_suspend); - -/** - * blk_pre_runtime_resume - Pre runtime resume processing - * @q: the queue of the device - * - * Description: - * Update the queue's runtime status to RESUMING in preparation for the - * runtime resume of the device. - * - * This function should be called near the start of the device's - * runtime_resume callback. - */ -void blk_pre_runtime_resume(struct request_queue *q) -{ - if (!q->dev) - return; - - spin_lock_irq(q->queue_lock); - q->rpm_status = RPM_RESUMING; - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_pre_runtime_resume); - -/** - * blk_post_runtime_resume - Post runtime resume processing - * @q: the queue of the device - * @err: return value of the device's runtime_resume function - * - * Description: - * Update the queue's runtime status according to the return value of the - * device's runtime_resume function. If it is successfully resumed, process - * the requests that are queued into the device's queue when it is resuming - * and then mark last busy and initiate autosuspend for it. - * - * This function should be called near the end of the device's - * runtime_resume callback. - */ -void blk_post_runtime_resume(struct request_queue *q, int err) -{ - if (!q->dev) - return; - - spin_lock_irq(q->queue_lock); - if (!err) { - q->rpm_status = RPM_ACTIVE; - __blk_run_queue(q); - pm_runtime_mark_last_busy(q->dev); - pm_request_autosuspend(q->dev); - } else { - q->rpm_status = RPM_SUSPENDED; - } - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_post_runtime_resume); - -/** - * blk_set_runtime_active - Force runtime status of the queue to be active - * @q: the queue of the device - * - * If the device is left runtime suspended during system suspend the resume - * hook typically resumes the device and corrects runtime status - * accordingly. However, that does not affect the queue runtime PM status - * which is still "suspended". This prevents processing requests from the - * queue. - * - * This function can be used in driver's resume hook to correct queue - * runtime PM status and re-enable peeking requests from the queue. It - * should be called before first request is added to the queue. - */ -void blk_set_runtime_active(struct request_queue *q) -{ - spin_lock_irq(q->queue_lock); - q->rpm_status = RPM_ACTIVE; - pm_runtime_mark_last_busy(q->dev); - pm_request_autosuspend(q->dev); - spin_unlock_irq(q->queue_lock); -} -EXPORT_SYMBOL(blk_set_runtime_active); -#endif - int __init blk_dev_init(void) { BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS)); diff --git a/block/blk-pm.c b/block/blk-pm.c new file mode 100644 index 000000000000..9b636960d285 --- /dev/null +++ b/block/blk-pm.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +/** + * blk_pm_runtime_init - Block layer runtime PM initialization routine + * @q: the queue of the device + * @dev: the device the queue belongs to + * + * Description: + * Initialize runtime-PM-related fields for @q and start auto suspend for + * @dev. Drivers that want to take advantage of request-based runtime PM + * should call this function after @dev has been initialized, and its + * request queue @q has been allocated, and runtime PM for it can not happen + * yet(either due to disabled/forbidden or its usage_count > 0). In most + * cases, driver should call this function before any I/O has taken place. + * + * This function takes care of setting up using auto suspend for the device, + * the autosuspend delay is set to -1 to make runtime suspend impossible + * until an updated value is either set by user or by driver. Drivers do + * not need to touch other autosuspend settings. + * + * The block layer runtime PM is request based, so only works for drivers + * that use request as their IO unit instead of those directly use bio's. + */ +void blk_pm_runtime_init(struct request_queue *q, struct device *dev) +{ + /* Don't enable runtime PM for blk-mq until it is ready */ + if (q->mq_ops) { + pm_runtime_disable(dev); + return; + } + + q->dev = dev; + q->rpm_status = RPM_ACTIVE; + pm_runtime_set_autosuspend_delay(q->dev, -1); + pm_runtime_use_autosuspend(q->dev); +} +EXPORT_SYMBOL(blk_pm_runtime_init); + +/** + * blk_pre_runtime_suspend - Pre runtime suspend check + * @q: the queue of the device + * + * Description: + * This function will check if runtime suspend is allowed for the device + * by examining if there are any requests pending in the queue. If there + * are requests pending, the device can not be runtime suspended; otherwise, + * the queue's status will be updated to SUSPENDING and the driver can + * proceed to suspend the device. + * + * For the not allowed case, we mark last busy for the device so that + * runtime PM core will try to autosuspend it some time later. + * + * This function should be called near the start of the device's + * runtime_suspend callback. + * + * Return: + * 0 - OK to runtime suspend the device + * -EBUSY - Device should not be runtime suspended + */ +int blk_pre_runtime_suspend(struct request_queue *q) +{ + int ret = 0; + + if (!q->dev) + return ret; + + spin_lock_irq(q->queue_lock); + if (q->nr_pending) { + ret = -EBUSY; + pm_runtime_mark_last_busy(q->dev); + } else { + q->rpm_status = RPM_SUSPENDING; + } + spin_unlock_irq(q->queue_lock); + return ret; +} +EXPORT_SYMBOL(blk_pre_runtime_suspend); + +/** + * blk_post_runtime_suspend - Post runtime suspend processing + * @q: the queue of the device + * @err: return value of the device's runtime_suspend function + * + * Description: + * Update the queue's runtime status according to the return value of the + * device's runtime suspend function and mark last busy for the device so + * that PM core will try to auto suspend the device at a later time. + * + * This function should be called near the end of the device's + * runtime_suspend callback. + */ +void blk_post_runtime_suspend(struct request_queue *q, int err) +{ + if (!q->dev) + return; + + spin_lock_irq(q->queue_lock); + if (!err) { + q->rpm_status = RPM_SUSPENDED; + } else { + q->rpm_status = RPM_ACTIVE; + pm_runtime_mark_last_busy(q->dev); + } + spin_unlock_irq(q->queue_lock); +} +EXPORT_SYMBOL(blk_post_runtime_suspend); + +/** + * blk_pre_runtime_resume - Pre runtime resume processing + * @q: the queue of the device + * + * Description: + * Update the queue's runtime status to RESUMING in preparation for the + * runtime resume of the device. + * + * This function should be called near the start of the device's + * runtime_resume callback. + */ +void blk_pre_runtime_resume(struct request_queue *q) +{ + if (!q->dev) + return; + + spin_lock_irq(q->queue_lock); + q->rpm_status = RPM_RESUMING; + spin_unlock_irq(q->queue_lock); +} +EXPORT_SYMBOL(blk_pre_runtime_resume); + +/** + * blk_post_runtime_resume - Post runtime resume processing + * @q: the queue of the device + * @err: return value of the device's runtime_resume function + * + * Description: + * Update the queue's runtime status according to the return value of the + * device's runtime_resume function. If it is successfully resumed, process + * the requests that are queued into the device's queue when it is resuming + * and then mark last busy and initiate autosuspend for it. + * + * This function should be called near the end of the device's + * runtime_resume callback. + */ +void blk_post_runtime_resume(struct request_queue *q, int err) +{ + if (!q->dev) + return; + + spin_lock_irq(q->queue_lock); + if (!err) { + q->rpm_status = RPM_ACTIVE; + __blk_run_queue(q); + pm_runtime_mark_last_busy(q->dev); + pm_request_autosuspend(q->dev); + } else { + q->rpm_status = RPM_SUSPENDED; + } + spin_unlock_irq(q->queue_lock); +} +EXPORT_SYMBOL(blk_post_runtime_resume); + +/** + * blk_set_runtime_active - Force runtime status of the queue to be active + * @q: the queue of the device + * + * If the device is left runtime suspended during system suspend the resume + * hook typically resumes the device and corrects runtime status + * accordingly. However, that does not affect the queue runtime PM status + * which is still "suspended". This prevents processing requests from the + * queue. + * + * This function can be used in driver's resume hook to correct queue + * runtime PM status and re-enable peeking requests from the queue. It + * should be called before first request is added to the queue. + */ +void blk_set_runtime_active(struct request_queue *q) +{ + spin_lock_irq(q->queue_lock); + q->rpm_status = RPM_ACTIVE; + pm_runtime_mark_last_busy(q->dev); + pm_request_autosuspend(q->dev); + spin_unlock_irq(q->queue_lock); +} +EXPORT_SYMBOL(blk_set_runtime_active); diff --git a/block/blk-pm.h b/block/blk-pm.h new file mode 100644 index 000000000000..1ffc8ef203ec --- /dev/null +++ b/block/blk-pm.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _BLOCK_BLK_PM_H_ +#define _BLOCK_BLK_PM_H_ + +#include + +#ifdef CONFIG_PM +static inline void blk_pm_requeue_request(struct request *rq) +{ + if (rq->q->dev && !(rq->rq_flags & RQF_PM)) + rq->q->nr_pending--; +} + +static inline void blk_pm_add_request(struct request_queue *q, + struct request *rq) +{ + if (q->dev && !(rq->rq_flags & RQF_PM) && q->nr_pending++ == 0 && + (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING)) + pm_request_resume(q->dev); +} + +static inline void blk_pm_put_request(struct request *rq) +{ + if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending) + pm_runtime_mark_last_busy(rq->q->dev); +} +#else +static inline void blk_pm_requeue_request(struct request *rq) +{ +} + +static inline void blk_pm_add_request(struct request_queue *q, + struct request *rq) +{ +} + +static inline void blk_pm_put_request(struct request *rq) +{ +} +#endif + +#endif /* _BLOCK_BLK_PM_H_ */ diff --git a/block/elevator.c b/block/elevator.c index 6a06b5d040e5..e18ac68626e3 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -41,6 +41,7 @@ #include "blk.h" #include "blk-mq-sched.h" +#include "blk-pm.h" #include "blk-wbt.h" static DEFINE_SPINLOCK(elv_list_lock); @@ -557,27 +558,6 @@ void elv_bio_merged(struct request_queue *q, struct request *rq, e->type->ops.sq.elevator_bio_merged_fn(q, rq, bio); } -#ifdef CONFIG_PM -static void blk_pm_requeue_request(struct request *rq) -{ - if (rq->q->dev && !(rq->rq_flags & RQF_PM)) - rq->q->nr_pending--; -} - -static void blk_pm_add_request(struct request_queue *q, struct request *rq) -{ - if (q->dev && !(rq->rq_flags & RQF_PM) && q->nr_pending++ == 0 && - (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING)) - pm_request_resume(q->dev); -} -#else -static inline void blk_pm_requeue_request(struct request *rq) {} -static inline void blk_pm_add_request(struct request_queue *q, - struct request *rq) -{ -} -#endif - void elv_requeue_request(struct request_queue *q, struct request *rq) { /* diff --git a/drivers/scsi/scsi_pm.c b/drivers/scsi/scsi_pm.c index b44c1bb687a2..a2b4179bfdf7 100644 --- a/drivers/scsi/scsi_pm.c +++ b/drivers/scsi/scsi_pm.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index b79b366a94f7..64514e8359e4 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index d0389b20574d..4f07b3410595 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h new file mode 100644 index 000000000000..b80c65aba249 --- /dev/null +++ b/include/linux/blk-pm.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _BLK_PM_H_ +#define _BLK_PM_H_ + +struct device; +struct request_queue; + +/* + * block layer runtime pm functions + */ +#ifdef CONFIG_PM +extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev); +extern int blk_pre_runtime_suspend(struct request_queue *q); +extern void blk_post_runtime_suspend(struct request_queue *q, int err); +extern void blk_pre_runtime_resume(struct request_queue *q); +extern void blk_post_runtime_resume(struct request_queue *q, int err); +extern void blk_set_runtime_active(struct request_queue *q); +#else +static inline void blk_pm_runtime_init(struct request_queue *q, + struct device *dev) {} +#endif + +#endif /* _BLK_PM_H_ */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 1d5e14139795..cd863511dedb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1280,29 +1280,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id, extern void blk_put_queue(struct request_queue *); extern void blk_set_queue_dying(struct request_queue *); -/* - * block layer runtime pm functions - */ -#ifdef CONFIG_PM -extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev); -extern int blk_pre_runtime_suspend(struct request_queue *q); -extern void blk_post_runtime_suspend(struct request_queue *q, int err); -extern void blk_pre_runtime_resume(struct request_queue *q); -extern void blk_post_runtime_resume(struct request_queue *q, int err); -extern void blk_set_runtime_active(struct request_queue *q); -#else -static inline void blk_pm_runtime_init(struct request_queue *q, - struct device *dev) {} -static inline int blk_pre_runtime_suspend(struct request_queue *q) -{ - return -ENOSYS; -} -static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {} -static inline void blk_pre_runtime_resume(struct request_queue *q) {} -static inline void blk_post_runtime_resume(struct request_queue *q, int err) {} -static inline void blk_set_runtime_active(struct request_queue *q) {} -#endif - /* * blk_plug permits building a queue of related requests by holding the I/O * fragments for a short period. This allows merging of sequential requests -- cgit v1.2.3-71-gd317 From cd84a62e0078dce09f4ed349bec84f86c9d54b30 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 26 Sep 2018 14:01:04 -0700 Subject: block, scsi: Change the preempt-only flag into a counter The RQF_PREEMPT flag is used for three purposes: - In the SCSI core, for making sure that power management requests are executed even if a device is in the "quiesced" state. - For domain validation by SCSI drivers that use the parallel port. - In the IDE driver, for IDE preempt requests. Rename "preempt-only" into "pm-only" because the primary purpose of this mode is power management. Since the power management core may but does not have to resume a runtime suspended device before performing system-wide suspend and since a later patch will set "pm-only" mode as long as a block device is runtime suspended, make it possible to set "pm-only" mode from more than one context. Since with this change scsi_device_quiesce() is no longer idempotent, make that function return early if it is called for a quiesced queue. Signed-off-by: Bart Van Assche Acked-by: Martin K. Petersen Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: Ming Lei Cc: Jianchao Wang Cc: Johannes Thumshirn Cc: Alan Stern Signed-off-by: Jens Axboe --- block/blk-core.c | 35 ++++++++++++++++++----------------- block/blk-mq-debugfs.c | 10 +++++++++- drivers/scsi/scsi_lib.c | 11 +++++++---- include/linux/blkdev.h | 14 +++++++++----- 4 files changed, 43 insertions(+), 27 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-core.c b/block/blk-core.c index 6d4dd176bd9d..1a691f5269bb 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -422,24 +422,25 @@ void blk_sync_queue(struct request_queue *q) EXPORT_SYMBOL(blk_sync_queue); /** - * blk_set_preempt_only - set QUEUE_FLAG_PREEMPT_ONLY + * blk_set_pm_only - increment pm_only counter * @q: request queue pointer - * - * Returns the previous value of the PREEMPT_ONLY flag - 0 if the flag was not - * set and 1 if the flag was already set. */ -int blk_set_preempt_only(struct request_queue *q) +void blk_set_pm_only(struct request_queue *q) { - return blk_queue_flag_test_and_set(QUEUE_FLAG_PREEMPT_ONLY, q); + atomic_inc(&q->pm_only); } -EXPORT_SYMBOL_GPL(blk_set_preempt_only); +EXPORT_SYMBOL_GPL(blk_set_pm_only); -void blk_clear_preempt_only(struct request_queue *q) +void blk_clear_pm_only(struct request_queue *q) { - blk_queue_flag_clear(QUEUE_FLAG_PREEMPT_ONLY, q); - wake_up_all(&q->mq_freeze_wq); + int pm_only; + + pm_only = atomic_dec_return(&q->pm_only); + WARN_ON_ONCE(pm_only < 0); + if (pm_only == 0) + wake_up_all(&q->mq_freeze_wq); } -EXPORT_SYMBOL_GPL(blk_clear_preempt_only); +EXPORT_SYMBOL_GPL(blk_clear_pm_only); /** * __blk_run_queue_uncond - run a queue whether or not it has been stopped @@ -918,7 +919,7 @@ EXPORT_SYMBOL(blk_alloc_queue); */ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) { - const bool preempt = flags & BLK_MQ_REQ_PREEMPT; + const bool pm = flags & BLK_MQ_REQ_PREEMPT; while (true) { bool success = false; @@ -926,11 +927,11 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) rcu_read_lock(); if (percpu_ref_tryget_live(&q->q_usage_counter)) { /* - * The code that sets the PREEMPT_ONLY flag is - * responsible for ensuring that that flag is globally - * visible before the queue is unfrozen. + * The code that increments the pm_only counter is + * responsible for ensuring that that counter is + * globally visible before the queue is unfrozen. */ - if (preempt || !blk_queue_preempt_only(q)) { + if (pm || !blk_queue_pm_only(q)) { success = true; } else { percpu_ref_put(&q->q_usage_counter); @@ -955,7 +956,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) wait_event(q->mq_freeze_wq, (atomic_read(&q->mq_freeze_depth) == 0 && - (preempt || !blk_queue_preempt_only(q))) || + (pm || !blk_queue_pm_only(q))) || blk_queue_dying(q)); if (blk_queue_dying(q)) return -ENODEV; diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index cb1e6cf7ac48..a5ea86835fcb 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -102,6 +102,14 @@ static int blk_flags_show(struct seq_file *m, const unsigned long flags, return 0; } +static int queue_pm_only_show(void *data, struct seq_file *m) +{ + struct request_queue *q = data; + + seq_printf(m, "%d\n", atomic_read(&q->pm_only)); + return 0; +} + #define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(QUEUED), @@ -132,7 +140,6 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(REGISTERED), QUEUE_FLAG_NAME(SCSI_PASSTHROUGH), QUEUE_FLAG_NAME(QUIESCED), - QUEUE_FLAG_NAME(PREEMPT_ONLY), }; #undef QUEUE_FLAG_NAME @@ -209,6 +216,7 @@ static ssize_t queue_write_hint_store(void *data, const char __user *buf, static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = { { "poll_stat", 0400, queue_poll_stat_show }, { "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops }, + { "pm_only", 0600, queue_pm_only_show, NULL }, { "state", 0600, queue_state_show, queue_state_write }, { "write_hints", 0600, queue_write_hint_show, queue_write_hint_store }, { "zone_wlock", 0400, queue_zone_wlock_show, NULL }, diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index eb97d2dd3651..62348412ed1b 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -3046,11 +3046,14 @@ scsi_device_quiesce(struct scsi_device *sdev) */ WARN_ON_ONCE(sdev->quiesced_by && sdev->quiesced_by != current); - blk_set_preempt_only(q); + if (sdev->quiesced_by == current) + return 0; + + blk_set_pm_only(q); blk_mq_freeze_queue(q); /* - * Ensure that the effect of blk_set_preempt_only() will be visible + * Ensure that the effect of blk_set_pm_only() will be visible * for percpu_ref_tryget() callers that occur after the queue * unfreeze even if the queue was already frozen before this function * was called. See also https://lwn.net/Articles/573497/. @@ -3063,7 +3066,7 @@ scsi_device_quiesce(struct scsi_device *sdev) if (err == 0) sdev->quiesced_by = current; else - blk_clear_preempt_only(q); + blk_clear_pm_only(q); mutex_unlock(&sdev->state_mutex); return err; @@ -3088,7 +3091,7 @@ void scsi_device_resume(struct scsi_device *sdev) mutex_lock(&sdev->state_mutex); WARN_ON_ONCE(!sdev->quiesced_by); sdev->quiesced_by = NULL; - blk_clear_preempt_only(sdev->request_queue); + blk_clear_pm_only(sdev->request_queue); if (sdev->sdev_state == SDEV_QUIESCE) scsi_device_set_state(sdev, SDEV_RUNNING); mutex_unlock(&sdev->state_mutex); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index cd863511dedb..13bb54f26736 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -504,6 +504,12 @@ struct request_queue { * various queue flags, see QUEUE_* below */ unsigned long queue_flags; + /* + * Number of contexts that have called blk_set_pm_only(). If this + * counter is above zero then only RQF_PM and RQF_PREEMPT requests are + * processed. + */ + atomic_t pm_only; /* * ida allocated id for this queue. Used to index queues from @@ -698,7 +704,6 @@ struct request_queue { #define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */ #define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */ #define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */ -#define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */ #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_SAME_COMP) | \ @@ -736,12 +741,11 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q); ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ REQ_FAILFAST_DRIVER)) #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) -#define blk_queue_preempt_only(q) \ - test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags) +#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) #define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags) -extern int blk_set_preempt_only(struct request_queue *q); -extern void blk_clear_preempt_only(struct request_queue *q); +extern void blk_set_pm_only(struct request_queue *q); +extern void blk_clear_pm_only(struct request_queue *q); static inline int queue_in_flight(struct request_queue *q) { -- cgit v1.2.3-71-gd317 From 4822e902f9bdffaea2817471365e000966f0d1a1 Mon Sep 17 00:00:00 2001 From: Konstantin Khlebnikov Date: Thu, 11 Oct 2018 10:07:06 +0300 Subject: block: describe difference between flags IO_STAT and STATS This adds reasonable comments, but they definitely needs better names. Signed-off-by: Konstantin Khlebnikov Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index dee46c20701b..61207560e826 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -108,7 +108,7 @@ typedef __u32 __bitwise req_flags_t; #define RQF_QUIET ((__force req_flags_t)(1 << 11)) /* elevator private data attached */ #define RQF_ELVPRIV ((__force req_flags_t)(1 << 12)) -/* account I/O stat */ +/* account into disk and partition IO statistics */ #define RQF_IO_STAT ((__force req_flags_t)(1 << 13)) /* request came from our alloc pool */ #define RQF_ALLOCED ((__force req_flags_t)(1 << 14)) @@ -116,7 +116,7 @@ typedef __u32 __bitwise req_flags_t; #define RQF_PM ((__force req_flags_t)(1 << 15)) /* on IO scheduler merge hash */ #define RQF_HASHED ((__force req_flags_t)(1 << 16)) -/* IO stats tracking on */ +/* track IO completion time */ #define RQF_STATS ((__force req_flags_t)(1 << 17)) /* Look at ->special_vec for the actual data payload instead of the bio chain. */ @@ -685,7 +685,7 @@ struct request_queue { #define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */ #define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */ #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ -#define QUEUE_FLAG_IO_STAT 10 /* do IO stats */ +#define QUEUE_FLAG_IO_STAT 10 /* do disk/partitions IO accounting */ #define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */ #define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */ #define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */ @@ -699,7 +699,7 @@ struct request_queue { #define QUEUE_FLAG_FUA 21 /* device supports FUA writes */ #define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */ #define QUEUE_FLAG_DAX 23 /* device supports DAX */ -#define QUEUE_FLAG_STATS 24 /* track rq completion times */ +#define QUEUE_FLAG_STATS 24 /* track IO start and completion times */ #define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */ #define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */ #define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */ -- cgit v1.2.3-71-gd317 From 49d92c0dd64ae769c2d67fe27ac260ae31259ba6 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Thu, 4 Oct 2018 15:27:41 -0600 Subject: block: Add PCI P2P flag for request queue Add QUEUE_FLAG_PCI_P2P, meaning a driver's request queue supports targeting P2P memory. This will be used by P2P providers and orchestrators (in subsequent patches) to ensure block devices can support P2P memory before submitting P2P-backed pages to submit_bio(). Signed-off-by: Logan Gunthorpe Signed-off-by: Bjorn Helgaas Reviewed-by: Christoph Hellwig Acked-by: Jens Axboe --- include/linux/blkdev.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6980014357d4..c32f7171899b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -699,6 +699,7 @@ struct request_queue { #define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */ #define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */ #define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */ +#define QUEUE_FLAG_PCI_P2PDMA 30 /* device supports PCI p2p requests */ #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_SAME_COMP) | \ @@ -731,6 +732,8 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q); #define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags) #define blk_queue_scsi_passthrough(q) \ test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags) +#define blk_queue_pci_p2pdma(q) \ + test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags) #define blk_noretry_request(rq) \ ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ -- cgit v1.2.3-71-gd317 From a91e138022bc29b5d2bbc56b41de3e0db6261e28 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Fri, 12 Oct 2018 19:08:43 +0900 Subject: block: Introduce blkdev_nr_zones() helper Introduce the blkdev_nr_zones() helper function to get the total number of zones of a zoned block device. This number is always 0 for a regular block device (q->limits.zoned == BLK_ZONED_NONE case). Replace hard-coded number of zones calculation in dmz_get_zoned_device() with a call to this helper. Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/blk-zoned.c | 27 +++++++++++++++++++++++++++ drivers/md/dm-zoned-target.c | 3 +-- include/linux/blkdev.h | 5 +++++ 3 files changed, 33 insertions(+), 2 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-zoned.c b/block/blk-zoned.c index c461cf63f1f4..32e377f755d8 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -63,6 +63,33 @@ void __blk_req_zone_write_unlock(struct request *rq) } EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock); +static inline unsigned int __blkdev_nr_zones(struct request_queue *q, + sector_t nr_sectors) +{ + unsigned long zone_sectors = blk_queue_zone_sectors(q); + + return (nr_sectors + zone_sectors - 1) >> ilog2(zone_sectors); +} + +/** + * blkdev_nr_zones - Get number of zones + * @bdev: Target block device + * + * Description: + * Return the total number of zones of a zoned block device. + * For a regular block device, the number of zones is always 0. + */ +unsigned int blkdev_nr_zones(struct block_device *bdev) +{ + struct request_queue *q = bdev_get_queue(bdev); + + if (!blk_queue_is_zoned(q)) + return 0; + + return __blkdev_nr_zones(q, bdev->bd_part->nr_sects); +} +EXPORT_SYMBOL_GPL(blkdev_nr_zones); + /* * Check that a zone report belongs to the partition. * If yes, fix its start sector and write pointer, copy it in the diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c index a44183ff4be0..12d96a263623 100644 --- a/drivers/md/dm-zoned-target.c +++ b/drivers/md/dm-zoned-target.c @@ -702,8 +702,7 @@ static int dmz_get_zoned_device(struct dm_target *ti, char *path) dev->zone_nr_blocks = dmz_sect2blk(dev->zone_nr_sectors); dev->zone_nr_blocks_shift = ilog2(dev->zone_nr_blocks); - dev->nr_zones = (dev->capacity + dev->zone_nr_sectors - 1) - >> dev->zone_nr_sectors_shift; + dev->nr_zones = blkdev_nr_zones(dev->bdev); dmz->dev = dev; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 7d423721b327..ca5fdc1b7745 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -401,6 +401,7 @@ struct blk_zone_report_hdr { u8 padding[60]; }; +extern unsigned int blkdev_nr_zones(struct block_device *bdev); extern int blkdev_report_zones(struct block_device *bdev, sector_t sector, struct blk_zone *zones, unsigned int *nr_zones, gfp_t gfp_mask); @@ -414,6 +415,10 @@ extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode, #else /* CONFIG_BLK_DEV_ZONED */ +static inline unsigned int blkdev_nr_zones(struct block_device *bdev) +{ + return 0; +} static inline int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) -- cgit v1.2.3-71-gd317 From 965b652e901886ea54f93c60027b5be76328d958 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Fri, 12 Oct 2018 19:08:48 +0900 Subject: block: Expose queue nr_zones in sysfs Expose through sysfs the nr_zones field of struct request_queue. Exposing this value helps in debugging disk issues as well as facilitating scripts based use of the disk (e.g. blktests). For zoned block devices, the nr_zones field indicates the total number of zones of the device calculated using the known disk capacity and zone size. This number of zones is always 0 for regular block devices. Since nr_zones is defined conditionally with CONFIG_BLK_DEV_ZONED, introduce the blk_queue_nr_zones() function to return the correct value for any device, regardless if CONFIG_BLK_DEV_ZONED is set. Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/blk-sysfs.c | 11 +++++++++++ include/linux/blkdev.h | 10 ++++++++++ 2 files changed, 21 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 3772671cf2bc..92be8092ca4f 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -300,6 +300,11 @@ static ssize_t queue_zoned_show(struct request_queue *q, char *page) } } +static ssize_t queue_nr_zones_show(struct request_queue *q, char *page) +{ + return queue_var_show(blk_queue_nr_zones(q), page); +} + static ssize_t queue_nomerges_show(struct request_queue *q, char *page) { return queue_var_show((blk_queue_nomerges(q) << 1) | @@ -637,6 +642,11 @@ static struct queue_sysfs_entry queue_zoned_entry = { .show = queue_zoned_show, }; +static struct queue_sysfs_entry queue_nr_zones_entry = { + .attr = {.name = "nr_zones", .mode = 0444 }, + .show = queue_nr_zones_show, +}; + static struct queue_sysfs_entry queue_nomerges_entry = { .attr = {.name = "nomerges", .mode = 0644 }, .show = queue_nomerges_show, @@ -727,6 +737,7 @@ static struct attribute *default_attrs[] = { &queue_write_zeroes_max_entry.attr, &queue_nonrot_entry.attr, &queue_zoned_entry.attr, + &queue_nr_zones_entry.attr, &queue_nomerges_entry.attr, &queue_rq_affinity_entry.attr, &queue_iostats_entry.attr, diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index ca5fdc1b7745..6bb845f9601a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -811,6 +811,11 @@ static inline unsigned int blk_queue_zone_sectors(struct request_queue *q) } #ifdef CONFIG_BLK_DEV_ZONED +static inline unsigned int blk_queue_nr_zones(struct request_queue *q) +{ + return blk_queue_is_zoned(q) ? q->nr_zones : 0; +} + static inline unsigned int blk_queue_zone_no(struct request_queue *q, sector_t sector) { @@ -826,6 +831,11 @@ static inline bool blk_queue_zone_is_seq(struct request_queue *q, return false; return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap); } +#else /* CONFIG_BLK_DEV_ZONED */ +static inline unsigned int blk_queue_nr_zones(struct request_queue *q) +{ + return 0; +} #endif /* CONFIG_BLK_DEV_ZONED */ static inline bool rq_is_sync(struct request *rq) -- cgit v1.2.3-71-gd317 From e76239a3748c90a8b0e197f8f4544a8ce52f126e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 12 Oct 2018 19:08:49 +0900 Subject: block: add a report_zones method Dispatching a report zones command through the request queue is a major pain due to the command reply payload rewriting necessary. Given that blkdev_report_zones() is executing everything synchronously, implement report zones as a block device file operation instead, allowing major simplification of the code in many places. sd, null-blk, dm-linear and dm-flakey being the only block device drivers supporting exposing zoned block devices, these drivers are modified to provide the device side implementation of the report_zones() block device file operation. For device mappers, a new report_zones() target type operation is defined so that the upper block layer calls blkdev_report_zones() can be propagated down to the underlying devices of the dm targets. Implementation for this new operation is added to the dm-linear and dm-flakey targets. Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig [Damien] * Changed method block_device argument to gendisk * Various bug fixes and improvements * Added support for null_blk, dm-linear and dm-flakey. Reviewed-by: Martin K. Petersen Reviewed-by: Mike Snitzer Signed-off-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/blk-core.c | 1 - block/blk-mq-debugfs.c | 1 - block/blk-zoned.c | 164 +++++++++++++-------------------------- drivers/block/null_blk.h | 11 ++- drivers/block/null_blk_main.c | 23 +----- drivers/block/null_blk_zoned.c | 57 ++++---------- drivers/md/dm-flakey.c | 30 +++++--- drivers/md/dm-linear.c | 35 +++++---- drivers/md/dm.c | 169 ++++++++++++++++++++--------------------- drivers/scsi/sd.c | 13 +--- drivers/scsi/sd.h | 11 ++- drivers/scsi/sd_zbc.c | 153 ++++++++++++------------------------- include/linux/blk_types.h | 2 - include/linux/blkdev.h | 8 +- include/linux/device-mapper.h | 12 ++- include/trace/events/f2fs.h | 1 - 16 files changed, 266 insertions(+), 425 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-core.c b/block/blk-core.c index 3ed60723e242..bc6ea87d10e0 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -2300,7 +2300,6 @@ generic_make_request_checks(struct bio *bio) if (!q->limits.max_write_same_sectors) goto not_supported; break; - case REQ_OP_ZONE_REPORT: case REQ_OP_ZONE_RESET: if (!blk_queue_is_zoned(q)) goto not_supported; diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 41b86f50d126..10b284a1f18d 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -283,7 +283,6 @@ static const char *const op_name[] = { REQ_OP_NAME(WRITE), REQ_OP_NAME(FLUSH), REQ_OP_NAME(DISCARD), - REQ_OP_NAME(ZONE_REPORT), REQ_OP_NAME(SECURE_ERASE), REQ_OP_NAME(ZONE_RESET), REQ_OP_NAME(WRITE_SAME), diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 5d967fd39fbd..90cf503091d5 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -93,13 +93,10 @@ unsigned int blkdev_nr_zones(struct block_device *bdev) EXPORT_SYMBOL_GPL(blkdev_nr_zones); /* - * Check that a zone report belongs to the partition. - * If yes, fix its start sector and write pointer, copy it in the - * zone information array and return true. Return false otherwise. + * Check that a zone report belongs to this partition, and if yes, fix its start + * sector and write pointer and return true. Return false otherwise. */ -static bool blkdev_report_zone(struct block_device *bdev, - struct blk_zone *rep, - struct blk_zone *zone) +static bool blkdev_report_zone(struct block_device *bdev, struct blk_zone *rep) { sector_t offset = get_start_sect(bdev); @@ -114,11 +111,36 @@ static bool blkdev_report_zone(struct block_device *bdev, rep->wp = rep->start + rep->len; else rep->wp -= offset; - memcpy(zone, rep, sizeof(struct blk_zone)); - return true; } +static int blk_report_zones(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask) +{ + struct request_queue *q = disk->queue; + unsigned int z = 0, n, nrz = *nr_zones; + sector_t capacity = get_capacity(disk); + int ret; + + while (z < nrz && sector < capacity) { + n = nrz - z; + ret = disk->fops->report_zones(disk, sector, &zones[z], &n, + gfp_mask); + if (ret) + return ret; + if (!n) + break; + sector += blk_queue_zone_sectors(q) * n; + z += n; + } + + WARN_ON(z > *nr_zones); + *nr_zones = z; + + return 0; +} + /** * blkdev_report_zones - Get zones information * @bdev: Target block device @@ -133,130 +155,46 @@ static bool blkdev_report_zone(struct block_device *bdev, * requested by @nr_zones. The number of zones actually reported is * returned in @nr_zones. */ -int blkdev_report_zones(struct block_device *bdev, - sector_t sector, - struct blk_zone *zones, - unsigned int *nr_zones, +int blkdev_report_zones(struct block_device *bdev, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, gfp_t gfp_mask) { struct request_queue *q = bdev_get_queue(bdev); - struct blk_zone_report_hdr *hdr; - unsigned int nrz = *nr_zones; - struct page *page; - unsigned int nr_rep; - size_t rep_bytes; - unsigned int nr_pages; - struct bio *bio; - struct bio_vec *bv; - unsigned int i, n, nz; - unsigned int ofst; - void *addr; + unsigned int i, nrz; int ret; - if (!q) - return -ENXIO; - if (!blk_queue_is_zoned(q)) return -EOPNOTSUPP; - if (!nrz) - return 0; - - if (sector > bdev->bd_part->nr_sects) { - *nr_zones = 0; - return 0; - } - /* - * The zone report has a header. So make room for it in the - * payload. Also make sure that the report fits in a single BIO - * that will not be split down the stack. + * A block device that advertized itself as zoned must have a + * report_zones method. If it does not have one defined, the device + * driver has a bug. So warn about that. */ - rep_bytes = sizeof(struct blk_zone_report_hdr) + - sizeof(struct blk_zone) * nrz; - rep_bytes = (rep_bytes + PAGE_SIZE - 1) & PAGE_MASK; - if (rep_bytes > (queue_max_sectors(q) << 9)) - rep_bytes = queue_max_sectors(q) << 9; - - nr_pages = min_t(unsigned int, BIO_MAX_PAGES, - rep_bytes >> PAGE_SHIFT); - nr_pages = min_t(unsigned int, nr_pages, - queue_max_segments(q)); - - bio = bio_alloc(gfp_mask, nr_pages); - if (!bio) - return -ENOMEM; + if (WARN_ON_ONCE(!bdev->bd_disk->fops->report_zones)) + return -EOPNOTSUPP; - bio_set_dev(bio, bdev); - bio->bi_iter.bi_sector = blk_zone_start(q, sector); - bio_set_op_attrs(bio, REQ_OP_ZONE_REPORT, 0); - - for (i = 0; i < nr_pages; i++) { - page = alloc_page(gfp_mask); - if (!page) { - ret = -ENOMEM; - goto out; - } - if (!bio_add_page(bio, page, PAGE_SIZE, 0)) { - __free_page(page); - break; - } + if (!*nr_zones || sector >= bdev->bd_part->nr_sects) { + *nr_zones = 0; + return 0; } - if (i == 0) - ret = -ENOMEM; - else - ret = submit_bio_wait(bio); + nrz = min(*nr_zones, + __blkdev_nr_zones(q, bdev->bd_part->nr_sects - sector)); + ret = blk_report_zones(bdev->bd_disk, get_start_sect(bdev) + sector, + zones, &nrz, gfp_mask); if (ret) - goto out; - - /* - * Process the report result: skip the header and go through the - * reported zones to fixup and fixup the zone information for - * partitions. At the same time, return the zone information into - * the zone array. - */ - n = 0; - nz = 0; - nr_rep = 0; - bio_for_each_segment_all(bv, bio, i) { + return ret; - if (!bv->bv_page) + for (i = 0; i < nrz; i++) { + if (!blkdev_report_zone(bdev, zones)) break; - - addr = kmap_atomic(bv->bv_page); - - /* Get header in the first page */ - ofst = 0; - if (!nr_rep) { - hdr = addr; - nr_rep = hdr->nr_zones; - ofst = sizeof(struct blk_zone_report_hdr); - } - - /* Fixup and report zones */ - while (ofst < bv->bv_len && - n < nr_rep && nz < nrz) { - if (blkdev_report_zone(bdev, addr + ofst, &zones[nz])) - nz++; - ofst += sizeof(struct blk_zone); - n++; - } - - kunmap_atomic(addr); - - if (n >= nr_rep || nz >= nrz) - break; - + zones++; } - *nr_zones = nz; -out: - bio_for_each_segment_all(bv, bio, i) - __free_page(bv->bv_page); - bio_put(bio); + *nr_zones = i; - return ret; + return 0; } EXPORT_SYMBOL_GPL(blkdev_report_zones); diff --git a/drivers/block/null_blk.h b/drivers/block/null_blk.h index 34e0030f0592..7685df43f1ef 100644 --- a/drivers/block/null_blk.h +++ b/drivers/block/null_blk.h @@ -87,7 +87,9 @@ struct nullb { #ifdef CONFIG_BLK_DEV_ZONED int null_zone_init(struct nullb_device *dev); void null_zone_exit(struct nullb_device *dev); -blk_status_t null_zone_report(struct nullb *nullb, struct bio *bio); +int null_zone_report(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask); void null_zone_write(struct nullb_cmd *cmd, sector_t sector, unsigned int nr_sectors); void null_zone_reset(struct nullb_cmd *cmd, sector_t sector); @@ -97,10 +99,11 @@ static inline int null_zone_init(struct nullb_device *dev) return -EINVAL; } static inline void null_zone_exit(struct nullb_device *dev) {} -static inline blk_status_t null_zone_report(struct nullb *nullb, - struct bio *bio) +static inline int null_zone_report(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, + unsigned int *nr_zones, gfp_t gfp_mask) { - return BLK_STS_NOTSUPP; + return -EOPNOTSUPP; } static inline void null_zone_write(struct nullb_cmd *cmd, sector_t sector, unsigned int nr_sectors) diff --git a/drivers/block/null_blk_main.c b/drivers/block/null_blk_main.c index e94591021682..5ba426dbf377 100644 --- a/drivers/block/null_blk_main.c +++ b/drivers/block/null_blk_main.c @@ -1129,34 +1129,12 @@ static void null_restart_queue_async(struct nullb *nullb) blk_mq_start_stopped_hw_queues(q, true); } -static bool cmd_report_zone(struct nullb *nullb, struct nullb_cmd *cmd) -{ - struct nullb_device *dev = cmd->nq->dev; - - if (dev->queue_mode == NULL_Q_BIO) { - if (bio_op(cmd->bio) == REQ_OP_ZONE_REPORT) { - cmd->error = null_zone_report(nullb, cmd->bio); - return true; - } - } else { - if (req_op(cmd->rq) == REQ_OP_ZONE_REPORT) { - cmd->error = null_zone_report(nullb, cmd->rq->bio); - return true; - } - } - - return false; -} - static blk_status_t null_handle_cmd(struct nullb_cmd *cmd) { struct nullb_device *dev = cmd->nq->dev; struct nullb *nullb = dev->nullb; int err = 0; - if (cmd_report_zone(nullb, cmd)) - goto out; - if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) { struct request *rq = cmd->rq; @@ -1443,6 +1421,7 @@ static const struct block_device_operations null_fops = { .owner = THIS_MODULE, .open = null_open, .release = null_release, + .report_zones = null_zone_report, }; static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) diff --git a/drivers/block/null_blk_zoned.c b/drivers/block/null_blk_zoned.c index 7c6b86d98700..c0b0e4a3fa8f 100644 --- a/drivers/block/null_blk_zoned.c +++ b/drivers/block/null_blk_zoned.c @@ -48,54 +48,27 @@ void null_zone_exit(struct nullb_device *dev) kvfree(dev->zones); } -static void null_zone_fill_bio(struct nullb_device *dev, struct bio *bio, - unsigned int zno, unsigned int nr_zones) +int null_zone_report(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask) { - struct blk_zone_report_hdr *hdr = NULL; - struct bio_vec bvec; - struct bvec_iter iter; - void *addr; - unsigned int zones_to_cpy; - - bio_for_each_segment(bvec, bio, iter) { - addr = kmap_atomic(bvec.bv_page); - - zones_to_cpy = bvec.bv_len / sizeof(struct blk_zone); - - if (!hdr) { - hdr = (struct blk_zone_report_hdr *)addr; - hdr->nr_zones = nr_zones; - zones_to_cpy--; - addr += sizeof(struct blk_zone_report_hdr); - } - - zones_to_cpy = min_t(unsigned int, zones_to_cpy, nr_zones); - - memcpy(addr, &dev->zones[zno], - zones_to_cpy * sizeof(struct blk_zone)); - - kunmap_atomic(addr); + struct nullb *nullb = disk->private_data; + struct nullb_device *dev = nullb->dev; + unsigned int zno, nrz = 0; - nr_zones -= zones_to_cpy; - zno += zones_to_cpy; + if (!dev->zoned) + /* Not a zoned null device */ + return -EOPNOTSUPP; - if (!nr_zones) - break; + zno = null_zone_no(dev, sector); + if (zno < dev->nr_zones) { + nrz = min_t(unsigned int, *nr_zones, dev->nr_zones - zno); + memcpy(zones, &dev->zones[zno], nrz * sizeof(struct blk_zone)); } -} -blk_status_t null_zone_report(struct nullb *nullb, struct bio *bio) -{ - struct nullb_device *dev = nullb->dev; - unsigned int zno = null_zone_no(dev, bio->bi_iter.bi_sector); - unsigned int nr_zones = dev->nr_zones - zno; - unsigned int max_zones; + *nr_zones = nrz; - max_zones = (bio->bi_iter.bi_size / sizeof(struct blk_zone)) - 1; - nr_zones = min_t(unsigned int, nr_zones, max_zones); - null_zone_fill_bio(nullb->dev, bio, zno, nr_zones); - - return BLK_STS_OK; + return 0; } void null_zone_write(struct nullb_cmd *cmd, sector_t sector, diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index 32aabe27b37c..3cb97fa4c11d 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c @@ -315,10 +315,6 @@ static int flakey_map(struct dm_target *ti, struct bio *bio) if (bio_op(bio) == REQ_OP_ZONE_RESET) goto map_bio; - /* We need to remap reported zones, so remember the BIO iter */ - if (bio_op(bio) == REQ_OP_ZONE_REPORT) - goto map_bio; - /* Are we alive ? */ elapsed = (jiffies - fc->start_time) / HZ; if (elapsed % (fc->up_interval + fc->down_interval) >= fc->up_interval) { @@ -380,11 +376,6 @@ static int flakey_end_io(struct dm_target *ti, struct bio *bio, if (bio_op(bio) == REQ_OP_ZONE_RESET) return DM_ENDIO_DONE; - if (bio_op(bio) == REQ_OP_ZONE_REPORT) { - dm_remap_zone_report(ti, bio, fc->start); - return DM_ENDIO_DONE; - } - if (!*error && pb->bio_submitted && (bio_data_dir(bio) == READ)) { if (fc->corrupt_bio_byte && (fc->corrupt_bio_rw == READ) && all_corrupt_bio_flags_match(bio, fc)) { @@ -457,6 +448,26 @@ static int flakey_prepare_ioctl(struct dm_target *ti, struct block_device **bdev return 0; } +#ifdef CONFIG_BLK_DEV_ZONED +static int flakey_report_zones(struct dm_target *ti, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask) +{ + struct flakey_c *fc = ti->private; + int ret; + + /* Do report and remap it */ + ret = blkdev_report_zones(fc->dev->bdev, flakey_map_sector(ti, sector), + zones, nr_zones, gfp_mask); + if (ret != 0) + return ret; + + if (*nr_zones) + dm_remap_zone_report(ti, fc->start, zones, nr_zones); + return 0; +} +#endif + static int flakey_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { struct flakey_c *fc = ti->private; @@ -469,6 +480,7 @@ static struct target_type flakey_target = { .version = {1, 5, 0}, #ifdef CONFIG_BLK_DEV_ZONED .features = DM_TARGET_ZONED_HM, + .report_zones = flakey_report_zones, #endif .module = THIS_MODULE, .ctr = flakey_ctr, diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 2f7c44a006c4..8d7ddee6ac4d 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c @@ -102,19 +102,6 @@ static int linear_map(struct dm_target *ti, struct bio *bio) return DM_MAPIO_REMAPPED; } -#ifdef CONFIG_BLK_DEV_ZONED -static int linear_end_io(struct dm_target *ti, struct bio *bio, - blk_status_t *error) -{ - struct linear_c *lc = ti->private; - - if (!*error && bio_op(bio) == REQ_OP_ZONE_REPORT) - dm_remap_zone_report(ti, bio, lc->start); - - return DM_ENDIO_DONE; -} -#endif - static void linear_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { @@ -148,6 +135,26 @@ static int linear_prepare_ioctl(struct dm_target *ti, struct block_device **bdev return 0; } +#ifdef CONFIG_BLK_DEV_ZONED +static int linear_report_zones(struct dm_target *ti, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask) +{ + struct linear_c *lc = (struct linear_c *) ti->private; + int ret; + + /* Do report and remap it */ + ret = blkdev_report_zones(lc->dev->bdev, linear_map_sector(ti, sector), + zones, nr_zones, gfp_mask); + if (ret != 0) + return ret; + + if (*nr_zones) + dm_remap_zone_report(ti, lc->start, zones, nr_zones); + return 0; +} +#endif + static int linear_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) { @@ -211,8 +218,8 @@ static struct target_type linear_target = { .name = "linear", .version = {1, 4, 0}, #ifdef CONFIG_BLK_DEV_ZONED - .end_io = linear_end_io, .features = DM_TARGET_PASSES_INTEGRITY | DM_TARGET_ZONED_HM, + .report_zones = linear_report_zones, #else .features = DM_TARGET_PASSES_INTEGRITY, #endif diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 45abb54037fc..6be21dc210a1 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -458,6 +458,57 @@ static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) return dm_get_geometry(md, geo); } +static int dm_blk_report_zones(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask) +{ +#ifdef CONFIG_BLK_DEV_ZONED + struct mapped_device *md = disk->private_data; + struct dm_target *tgt; + struct dm_table *map; + int srcu_idx, ret; + + if (dm_suspended_md(md)) + return -EAGAIN; + + map = dm_get_live_table(md, &srcu_idx); + if (!map) + return -EIO; + + tgt = dm_table_find_target(map, sector); + if (!dm_target_is_valid(tgt)) { + ret = -EIO; + goto out; + } + + /* + * If we are executing this, we already know that the block device + * is a zoned device and so each target should have support for that + * type of drive. A missing report_zones method means that the target + * driver has a problem. + */ + if (WARN_ON(!tgt->type->report_zones)) { + ret = -EIO; + goto out; + } + + /* + * blkdev_report_zones() will loop and call this again to cover all the + * zones of the target, eventually moving on to the next target. + * So there is no need to loop here trying to fill the entire array + * of zones. + */ + ret = tgt->type->report_zones(tgt, sector, zones, + nr_zones, gfp_mask); + +out: + dm_put_live_table(md, srcu_idx); + return ret; +#else + return -ENOTSUPP; +#endif +} + static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, struct block_device **bdev) __acquires(md->io_barrier) @@ -1155,93 +1206,49 @@ void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) EXPORT_SYMBOL_GPL(dm_accept_partial_bio); /* - * The zone descriptors obtained with a zone report indicate zone positions - * within the target backing device, regardless of that device is a partition - * and regardless of the target mapping start sector on the device or partition. - * The zone descriptors start sector and write pointer position must be adjusted - * to match their relative position within the dm device. - * A target may call dm_remap_zone_report() after completion of a - * REQ_OP_ZONE_REPORT bio to remap the zone descriptors obtained from the - * backing device. + * The zone descriptors obtained with a zone report indicate + * zone positions within the underlying device of the target. The zone + * descriptors must be remapped to match their position within the dm device. + * The caller target should obtain the zones information using + * blkdev_report_zones() to ensure that remapping for partition offset is + * already handled. */ -void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, sector_t start) +void dm_remap_zone_report(struct dm_target *ti, sector_t start, + struct blk_zone *zones, unsigned int *nr_zones) { #ifdef CONFIG_BLK_DEV_ZONED - struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); - struct bio *report_bio = tio->io->orig_bio; - struct blk_zone_report_hdr *hdr = NULL; struct blk_zone *zone; - unsigned int nr_rep = 0; - unsigned int ofst; - sector_t part_offset; - struct bio_vec bvec; - struct bvec_iter iter; - void *addr; - - if (bio->bi_status) - return; - - /* - * bio sector was incremented by the request size on completion. Taking - * into account the original request sector, the target start offset on - * the backing device and the target mapping offset (ti->begin), the - * start sector of the backing device. The partition offset is always 0 - * if the target uses a whole device. - */ - part_offset = bio->bi_iter.bi_sector + ti->begin - (start + bio_end_sector(report_bio)); + unsigned int nrz = *nr_zones; + int i; /* - * Remap the start sector of the reported zones. For sequential zones, - * also remap the write pointer position. + * Remap the start sector and write pointer position of the zones in + * the array. Since we may have obtained from the target underlying + * device more zones that the target size, also adjust the number + * of zones. */ - bio_for_each_segment(bvec, report_bio, iter) { - addr = kmap_atomic(bvec.bv_page); - - /* Remember the report header in the first page */ - if (!hdr) { - hdr = addr; - ofst = sizeof(struct blk_zone_report_hdr); - } else - ofst = 0; - - /* Set zones start sector */ - while (hdr->nr_zones && ofst < bvec.bv_len) { - zone = addr + ofst; - zone->start -= part_offset; - if (zone->start >= start + ti->len) { - hdr->nr_zones = 0; - break; - } - zone->start = zone->start + ti->begin - start; - if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) { - if (zone->cond == BLK_ZONE_COND_FULL) - zone->wp = zone->start + zone->len; - else if (zone->cond == BLK_ZONE_COND_EMPTY) - zone->wp = zone->start; - else - zone->wp = zone->wp + ti->begin - start - part_offset; - } - ofst += sizeof(struct blk_zone); - hdr->nr_zones--; - nr_rep++; + for (i = 0; i < nrz; i++) { + zone = zones + i; + if (zone->start >= start + ti->len) { + memset(zone, 0, sizeof(struct blk_zone) * (nrz - i)); + break; } - if (addr != hdr) - kunmap_atomic(addr); + zone->start = zone->start + ti->begin - start; + if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) + continue; - if (!hdr->nr_zones) - break; - } - - if (hdr) { - hdr->nr_zones = nr_rep; - kunmap_atomic(hdr); + if (zone->cond == BLK_ZONE_COND_FULL) + zone->wp = zone->start + zone->len; + else if (zone->cond == BLK_ZONE_COND_EMPTY) + zone->wp = zone->start; + else + zone->wp = zone->wp + ti->begin - start; } - bio_advance(report_bio, report_bio->bi_iter.bi_size); - + *nr_zones = i; #else /* !CONFIG_BLK_DEV_ZONED */ - bio->bi_status = BLK_STS_NOTSUPP; + *nr_zones = 0; #endif } EXPORT_SYMBOL_GPL(dm_remap_zone_report); @@ -1327,8 +1334,7 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, return r; } - if (bio_op(bio) != REQ_OP_ZONE_REPORT) - bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); + bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); clone->bi_iter.bi_size = to_bytes(len); if (unlikely(bio_integrity(bio) != NULL)) @@ -1541,7 +1547,6 @@ static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti, */ static int __split_and_process_non_flush(struct clone_info *ci) { - struct bio *bio = ci->bio; struct dm_target *ti; unsigned len; int r; @@ -1553,11 +1558,7 @@ static int __split_and_process_non_flush(struct clone_info *ci) if (unlikely(__process_abnormal_io(ci, ti, &r))) return r; - if (bio_op(bio) == REQ_OP_ZONE_REPORT) - len = ci->sector_count; - else - len = min_t(sector_t, max_io_len(ci->sector, ti), - ci->sector_count); + len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count); r = __clone_and_map_data_bio(ci, ti, ci->sector, &len); if (r < 0) @@ -1616,9 +1617,6 @@ static blk_qc_t __split_and_process_bio(struct mapped_device *md, * We take a clone of the original to store in * ci.io->orig_bio to be used by end_io_acct() and * for dec_pending to use for completion handling. - * As this path is not used for REQ_OP_ZONE_REPORT, - * the usage of io->orig_bio in dm_remap_zone_report() - * won't be affected by this reassignment. */ struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count, GFP_NOIO, &md->queue->bio_split); @@ -3167,6 +3165,7 @@ static const struct block_device_operations dm_blk_dops = { .release = dm_blk_close, .ioctl = dm_blk_ioctl, .getgeo = dm_blk_getgeo, + .report_zones = dm_blk_report_zones, .pr_ops = &dm_pr_ops, .owner = THIS_MODULE }; diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index b762d0fd773c..42c0f299021d 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -1272,8 +1272,6 @@ static int sd_init_command(struct scsi_cmnd *cmd) case REQ_OP_READ: case REQ_OP_WRITE: return sd_setup_read_write_cmnd(cmd); - case REQ_OP_ZONE_REPORT: - return sd_zbc_setup_report_cmnd(cmd); case REQ_OP_ZONE_RESET: return sd_zbc_setup_reset_cmnd(cmd); default: @@ -1802,6 +1800,7 @@ static const struct block_device_operations sd_fops = { .check_events = sd_check_events, .revalidate_disk = sd_revalidate_disk, .unlock_native_capacity = sd_unlock_native_capacity, + .report_zones = sd_zbc_report_zones, .pr_ops = &sd_pr_ops, }; @@ -1953,16 +1952,6 @@ static int sd_done(struct scsi_cmnd *SCpnt) scsi_set_resid(SCpnt, blk_rq_bytes(req)); } break; - case REQ_OP_ZONE_REPORT: - if (!result) { - good_bytes = scsi_bufflen(SCpnt) - - scsi_get_resid(SCpnt); - scsi_set_resid(SCpnt, 0); - } else { - good_bytes = 0; - scsi_set_resid(SCpnt, blk_rq_bytes(req)); - } - break; default: /* * In case of bogus fw or device, we could end up having diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index a7d4f50b67d4..f72f20fd0d8b 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -273,10 +273,12 @@ static inline int sd_is_zoned(struct scsi_disk *sdkp) extern int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buffer); extern void sd_zbc_remove(struct scsi_disk *sdkp); extern void sd_zbc_print_zones(struct scsi_disk *sdkp); -extern int sd_zbc_setup_report_cmnd(struct scsi_cmnd *cmd); extern int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd); extern void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, struct scsi_sense_hdr *sshdr); +extern int sd_zbc_report_zones(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask); #else /* CONFIG_BLK_DEV_ZONED */ @@ -290,11 +292,6 @@ static inline void sd_zbc_remove(struct scsi_disk *sdkp) {} static inline void sd_zbc_print_zones(struct scsi_disk *sdkp) {} -static inline int sd_zbc_setup_report_cmnd(struct scsi_cmnd *cmd) -{ - return BLKPREP_INVALID; -} - static inline int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd) { return BLKPREP_INVALID; @@ -304,6 +301,8 @@ static inline void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, struct scsi_sense_hdr *sshdr) {} +#define sd_zbc_report_zones NULL + #endif /* CONFIG_BLK_DEV_ZONED */ #endif /* _SCSI_DISK_H */ diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 0678e1e108b0..0f2cfc81fce3 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -62,7 +62,7 @@ static void sd_zbc_parse_report(struct scsi_disk *sdkp, u8 *buf, } /** - * sd_zbc_report_zones - Issue a REPORT ZONES scsi command. + * sd_zbc_do_report_zones - Issue a REPORT ZONES scsi command. * @sdkp: The target disk * @buf: Buffer to use for the reply * @buflen: the buffer size @@ -75,9 +75,9 @@ static void sd_zbc_parse_report(struct scsi_disk *sdkp, u8 *buf, * zones and will only report the count of zones fitting in the command reply * buffer. */ -static int sd_zbc_report_zones(struct scsi_disk *sdkp, unsigned char *buf, - unsigned int buflen, sector_t lba, - bool partial) +static int sd_zbc_do_report_zones(struct scsi_disk *sdkp, unsigned char *buf, + unsigned int buflen, sector_t lba, + bool partial) { struct scsi_device *sdp = sdkp->device; const int timeout = sdp->request_queue->rq_timeout; @@ -118,108 +118,56 @@ static int sd_zbc_report_zones(struct scsi_disk *sdkp, unsigned char *buf, } /** - * sd_zbc_setup_report_cmnd - Prepare a REPORT ZONES scsi command - * @cmd: The command to setup + * sd_zbc_report_zones - Disk report zones operation. + * @disk: The target disk + * @sector: Start 512B sector of the report + * @zones: Array of zone descriptors + * @nr_zones: Number of descriptors in the array + * @gfp_mask: Memory allocation mask * - * Call in sd_init_command() for a REQ_OP_ZONE_REPORT request. + * Execute a report zones command on the target disk. */ -int sd_zbc_setup_report_cmnd(struct scsi_cmnd *cmd) +int sd_zbc_report_zones(struct gendisk *disk, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask) { - struct request *rq = cmd->request; - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); - sector_t lba, sector = blk_rq_pos(rq); - unsigned int nr_bytes = blk_rq_bytes(rq); - int ret; - - WARN_ON(nr_bytes == 0); + struct scsi_disk *sdkp = scsi_disk(disk); + unsigned int i, buflen, nrz = *nr_zones; + unsigned char *buf; + size_t offset = 0; + int ret = 0; if (!sd_is_zoned(sdkp)) /* Not a zoned device */ - return BLKPREP_KILL; - - ret = scsi_init_io(cmd); - if (ret != BLKPREP_OK) - return ret; - - cmd->cmd_len = 16; - memset(cmd->cmnd, 0, cmd->cmd_len); - cmd->cmnd[0] = ZBC_IN; - cmd->cmnd[1] = ZI_REPORT_ZONES; - lba = sectors_to_logical(sdkp->device, sector); - put_unaligned_be64(lba, &cmd->cmnd[2]); - put_unaligned_be32(nr_bytes, &cmd->cmnd[10]); - /* Do partial report for speeding things up */ - cmd->cmnd[14] = ZBC_REPORT_ZONE_PARTIAL; - - cmd->sc_data_direction = DMA_FROM_DEVICE; - cmd->sdb.length = nr_bytes; - cmd->transfersize = sdkp->device->sector_size; - cmd->allowed = 0; + return -EOPNOTSUPP; - return BLKPREP_OK; -} - -/** - * sd_zbc_report_zones_complete - Process a REPORT ZONES scsi command reply. - * @scmd: The completed report zones command - * @good_bytes: reply size in bytes - * - * Convert all reported zone descriptors to struct blk_zone. The conversion - * is done in-place, directly in the request specified sg buffer. - */ -static void sd_zbc_report_zones_complete(struct scsi_cmnd *scmd, - unsigned int good_bytes) -{ - struct request *rq = scmd->request; - struct scsi_disk *sdkp = scsi_disk(rq->rq_disk); - struct sg_mapping_iter miter; - struct blk_zone_report_hdr hdr; - struct blk_zone zone; - unsigned int offset, bytes = 0; - unsigned long flags; - u8 *buf; - - if (good_bytes < 64) - return; - - memset(&hdr, 0, sizeof(struct blk_zone_report_hdr)); - - sg_miter_start(&miter, scsi_sglist(scmd), scsi_sg_count(scmd), - SG_MITER_TO_SG | SG_MITER_ATOMIC); + /* + * Get a reply buffer for the number of requested zones plus a header. + * For ATA, buffers must be aligned to 512B. + */ + buflen = roundup((nrz + 1) * 64, 512); + buf = kmalloc(buflen, gfp_mask); + if (!buf) + return -ENOMEM; - local_irq_save(flags); - while (sg_miter_next(&miter) && bytes < good_bytes) { + ret = sd_zbc_do_report_zones(sdkp, buf, buflen, + sectors_to_logical(sdkp->device, sector), true); + if (ret) + goto out_free_buf; - buf = miter.addr; - offset = 0; + nrz = min(nrz, get_unaligned_be32(&buf[0]) / 64); + for (i = 0; i < nrz; i++) { + offset += 64; + sd_zbc_parse_report(sdkp, buf + offset, zones); + zones++; + } - if (bytes == 0) { - /* Set the report header */ - hdr.nr_zones = min_t(unsigned int, - (good_bytes - 64) / 64, - get_unaligned_be32(&buf[0]) / 64); - memcpy(buf, &hdr, sizeof(struct blk_zone_report_hdr)); - offset += 64; - bytes += 64; - } + *nr_zones = nrz; - /* Parse zone descriptors */ - while (offset < miter.length && hdr.nr_zones) { - WARN_ON(offset > miter.length); - buf = miter.addr + offset; - sd_zbc_parse_report(sdkp, buf, &zone); - memcpy(buf, &zone, sizeof(struct blk_zone)); - offset += 64; - bytes += 64; - hdr.nr_zones--; - } - - if (!hdr.nr_zones) - break; +out_free_buf: + kfree(buf); - } - sg_miter_stop(&miter); - local_irq_restore(flags); + return ret; } /** @@ -302,13 +250,6 @@ void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE_SAME: break; - - case REQ_OP_ZONE_REPORT: - - if (!result) - sd_zbc_report_zones_complete(cmd, good_bytes); - break; - } } @@ -390,7 +331,7 @@ static int sd_zbc_check_zones(struct scsi_disk *sdkp, u32 *zblocks) return -ENOMEM; /* Do a report zone to get max_lba and the same field */ - ret = sd_zbc_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, 0, false); + ret = sd_zbc_do_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, 0, false); if (ret) goto out_free; @@ -447,8 +388,8 @@ static int sd_zbc_check_zones(struct scsi_disk *sdkp, u32 *zblocks) } if (block < sdkp->capacity) { - ret = sd_zbc_report_zones(sdkp, buf, - SD_ZBC_BUF_SIZE, block, true); + ret = sd_zbc_do_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, + block, true); if (ret) goto out_free; } @@ -565,8 +506,8 @@ sd_zbc_setup_seq_zones_bitmap(struct scsi_disk *sdkp, u32 zone_shift, goto out; while (lba < sdkp->capacity) { - ret = sd_zbc_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, - lba, true); + ret = sd_zbc_do_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, lba, + true); if (ret) goto out; lba = sd_zbc_get_seq_zones(sdkp, buf, SD_ZBC_BUF_SIZE, diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 9578c7ab1eb6..093a818c5b68 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -283,8 +283,6 @@ enum req_opf { REQ_OP_FLUSH = 2, /* discard sectors */ REQ_OP_DISCARD = 3, - /* get zone information */ - REQ_OP_ZONE_REPORT = 4, /* securely erase sectors */ REQ_OP_SECURE_ERASE = 5, /* seset a zone write pointer */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6bb845f9601a..51fe6472ce02 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -396,11 +396,6 @@ struct queue_limits { #ifdef CONFIG_BLK_DEV_ZONED -struct blk_zone_report_hdr { - unsigned int nr_zones; - u8 padding[60]; -}; - extern unsigned int blkdev_nr_zones(struct block_device *bdev); extern int blkdev_report_zones(struct block_device *bdev, sector_t sector, struct blk_zone *zones, @@ -1867,6 +1862,9 @@ struct block_device_operations { int (*getgeo)(struct block_device *, struct hd_geometry *); /* this callback is with swap_lock and sometimes page table lock held */ void (*swap_slot_free_notify) (struct block_device *, unsigned long); + int (*report_zones)(struct gendisk *, sector_t sector, + struct blk_zone *zones, unsigned int *nr_zones, + gfp_t gfp_mask); struct module *owner; const struct pr_ops *pr_ops; }; diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 6fb0808e87c8..a23b396a8edc 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -92,6 +92,11 @@ typedef int (*dm_message_fn) (struct dm_target *ti, unsigned argc, char **argv, typedef int (*dm_prepare_ioctl_fn) (struct dm_target *ti, struct block_device **bdev); +typedef int (*dm_report_zones_fn) (struct dm_target *ti, sector_t sector, + struct blk_zone *zones, + unsigned int *nr_zones, + gfp_t gfp_mask); + /* * These iteration functions are typically used to check (and combine) * properties of underlying devices. @@ -180,6 +185,9 @@ struct target_type { dm_status_fn status; dm_message_fn message; dm_prepare_ioctl_fn prepare_ioctl; +#ifdef CONFIG_BLK_DEV_ZONED + dm_report_zones_fn report_zones; +#endif dm_busy_fn busy; dm_iterate_devices_fn iterate_devices; dm_io_hints_fn io_hints; @@ -420,8 +428,8 @@ struct gendisk *dm_disk(struct mapped_device *md); int dm_suspended(struct dm_target *ti); int dm_noflush_suspending(struct dm_target *ti); void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors); -void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, - sector_t start); +void dm_remap_zone_report(struct dm_target *ti, sector_t start, + struct blk_zone *zones, unsigned int *nr_zones); union map_info *dm_get_rq_mapinfo(struct request *rq); struct queue_limits *dm_get_queue_limits(struct mapped_device *md); diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 795698925d20..3ec73f17ee2a 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -82,7 +82,6 @@ TRACE_DEFINE_ENUM(CP_TRIMMED); { REQ_OP_WRITE, "WRITE" }, \ { REQ_OP_FLUSH, "FLUSH" }, \ { REQ_OP_DISCARD, "DISCARD" }, \ - { REQ_OP_ZONE_REPORT, "ZONE_REPORT" }, \ { REQ_OP_SECURE_ERASE, "SECURE_ERASE" }, \ { REQ_OP_ZONE_RESET, "ZONE_RESET" }, \ { REQ_OP_WRITE_SAME, "WRITE_SAME" }, \ -- cgit v1.2.3-71-gd317 From bf5054569653c491ece544cc7ee333ae53b47121 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Fri, 12 Oct 2018 19:08:50 +0900 Subject: block: Introduce blk_revalidate_disk_zones() Drivers exposing zoned block devices have to initialize and maintain correctness (i.e. revalidate) of the device zone bitmaps attached to the device request queue (seq_zones_bitmap and seq_zones_wlock). To simplify coding this, introduce a generic helper function blk_revalidate_disk_zones() suitable for most (and likely all) cases. This new function always update the seq_zones_bitmap and seq_zones_wlock bitmaps as well as the queue nr_zones field when called for a disk using a request based queue. For a disk using a BIO based queue, only the number of zones is updated since these queues do not have schedulers and so do not need the zone bitmaps. With this change, the zone bitmap initialization code in sd_zbc.c can be replaced with a call to this function in sd_zbc_read_zones(), which is called from the disk revalidate block operation method. A call to blk_revalidate_disk_zones() is also added to the null_blk driver for devices created with the zoned mode enabled. Finally, to ensure that zoned devices created with dm-linear or dm-flakey expose the correct number of zones through sysfs, a call to blk_revalidate_disk_zones() is added to dm_table_set_restrictions(). The zone bitmaps allocated and initialized with blk_revalidate_disk_zones() are freed automatically from __blk_release_queue() using the block internal function blk_queue_free_zone_bitmaps(). Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Reviewed-by: Mike Snitzer Signed-off-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/blk-sysfs.c | 2 + block/blk-zoned.c | 136 ++++++++++++++++++++++++++ block/blk.h | 6 ++ drivers/block/null_blk_main.c | 7 ++ drivers/md/dm-table.c | 10 ++ drivers/scsi/sd.c | 2 - drivers/scsi/sd.h | 4 - drivers/scsi/sd_zbc.c | 218 +++++------------------------------------- include/linux/blkdev.h | 7 ++ 9 files changed, 194 insertions(+), 198 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 92be8092ca4f..0641533597f1 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -852,6 +852,8 @@ static void __blk_release_queue(struct work_struct *work) if (q->queue_tags) __blk_queue_free_tags(q); + blk_queue_free_zone_bitmaps(q); + if (!q->mq_ops) { if (q->exit_rq_fn) q->exit_rq_fn(q, q->fq->flush_rq); diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 90cf503091d5..13ba2011a306 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "blk.h" @@ -359,3 +360,138 @@ int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode, return blkdev_reset_zones(bdev, zrange.sector, zrange.nr_sectors, GFP_KERNEL); } + +static inline unsigned long *blk_alloc_zone_bitmap(int node, + unsigned int nr_zones) +{ + return kcalloc_node(BITS_TO_LONGS(nr_zones), sizeof(unsigned long), + GFP_NOIO, node); +} + +/* + * Allocate an array of struct blk_zone to get nr_zones zone information. + * The allocated array may be smaller than nr_zones. + */ +static struct blk_zone *blk_alloc_zones(int node, unsigned int *nr_zones) +{ + size_t size = *nr_zones * sizeof(struct blk_zone); + struct page *page; + int order; + + for (order = get_order(size); order > 0; order--) { + page = alloc_pages_node(node, GFP_NOIO | __GFP_ZERO, order); + if (page) { + *nr_zones = min_t(unsigned int, *nr_zones, + (PAGE_SIZE << order) / sizeof(struct blk_zone)); + return page_address(page); + } + } + + return NULL; +} + +void blk_queue_free_zone_bitmaps(struct request_queue *q) +{ + kfree(q->seq_zones_bitmap); + q->seq_zones_bitmap = NULL; + kfree(q->seq_zones_wlock); + q->seq_zones_wlock = NULL; +} + +/** + * blk_revalidate_disk_zones - (re)allocate and initialize zone bitmaps + * @disk: Target disk + * + * Helper function for low-level device drivers to (re) allocate and initialize + * a disk request queue zone bitmaps. This functions should normally be called + * within the disk ->revalidate method. For BIO based queues, no zone bitmap + * is allocated. + */ +int blk_revalidate_disk_zones(struct gendisk *disk) +{ + struct request_queue *q = disk->queue; + unsigned int nr_zones = __blkdev_nr_zones(q, get_capacity(disk)); + unsigned long *seq_zones_wlock = NULL, *seq_zones_bitmap = NULL; + unsigned int i, rep_nr_zones = 0, z = 0, nrz; + struct blk_zone *zones = NULL; + sector_t sector = 0; + int ret = 0; + + /* + * BIO based queues do not use a scheduler so only q->nr_zones + * needs to be updated so that the sysfs exposed value is correct. + */ + if (!queue_is_rq_based(q)) { + q->nr_zones = nr_zones; + return 0; + } + + if (!blk_queue_is_zoned(q) || !nr_zones) { + nr_zones = 0; + goto update; + } + + /* Allocate bitmaps */ + ret = -ENOMEM; + seq_zones_wlock = blk_alloc_zone_bitmap(q->node, nr_zones); + if (!seq_zones_wlock) + goto out; + seq_zones_bitmap = blk_alloc_zone_bitmap(q->node, nr_zones); + if (!seq_zones_bitmap) + goto out; + + /* Get zone information and initialize seq_zones_bitmap */ + rep_nr_zones = nr_zones; + zones = blk_alloc_zones(q->node, &rep_nr_zones); + if (!zones) + goto out; + + while (z < nr_zones) { + nrz = min(nr_zones - z, rep_nr_zones); + ret = blk_report_zones(disk, sector, zones, &nrz, GFP_NOIO); + if (ret) + goto out; + if (!nrz) + break; + for (i = 0; i < nrz; i++) { + if (zones[i].type != BLK_ZONE_TYPE_CONVENTIONAL) + set_bit(z, seq_zones_bitmap); + z++; + } + sector += nrz * blk_queue_zone_sectors(q); + } + + if (WARN_ON(z != nr_zones)) { + ret = -EIO; + goto out; + } + +update: + /* + * Install the new bitmaps, making sure the queue is stopped and + * all I/Os are completed (i.e. a scheduler is not referencing the + * bitmaps). + */ + blk_mq_freeze_queue(q); + q->nr_zones = nr_zones; + swap(q->seq_zones_wlock, seq_zones_wlock); + swap(q->seq_zones_bitmap, seq_zones_bitmap); + blk_mq_unfreeze_queue(q); + +out: + free_pages((unsigned long)zones, + get_order(rep_nr_zones * sizeof(struct blk_zone))); + kfree(seq_zones_wlock); + kfree(seq_zones_bitmap); + + if (ret) { + pr_warn("%s: failed to revalidate zones\n", disk->disk_name); + blk_mq_freeze_queue(q); + blk_queue_free_zone_bitmaps(q); + blk_mq_unfreeze_queue(q); + } + + return ret; +} +EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones); + diff --git a/block/blk.h b/block/blk.h index 93574baaa6b8..a1841b8ff129 100644 --- a/block/blk.h +++ b/block/blk.h @@ -490,4 +490,10 @@ static inline int blk_iolatency_init(struct request_queue *q) { return 0; } struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp); +#ifdef CONFIG_BLK_DEV_ZONED +void blk_queue_free_zone_bitmaps(struct request_queue *q); +#else +static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {} +#endif + #endif /* BLK_INTERNAL_H */ diff --git a/drivers/block/null_blk_main.c b/drivers/block/null_blk_main.c index 5ba426dbf377..09339203dfba 100644 --- a/drivers/block/null_blk_main.c +++ b/drivers/block/null_blk_main.c @@ -1528,6 +1528,13 @@ static int null_gendisk_register(struct nullb *nullb) disk->queue = nullb->q; strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN); + if (nullb->dev->zoned) { + int ret = blk_revalidate_disk_zones(disk); + + if (ret != 0) + return ret; + } + add_disk(disk); return 0; } diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 3d0e2c198f06..fb4bea20657b 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1937,6 +1937,16 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, */ if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random)) blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q); + + /* + * For a zoned target, the number of zones should be updated for the + * correct value to be exposed in sysfs queue/nr_zones. For a BIO based + * target, this is all that is needed. For a request based target, the + * queue zone bitmaps must also be updated. + * Use blk_revalidate_disk_zones() to handle this. + */ + if (blk_queue_is_zoned(q)) + blk_revalidate_disk_zones(t->md->disk); } unsigned int dm_table_get_num_targets(struct dm_table *t) diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 42c0f299021d..3bb2b3351e35 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3414,8 +3414,6 @@ static int sd_remove(struct device *dev) del_gendisk(sdkp->disk); sd_shutdown(dev); - sd_zbc_remove(sdkp); - free_opal_dev(sdkp->opal_dev); blk_register_region(devt, SD_MINORS, NULL, diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index f72f20fd0d8b..1d63f3a23ffb 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -76,7 +76,6 @@ struct scsi_disk { #ifdef CONFIG_BLK_DEV_ZONED u32 nr_zones; u32 zone_blocks; - u32 zone_shift; u32 zones_optimal_open; u32 zones_optimal_nonseq; u32 zones_max_open; @@ -271,7 +270,6 @@ static inline int sd_is_zoned(struct scsi_disk *sdkp) #ifdef CONFIG_BLK_DEV_ZONED extern int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buffer); -extern void sd_zbc_remove(struct scsi_disk *sdkp); extern void sd_zbc_print_zones(struct scsi_disk *sdkp); extern int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd); extern void sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, @@ -288,8 +286,6 @@ static inline int sd_zbc_read_zones(struct scsi_disk *sdkp, return 0; } -static inline void sd_zbc_remove(struct scsi_disk *sdkp) {} - static inline void sd_zbc_print_zones(struct scsi_disk *sdkp) {} static inline int sd_zbc_setup_reset_cmnd(struct scsi_cmnd *cmd) diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 0f2cfc81fce3..e06c48c866e4 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -425,191 +425,10 @@ out_free: return ret; } -/** - * sd_zbc_alloc_zone_bitmap - Allocate a zone bitmap (one bit per zone). - * @nr_zones: Number of zones to allocate space for. - * @numa_node: NUMA node to allocate the memory from. - */ -static inline unsigned long * -sd_zbc_alloc_zone_bitmap(u32 nr_zones, int numa_node) -{ - return kcalloc_node(BITS_TO_LONGS(nr_zones), sizeof(unsigned long), - GFP_KERNEL, numa_node); -} - -/** - * sd_zbc_get_seq_zones - Parse report zones reply to identify sequential zones - * @sdkp: disk used - * @buf: report reply buffer - * @buflen: length of @buf - * @zone_shift: logarithm base 2 of the number of blocks in a zone - * @seq_zones_bitmap: bitmap of sequential zones to set - * - * Parse reported zone descriptors in @buf to identify sequential zones and - * set the reported zone bit in @seq_zones_bitmap accordingly. - * Since read-only and offline zones cannot be written, do not - * mark them as sequential in the bitmap. - * Return the LBA after the last zone reported. - */ -static sector_t sd_zbc_get_seq_zones(struct scsi_disk *sdkp, unsigned char *buf, - unsigned int buflen, u32 zone_shift, - unsigned long *seq_zones_bitmap) -{ - sector_t lba, next_lba = sdkp->capacity; - unsigned int buf_len, list_length; - unsigned char *rec; - u8 type, cond; - - list_length = get_unaligned_be32(&buf[0]) + 64; - buf_len = min(list_length, buflen); - rec = buf + 64; - - while (rec < buf + buf_len) { - type = rec[0] & 0x0f; - cond = (rec[1] >> 4) & 0xf; - lba = get_unaligned_be64(&rec[16]); - if (type != ZBC_ZONE_TYPE_CONV && - cond != ZBC_ZONE_COND_READONLY && - cond != ZBC_ZONE_COND_OFFLINE) - set_bit(lba >> zone_shift, seq_zones_bitmap); - next_lba = lba + get_unaligned_be64(&rec[8]); - rec += 64; - } - - return next_lba; -} - -/** - * sd_zbc_setup_seq_zones_bitmap - Initialize a seq zone bitmap. - * @sdkp: target disk - * @zone_shift: logarithm base 2 of the number of blocks in a zone - * @nr_zones: number of zones to set up a seq zone bitmap for - * - * Allocate a zone bitmap and initialize it by identifying sequential zones. - */ -static unsigned long * -sd_zbc_setup_seq_zones_bitmap(struct scsi_disk *sdkp, u32 zone_shift, - u32 nr_zones) -{ - struct request_queue *q = sdkp->disk->queue; - unsigned long *seq_zones_bitmap; - sector_t lba = 0; - unsigned char *buf; - int ret = -ENOMEM; - - seq_zones_bitmap = sd_zbc_alloc_zone_bitmap(nr_zones, q->node); - if (!seq_zones_bitmap) - return ERR_PTR(-ENOMEM); - - buf = kmalloc(SD_ZBC_BUF_SIZE, GFP_KERNEL); - if (!buf) - goto out; - - while (lba < sdkp->capacity) { - ret = sd_zbc_do_report_zones(sdkp, buf, SD_ZBC_BUF_SIZE, lba, - true); - if (ret) - goto out; - lba = sd_zbc_get_seq_zones(sdkp, buf, SD_ZBC_BUF_SIZE, - zone_shift, seq_zones_bitmap); - } - - if (lba != sdkp->capacity) { - /* Something went wrong */ - ret = -EIO; - } - -out: - kfree(buf); - if (ret) { - kfree(seq_zones_bitmap); - return ERR_PTR(ret); - } - return seq_zones_bitmap; -} - -static void sd_zbc_cleanup(struct scsi_disk *sdkp) -{ - struct request_queue *q = sdkp->disk->queue; - - kfree(q->seq_zones_bitmap); - q->seq_zones_bitmap = NULL; - - kfree(q->seq_zones_wlock); - q->seq_zones_wlock = NULL; - - q->nr_zones = 0; -} - -static int sd_zbc_setup(struct scsi_disk *sdkp, u32 zone_blocks) -{ - struct request_queue *q = sdkp->disk->queue; - u32 zone_shift = ilog2(zone_blocks); - u32 nr_zones; - int ret; - - /* chunk_sectors indicates the zone size */ - blk_queue_chunk_sectors(q, - logical_to_sectors(sdkp->device, zone_blocks)); - nr_zones = round_up(sdkp->capacity, zone_blocks) >> zone_shift; - - /* - * Initialize the device request queue information if the number - * of zones changed. - */ - if (nr_zones != sdkp->nr_zones || nr_zones != q->nr_zones) { - unsigned long *seq_zones_wlock = NULL, *seq_zones_bitmap = NULL; - size_t zone_bitmap_size; - - if (nr_zones) { - seq_zones_wlock = sd_zbc_alloc_zone_bitmap(nr_zones, - q->node); - if (!seq_zones_wlock) { - ret = -ENOMEM; - goto err; - } - - seq_zones_bitmap = sd_zbc_setup_seq_zones_bitmap(sdkp, - zone_shift, nr_zones); - if (IS_ERR(seq_zones_bitmap)) { - ret = PTR_ERR(seq_zones_bitmap); - kfree(seq_zones_wlock); - goto err; - } - } - zone_bitmap_size = BITS_TO_LONGS(nr_zones) * - sizeof(unsigned long); - blk_mq_freeze_queue(q); - if (q->nr_zones != nr_zones) { - /* READ16/WRITE16 is mandatory for ZBC disks */ - sdkp->device->use_16_for_rw = 1; - sdkp->device->use_10_for_rw = 0; - - sdkp->zone_blocks = zone_blocks; - sdkp->zone_shift = zone_shift; - sdkp->nr_zones = nr_zones; - q->nr_zones = nr_zones; - swap(q->seq_zones_wlock, seq_zones_wlock); - swap(q->seq_zones_bitmap, seq_zones_bitmap); - } else if (memcmp(q->seq_zones_bitmap, seq_zones_bitmap, - zone_bitmap_size) != 0) { - memcpy(q->seq_zones_bitmap, seq_zones_bitmap, - zone_bitmap_size); - } - blk_mq_unfreeze_queue(q); - kfree(seq_zones_wlock); - kfree(seq_zones_bitmap); - } - - return 0; - -err: - sd_zbc_cleanup(sdkp); - return ret; -} - int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buf) { + struct gendisk *disk = sdkp->disk; + unsigned int nr_zones; u32 zone_blocks; int ret; @@ -634,24 +453,39 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buf) goto err; /* The drive satisfies the kernel restrictions: set it up */ - ret = sd_zbc_setup(sdkp, zone_blocks); - if (ret) - goto err; + blk_queue_chunk_sectors(sdkp->disk->queue, + logical_to_sectors(sdkp->device, zone_blocks)); + nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); + + /* READ16/WRITE16 is mandatory for ZBC disks */ + sdkp->device->use_16_for_rw = 1; + sdkp->device->use_10_for_rw = 0; + + /* + * If something changed, revalidate the disk zone bitmaps once we have + * the capacity, that is on the second revalidate execution during disk + * scan and always during normal revalidate. + */ + if (sdkp->first_scan) + return 0; + if (sdkp->zone_blocks != zone_blocks || + sdkp->nr_zones != nr_zones || + disk->queue->nr_zones != nr_zones) { + ret = blk_revalidate_disk_zones(disk); + if (ret != 0) + goto err; + sdkp->zone_blocks = zone_blocks; + sdkp->nr_zones = nr_zones; + } return 0; err: sdkp->capacity = 0; - sd_zbc_cleanup(sdkp); return ret; } -void sd_zbc_remove(struct scsi_disk *sdkp) -{ - sd_zbc_cleanup(sdkp); -} - void sd_zbc_print_zones(struct scsi_disk *sdkp) { if (!sd_is_zoned(sdkp) || !sdkp->capacity) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 51fe6472ce02..4293dc1cd160 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -402,6 +402,7 @@ extern int blkdev_report_zones(struct block_device *bdev, unsigned int *nr_zones, gfp_t gfp_mask); extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors, sector_t nr_sectors, gfp_t gfp_mask); +extern int blk_revalidate_disk_zones(struct gendisk *disk); extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg); @@ -414,6 +415,12 @@ static inline unsigned int blkdev_nr_zones(struct block_device *bdev) { return 0; } + +static inline int blk_revalidate_disk_zones(struct gendisk *disk) +{ + return 0; +} + static inline int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) -- cgit v1.2.3-71-gd317