diff options
Diffstat (limited to 'packages/sys-kernel/linux-image-redcore-lts/files/0002-BFQ-v8r12-20180404.patch')
| -rw-r--r-- | packages/sys-kernel/linux-image-redcore-lts/files/0002-BFQ-v8r12-20180404.patch | 4611 |
1 files changed, 4611 insertions, 0 deletions
diff --git a/packages/sys-kernel/linux-image-redcore-lts/files/0002-BFQ-v8r12-20180404.patch b/packages/sys-kernel/linux-image-redcore-lts/files/0002-BFQ-v8r12-20180404.patch new file mode 100644 index 0000000..104325d --- /dev/null +++ b/packages/sys-kernel/linux-image-redcore-lts/files/0002-BFQ-v8r12-20180404.patch @@ -0,0 +1,4611 @@ +From 7bd365a925748767d7ed807e5498f90bae0ebc25 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Tue, 14 Nov 2017 08:28:45 +0100 +Subject: [PATCH 01/23] block, bfq-mq: turn BUG_ON on request-size into WARN_ON + +BFQ has many checks of internal and external consistency. One of them +checks that an I/O request has still sectors to serve, if it happens +to be retired without being served. If the request has no sector to +serve, a BUG_ON signals the failure and causes the kernel to +terminate. Yet, from a crash report by a user [1], this condition may +happen to hold, in apparently correct functioning, for I/O with a +CD/DVD. + +To address this issue, this commit turns the above BUG_ON into a +WARN_ON. This commit also adds a companion WARN_ON on request +insertion into the scheduler. + +[1] https://groups.google.com/d/msg/bfq-iosched/DDOTJBroBa4/VyU1zUFtCgAJ + +Reported-by: Alexandre Frade <admfrade@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 0c09609a6099..0fc757ae7a42 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -1540,6 +1540,8 @@ static void bfq_add_request(struct request *rq) + + BUG_ON(!RQ_BFQQ(rq)); + BUG_ON(RQ_BFQQ(rq) != bfqq); ++ WARN_ON(blk_rq_sectors(rq) == 0); ++ + elv_rb_add(&bfqq->sort_list, rq); + + /* +@@ -4962,7 +4964,7 @@ static void bfq_finish_request(struct request *rq) + rq_io_start_time_ns(rq), + rq->cmd_flags); + +- BUG_ON(blk_rq_sectors(rq) == 0 && !(rq->rq_flags & RQF_STARTED)); ++ WARN_ON(blk_rq_sectors(rq) == 0 && !(rq->rq_flags & RQF_STARTED)); + + if (likely(rq->rq_flags & RQF_STARTED)) { + unsigned long flags; + +From 1097d368a20456c88acd75b3184c68df38e8f7b8 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Sun, 12 Nov 2017 22:43:46 +0100 +Subject: [PATCH 02/23] block, bfq-sq, bfq-mq: consider also past I/O in soft + real-time detection + +BFQ privileges the I/O of soft real-time applications, such as video +players, to guarantee to these application a high bandwidth and a low +latency. In this respect, it is not easy to correctly detect when an +application is soft real-time. A particularly nasty false positive is +that of an I/O-bound application that occasionally happens to meet all +requirements to be deemed as soft real-time. After being detected as +soft real-time, such an application monopolizes the device. Fortunately, +BFQ will realize soon that the application is actually not soft +real-time and suspend every privilege. Yet, the application may happen +again to be wrongly detected as soft real-time, and so on. + +As highlighted by our tests, this problem causes BFQ to occasionally +fail to guarantee a high responsiveness, in the presence of heavy +background I/O workloads. The reason is that the background workload +happens to be detected as soft real-time, more or less frequently, +during the execution of the interactive task under test. To give an +idea, because of this problem, Libreoffice Writer occasionally takes 8 +seconds, instead of 3, to start up, if there are sequential reads and +writes in the background, on a Kingston SSDNow V300. + +This commit addresses this issue by leveraging the following facts. + +The reason why some applications are detected as soft real-time despite +all BFQ checks to avoid false positives, is simply that, during high +CPU or storage-device load, I/O-bound applications may happen to do +I/O slowly enough to meet all soft real-time requirements, and pass +all BFQ extra checks. Yet, this happens only for limited time periods: +slow-speed time intervals are usually interspersed between other time +intervals during which these applications do I/O at a very high speed. +To exploit these facts, this commit introduces a little change, in the +detection of soft real-time behavior, to systematically consider also +the recent past: the higher the speed was in the recent past, the +later next I/O should arrive for the application to be considered as +soft real-time. At the beginning of a slow-speed interval, the minimum +arrival time allowed for the next I/O usually happens to still be so +high, to fall *after* the end of the slow-speed period itself. As a +consequence, the application does not risk to be deemed as soft +real-time during the slow-speed interval. Then, during the next +high-speed interval, the application cannot, evidently, be deemed as +soft real-time (exactly because of its speed), and so on. + +This extra filtering proved to be rather effective: in the above test, +the frequency of false positives became so low that the start-up time +was 3 seconds in all iterations (apart from occasional outliers, +caused by page-cache-management issues, which are out of the scope of +this commit, and cannot be solved by an I/O scheduler). + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +Signed-off-by: Angelo Ruocco <angeloruocco90@gmail.com> +--- + block/bfq-mq-iosched.c | 115 ++++++++++++++++++++++++++++++++++--------------- + block/bfq-sq-iosched.c | 115 ++++++++++++++++++++++++++++++++++--------------- + 2 files changed, 162 insertions(+), 68 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 0fc757ae7a42..4d06d900f45e 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -3201,37 +3201,78 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * whereas soft_rt_next_start is set to infinity for applications that do + * not. + * +- * Unfortunately, even a greedy application may happen to behave in an +- * isochronous way if the CPU load is high. In fact, the application may +- * stop issuing requests while the CPUs are busy serving other processes, +- * then restart, then stop again for a while, and so on. In addition, if +- * the disk achieves a low enough throughput with the request pattern +- * issued by the application (e.g., because the request pattern is random +- * and/or the device is slow), then the application may meet the above +- * bandwidth requirement too. To prevent such a greedy application to be +- * deemed as soft real-time, a further rule is used in the computation of +- * soft_rt_next_start: soft_rt_next_start must be higher than the current +- * time plus the maximum time for which the arrival of a request is waited +- * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle. +- * This filters out greedy applications, as the latter issue instead their +- * next request as soon as possible after the last one has been completed +- * (in contrast, when a batch of requests is completed, a soft real-time +- * application spends some time processing data). ++ * Unfortunately, even a greedy (i.e., I/O-bound) application may ++ * happen to meet, occasionally or systematically, both the above ++ * bandwidth and isochrony requirements. This may happen at least in ++ * the following circumstances. First, if the CPU load is high. The ++ * application may stop issuing requests while the CPUs are busy ++ * serving other processes, then restart, then stop again for a while, ++ * and so on. The other circumstances are related to the storage ++ * device: the storage device is highly loaded or reaches a low-enough ++ * throughput with the I/O of the application (e.g., because the I/O ++ * is random and/or the device is slow). In all these cases, the ++ * I/O of the application may be simply slowed down enough to meet ++ * the bandwidth and isochrony requirements. To reduce the probability ++ * that greedy applications are deemed as soft real-time in these ++ * corner cases, a further rule is used in the computation of ++ * soft_rt_next_start: the return value of this function is forced to ++ * be higher than the maximum between the following two quantities. + * +- * Unfortunately, the last filter may easily generate false positives if +- * only bfqd->bfq_slice_idle is used as a reference time interval and one +- * or both the following cases occur: +- * 1) HZ is so low that the duration of a jiffy is comparable to or higher +- * than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with +- * HZ=100. ++ * (a) Current time plus: (1) the maximum time for which the arrival ++ * of a request is waited for when a sync queue becomes idle, ++ * namely bfqd->bfq_slice_idle, and (2) a few extra jiffies. We ++ * postpone for a moment the reason for adding a few extra ++ * jiffies; we get back to it after next item (b). Lower-bounding ++ * the return value of this function with the current time plus ++ * bfqd->bfq_slice_idle tends to filter out greedy applications, ++ * because the latter issue their next request as soon as possible ++ * after the last one has been completed. In contrast, a soft ++ * real-time application spends some time processing data, after a ++ * batch of its requests has been completed. ++ * ++ * (b) Current value of bfqq->soft_rt_next_start. As pointed out ++ * above, greedy applications may happen to meet both the ++ * bandwidth and isochrony requirements under heavy CPU or ++ * storage-device load. In more detail, in these scenarios, these ++ * applications happen, only for limited time periods, to do I/O ++ * slowly enough to meet all the requirements described so far, ++ * including the filtering in above item (a). These slow-speed ++ * time intervals are usually interspersed between other time ++ * intervals during which these applications do I/O at a very high ++ * speed. Fortunately, exactly because of the high speed of the ++ * I/O in the high-speed intervals, the values returned by this ++ * function happen to be so high, near the end of any such ++ * high-speed interval, to be likely to fall *after* the end of ++ * the low-speed time interval that follows. These high values are ++ * stored in bfqq->soft_rt_next_start after each invocation of ++ * this function. As a consequence, if the last value of ++ * bfqq->soft_rt_next_start is constantly used to lower-bound the ++ * next value that this function may return, then, from the very ++ * beginning of a low-speed interval, bfqq->soft_rt_next_start is ++ * likely to be constantly kept so high that any I/O request ++ * issued during the low-speed interval is considered as arriving ++ * to soon for the application to be deemed as soft ++ * real-time. Then, in the high-speed interval that follows, the ++ * application will not be deemed as soft real-time, just because ++ * it will do I/O at a high speed. And so on. ++ * ++ * Getting back to the filtering in item (a), in the following two ++ * cases this filtering might be easily passed by a greedy ++ * application, if the reference quantity was just ++ * bfqd->bfq_slice_idle: ++ * 1) HZ is so low that the duration of a jiffy is comparable to or ++ * higher than bfqd->bfq_slice_idle. This happens, e.g., on slow ++ * devices with HZ=100. The time granularity may be so coarse ++ * that the approximation, in jiffies, of bfqd->bfq_slice_idle ++ * is rather lower than the exact value. + * 2) jiffies, instead of increasing at a constant rate, may stop increasing + * for a while, then suddenly 'jump' by several units to recover the lost + * increments. This seems to happen, e.g., inside virtual machines. +- * To address this issue, we do not use as a reference time interval just +- * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In +- * particular we add the minimum number of jiffies for which the filter +- * seems to be quite precise also in embedded systems and KVM/QEMU virtual +- * machines. ++ * To address this issue, in the filtering in (a) we do not use as a ++ * reference time interval just bfqd->bfq_slice_idle, but ++ * bfqd->bfq_slice_idle plus a few jiffies. In particular, we add the ++ * minimum number of jiffies for which the filter seems to be quite ++ * precise also in embedded systems and KVM/QEMU virtual machines. + */ + static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, + struct bfq_queue *bfqq) +@@ -3243,10 +3284,11 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, + jiffies_to_msecs(HZ * bfqq->service_from_backlogged / + bfqd->bfq_wr_max_softrt_rate)); + +- return max(bfqq->last_idle_bklogged + +- HZ * bfqq->service_from_backlogged / +- bfqd->bfq_wr_max_softrt_rate, +- jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); ++ return max3(bfqq->soft_rt_next_start, ++ bfqq->last_idle_bklogged + ++ HZ * bfqq->service_from_backlogged / ++ bfqd->bfq_wr_max_softrt_rate, ++ jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); + } + + /** +@@ -4395,10 +4437,15 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bfqq->split_time = bfq_smallest_from_now(); + + /* +- * Set to the value for which bfqq will not be deemed as +- * soft rt when it becomes backlogged. ++ * To not forget the possibly high bandwidth consumed by a ++ * process/queue in the recent past, ++ * bfq_bfqq_softrt_next_start() returns a value at least equal ++ * to the current value of bfqq->soft_rt_next_start (see ++ * comments on bfq_bfqq_softrt_next_start). Set ++ * soft_rt_next_start to now, to mean that bfqq has consumed ++ * no bandwidth so far. + */ +- bfqq->soft_rt_next_start = bfq_greatest_from_now(); ++ bfqq->soft_rt_next_start = jiffies; + + /* first request is almost certainly seeky */ + bfqq->seek_history = 1; +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index 4bbd7f4c0154..987dc255c82c 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -3089,37 +3089,78 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * whereas soft_rt_next_start is set to infinity for applications that do + * not. + * +- * Unfortunately, even a greedy application may happen to behave in an +- * isochronous way if the CPU load is high. In fact, the application may +- * stop issuing requests while the CPUs are busy serving other processes, +- * then restart, then stop again for a while, and so on. In addition, if +- * the disk achieves a low enough throughput with the request pattern +- * issued by the application (e.g., because the request pattern is random +- * and/or the device is slow), then the application may meet the above +- * bandwidth requirement too. To prevent such a greedy application to be +- * deemed as soft real-time, a further rule is used in the computation of +- * soft_rt_next_start: soft_rt_next_start must be higher than the current +- * time plus the maximum time for which the arrival of a request is waited +- * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle. +- * This filters out greedy applications, as the latter issue instead their +- * next request as soon as possible after the last one has been completed +- * (in contrast, when a batch of requests is completed, a soft real-time +- * application spends some time processing data). ++ * Unfortunately, even a greedy (i.e., I/O-bound) application may ++ * happen to meet, occasionally or systematically, both the above ++ * bandwidth and isochrony requirements. This may happen at least in ++ * the following circumstances. First, if the CPU load is high. The ++ * application may stop issuing requests while the CPUs are busy ++ * serving other processes, then restart, then stop again for a while, ++ * and so on. The other circumstances are related to the storage ++ * device: the storage device is highly loaded or reaches a low-enough ++ * throughput with the I/O of the application (e.g., because the I/O ++ * is random and/or the device is slow). In all these cases, the ++ * I/O of the application may be simply slowed down enough to meet ++ * the bandwidth and isochrony requirements. To reduce the probability ++ * that greedy applications are deemed as soft real-time in these ++ * corner cases, a further rule is used in the computation of ++ * soft_rt_next_start: the return value of this function is forced to ++ * be higher than the maximum between the following two quantities. + * +- * Unfortunately, the last filter may easily generate false positives if +- * only bfqd->bfq_slice_idle is used as a reference time interval and one +- * or both the following cases occur: +- * 1) HZ is so low that the duration of a jiffy is comparable to or higher +- * than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with +- * HZ=100. ++ * (a) Current time plus: (1) the maximum time for which the arrival ++ * of a request is waited for when a sync queue becomes idle, ++ * namely bfqd->bfq_slice_idle, and (2) a few extra jiffies. We ++ * postpone for a moment the reason for adding a few extra ++ * jiffies; we get back to it after next item (b). Lower-bounding ++ * the return value of this function with the current time plus ++ * bfqd->bfq_slice_idle tends to filter out greedy applications, ++ * because the latter issue their next request as soon as possible ++ * after the last one has been completed. In contrast, a soft ++ * real-time application spends some time processing data, after a ++ * batch of its requests has been completed. ++ * ++ * (b) Current value of bfqq->soft_rt_next_start. As pointed out ++ * above, greedy applications may happen to meet both the ++ * bandwidth and isochrony requirements under heavy CPU or ++ * storage-device load. In more detail, in these scenarios, these ++ * applications happen, only for limited time periods, to do I/O ++ * slowly enough to meet all the requirements described so far, ++ * including the filtering in above item (a). These slow-speed ++ * time intervals are usually interspersed between other time ++ * intervals during which these applications do I/O at a very high ++ * speed. Fortunately, exactly because of the high speed of the ++ * I/O in the high-speed intervals, the values returned by this ++ * function happen to be so high, near the end of any such ++ * high-speed interval, to be likely to fall *after* the end of ++ * the low-speed time interval that follows. These high values are ++ * stored in bfqq->soft_rt_next_start after each invocation of ++ * this function. As a consequence, if the last value of ++ * bfqq->soft_rt_next_start is constantly used to lower-bound the ++ * next value that this function may return, then, from the very ++ * beginning of a low-speed interval, bfqq->soft_rt_next_start is ++ * likely to be constantly kept so high that any I/O request ++ * issued during the low-speed interval is considered as arriving ++ * to soon for the application to be deemed as soft ++ * real-time. Then, in the high-speed interval that follows, the ++ * application will not be deemed as soft real-time, just because ++ * it will do I/O at a high speed. And so on. ++ * ++ * Getting back to the filtering in item (a), in the following two ++ * cases this filtering might be easily passed by a greedy ++ * application, if the reference quantity was just ++ * bfqd->bfq_slice_idle: ++ * 1) HZ is so low that the duration of a jiffy is comparable to or ++ * higher than bfqd->bfq_slice_idle. This happens, e.g., on slow ++ * devices with HZ=100. The time granularity may be so coarse ++ * that the approximation, in jiffies, of bfqd->bfq_slice_idle ++ * is rather lower than the exact value. + * 2) jiffies, instead of increasing at a constant rate, may stop increasing + * for a while, then suddenly 'jump' by several units to recover the lost + * increments. This seems to happen, e.g., inside virtual machines. +- * To address this issue, we do not use as a reference time interval just +- * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In +- * particular we add the minimum number of jiffies for which the filter +- * seems to be quite precise also in embedded systems and KVM/QEMU virtual +- * machines. ++ * To address this issue, in the filtering in (a) we do not use as a ++ * reference time interval just bfqd->bfq_slice_idle, but ++ * bfqd->bfq_slice_idle plus a few jiffies. In particular, we add the ++ * minimum number of jiffies for which the filter seems to be quite ++ * precise also in embedded systems and KVM/QEMU virtual machines. + */ + static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, + struct bfq_queue *bfqq) +@@ -3131,10 +3172,11 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, + jiffies_to_msecs(HZ * bfqq->service_from_backlogged / + bfqd->bfq_wr_max_softrt_rate)); + +- return max(bfqq->last_idle_bklogged + +- HZ * bfqq->service_from_backlogged / +- bfqd->bfq_wr_max_softrt_rate, +- jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); ++ return max3(bfqq->soft_rt_next_start, ++ bfqq->last_idle_bklogged + ++ HZ * bfqq->service_from_backlogged / ++ bfqd->bfq_wr_max_softrt_rate, ++ jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); + } + + /** +@@ -4167,10 +4209,15 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bfqq->split_time = bfq_smallest_from_now(); + + /* +- * Set to the value for which bfqq will not be deemed as +- * soft rt when it becomes backlogged. ++ * To not forget the possibly high bandwidth consumed by a ++ * process/queue in the recent past, ++ * bfq_bfqq_softrt_next_start() returns a value at least equal ++ * to the current value of bfqq->soft_rt_next_start (see ++ * comments on bfq_bfqq_softrt_next_start). Set ++ * soft_rt_next_start to now, to mean that bfqq has consumed ++ * no bandwidth so far. + */ +- bfqq->soft_rt_next_start = bfq_greatest_from_now(); ++ bfqq->soft_rt_next_start = jiffies; + + /* first request is almost certainly seeky */ + bfqq->seek_history = 1; + +From 2a09b505660c81dbb80a5d68c9bc558c326d041f Mon Sep 17 00:00:00 2001 +From: Chiara Bruschi <bruschi.chiara@outlook.it> +Date: Thu, 7 Dec 2017 09:57:19 +0100 +Subject: [PATCH 03/23] block, bfq-mq: fix occurrences of request + prepare/finish methods' old names + +Commits 'b01f1fa3bb19' (Port of "blk-mq-sched: unify request prepare +methods") and 'cc10d2d7d2c1' (Port of "blk-mq-sched: unify request +finished methods") changed the old names of current bfq_prepare_request +and bfq_finish_request methods, but left them unchanged elsewhere in +the code (related comments, part of function name bfq_put_rq_priv_body). + +This commit fixes every occurrence of the old names of these methods +by changing them into the current names. + +Fixes: b01f1fa3bb19 (Port of "blk-mq-sched: unify request prepare methods") +Fixes: cc10d2d7d2c1 (Port of "blk-mq-sched: unify request finished methods") +Reviewed-by: Paolo Valente <paolo.valente@linaro.org> +Signed-off-by: Federico Motta <federico@willer.it> +Signed-off-by: Chiara Bruschi <bruschi.chiara@outlook.it> +--- + block/bfq-mq-iosched.c | 38 +++++++++++++++++++------------------- + 1 file changed, 19 insertions(+), 19 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 4d06d900f45e..8f8d5eccb016 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -4018,20 +4018,20 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + /* + * TESTING: reset DISP_LIST flag, because: 1) + * this rq this request has passed through +- * get_rq_private, 2) then it will have +- * put_rq_private invoked on it, and 3) in +- * put_rq_private we use this flag to check +- * that put_rq_private is not invoked on +- * requests for which get_rq_private has been +- * invoked. ++ * bfq_prepare_request, 2) then it will have ++ * bfq_finish_request invoked on it, and 3) in ++ * bfq_finish_request we use this flag to check ++ * that bfq_finish_request is not invoked on ++ * requests for which bfq_prepare_request has ++ * been invoked. + */ + rq->rq_flags &= ~RQF_DISP_LIST; + goto inc_in_driver_start_rq; + } + + /* +- * We exploit the put_rq_private hook to decrement +- * rq_in_driver, but put_rq_private will not be ++ * We exploit the bfq_finish_request hook to decrement ++ * rq_in_driver, but bfq_finish_request will not be + * invoked on this request. So, to avoid unbalance, + * just start this request, without incrementing + * rq_in_driver. As a negative consequence, +@@ -4040,14 +4040,14 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + * bfq_schedule_dispatch to be invoked uselessly. + * + * As for implementing an exact solution, the +- * put_request hook, if defined, is probably invoked +- * also on this request. So, by exploiting this hook, +- * we could 1) increment rq_in_driver here, and 2) +- * decrement it in put_request. Such a solution would +- * let the value of the counter be always accurate, +- * but it would entail using an extra interface +- * function. This cost seems higher than the benefit, +- * being the frequency of non-elevator-private ++ * bfq_finish_request hook, if defined, is probably ++ * invoked also on this request. So, by exploiting ++ * this hook, we could 1) increment rq_in_driver here, ++ * and 2) decrement it in bfq_finish_request. Such a ++ * solution would let the value of the counter be ++ * always accurate, but it would entail using an extra ++ * interface function. This cost seems higher than the ++ * benefit, being the frequency of non-elevator-private + * requests very low. + */ + goto start_rq; +@@ -4963,7 +4963,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) + } + } + +-static void bfq_put_rq_priv_body(struct bfq_queue *bfqq) ++static void bfq_finish_request_body(struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqq->bfqd, bfqq, + "put_request_body: allocated %d", bfqq->allocated); +@@ -5019,7 +5019,7 @@ static void bfq_finish_request(struct request *rq) + spin_lock_irqsave(&bfqd->lock, flags); + + bfq_completed_request(bfqq, bfqd); +- bfq_put_rq_priv_body(bfqq); ++ bfq_finish_request_body(bfqq); + + spin_unlock_irqrestore(&bfqd->lock, flags); + } else { +@@ -5042,7 +5042,7 @@ static void bfq_finish_request(struct request *rq) + bfqg_stats_update_io_remove(bfqq_group(bfqq), + rq->cmd_flags); + } +- bfq_put_rq_priv_body(bfqq); ++ bfq_finish_request_body(bfqq); + } + + rq->elv.priv[0] = NULL; + +From 4df19943c3a767df453abea3d2ac3433c3326ce0 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Thu, 16 Nov 2017 18:38:13 +0100 +Subject: [PATCH 04/23] block, bfq-sq, bfq-mq: add missing rq_pos_tree update + on rq removal + +If two processes do I/O close to each other, then BFQ merges the +bfq_queues associated with these processes, to get a more sequential +I/O, and thus a higher throughput. In this respect, to detect whether +two processes are doing I/O close to each other, BFQ keeps a list of +the head-of-line I/O requests of all active bfq_queues. The list is +ordered by initial sectors, and implemented through a red-black tree +(rq_pos_tree). + +Unfortunately, the update of the rq_pos_tree was incomplete, because +the tree was not updated on the removal of the head-of-line I/O +request of a bfq_queue, in case the queue did not remain empty. This +commit adds the missing update. + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +Signed-off-by: Angelo Ruocco <angeloruocco90@gmail.com> +--- + block/bfq-mq-iosched.c | 3 +++ + block/bfq-sq-iosched.c | 3 +++ + 2 files changed, 6 insertions(+) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 8f8d5eccb016..603191c9008f 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -1729,6 +1729,9 @@ static void bfq_remove_request(struct request_queue *q, + rb_erase(&bfqq->pos_node, bfqq->pos_root); + bfqq->pos_root = NULL; + } ++ } else { ++ BUG_ON(!bfqq->next_rq); ++ bfq_pos_tree_add_move(bfqd, bfqq); + } + + if (rq->cmd_flags & REQ_META) { +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index 987dc255c82c..ea90ace79e49 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -1669,6 +1669,9 @@ static void bfq_remove_request(struct request *rq) + rb_erase(&bfqq->pos_node, bfqq->pos_root); + bfqq->pos_root = NULL; + } ++ } else { ++ BUG_ON(!bfqq->next_rq); ++ bfq_pos_tree_add_move(bfqd, bfqq); + } + + if (rq->cmd_flags & REQ_META) { + +From b844e345140aaea957d84a21d2aa67588b020cd5 Mon Sep 17 00:00:00 2001 +From: Angelo Ruocco <angeloruocco90@gmail.com> +Date: Mon, 18 Dec 2017 08:28:08 +0100 +Subject: [PATCH 05/23] block, bfq-sq, bfq-mq: check low_latency flag in + bfq_bfqq_save_state() + +A just-created bfq_queue will certainly be deemed as interactive on +the arrival of its first I/O request, if the low_latency flag is +set. Yet, if the queue is merged with another queue on the arrival of +its first I/O request, it will not have the chance to be flagged as +interactive. Nevertheless, if the queue is then split soon enough, it +has to be flagged as interactive after the split. + +To handle this early-merge scenario correctly, BFQ saves the state of +the queue, on the merge, as if the latter had already been deemed +interactive. So, if the queue is split soon, it will get +weight-raised, because the previous state of the queue is resumed on +the split. + +Unfortunately, in the act of saving the state of the newly-created +queue, BFQ doesn't check whether the low_latency flag is set, and this +causes early-merged queues to be then weight-raised, on queue splits, +even if low_latency is off. This commit addresses this problem by +adding the missing check. + +Signed-off-by: Angelo Ruocco <angeloruocco90@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 3 ++- + block/bfq-sq-iosched.c | 3 ++- + 2 files changed, 4 insertions(+), 2 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 603191c9008f..ff9776c8836a 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -2231,7 +2231,8 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) + bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); + bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); + if (unlikely(bfq_bfqq_just_created(bfqq) && +- !bfq_bfqq_in_large_burst(bfqq))) { ++ !bfq_bfqq_in_large_burst(bfqq) && ++ bfqq->bfqd->low_latency)) { + /* + * bfqq being merged ritgh after being created: bfqq + * would have deserved interactive weight raising, but +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index ea90ace79e49..3a2d764e760c 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -2109,7 +2109,8 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) + bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); + bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); + if (unlikely(bfq_bfqq_just_created(bfqq) && +- !bfq_bfqq_in_large_burst(bfqq))) { ++ !bfq_bfqq_in_large_burst(bfqq) && ++ bfqq->bfqd->low_latency)) { + /* + * bfqq being merged ritgh after being created: bfqq + * would have deserved interactive weight raising, but + +From 4cc6896fe1de2e0b4de151a6e70658f10b9ec2fa Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Fri, 27 Oct 2017 11:12:14 +0200 +Subject: [PATCH 06/23] block, bfq-sq, bfq-mq: let a queue be merged only + shortly after starting I/O + +In BFQ and CFQ, two processes are said to be cooperating if they do +I/O in such a way that the union of their I/O requests yields a +sequential I/O pattern. To get such a sequential I/O pattern out of +the non-sequential pattern of each cooperating process, BFQ and CFQ +merge the queues associated with these processes. In more detail, +cooperating processes, and thus their associated queues, usually +start, or restart, to do I/O shortly after each other. This is the +case, e.g., for the I/O threads of KVM/QEMU and of the dump +utility. Basing on this assumption, this commit allows a bfq_queue to +be merged only during a short time interval (100ms) after it starts, +or re-starts, to do I/O. This filtering provides two important +benefits. + +First, it greatly reduces the probability that two non-cooperating +processes have their queues merged by mistake, if they just happen to +do I/O close to each other for a short time interval. These spurious +merges cause loss of service guarantees. A low-weight bfq_queue may +unjustly get more than its expected share of the throughput: if such a +low-weight queue is merged with a high-weight queue, then the I/O for +the low-weight queue is served as if the queue had a high weight. This +may damage other high-weight queues unexpectedly. For instance, +because of this issue, lxterminal occasionally took 7.5 seconds to +start, instead of 6.5 seconds, when some sequential readers and +writers did I/O in the background on a FUJITSU MHX2300BT HDD. The +reason is that the bfq_queues associated with some of the readers or +the writers were merged with the high-weight queues of some processes +that had to do some urgent but little I/O. The readers then exploited +the inherited high weight for all or most of their I/O, during the +start-up of terminal. The filtering introduced by this commit +eliminated any outlier caused by spurious queue merges in our start-up +time tests. + +This filtering also provides a little boost of the throughput +sustainable by BFQ: 3-4%, depending on the CPU. The reason is that, +once a bfq_queue cannot be merged any longer, this commit makes BFQ +stop updating the data needed to handle merging for the queue. + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +Signed-off-by: Angelo Ruocco <angeloruocco90@gmail.com> +--- + block/bfq-mq-iosched.c | 64 +++++++++++++++++++++++++++++++++++++++++--------- + block/bfq-mq.h | 1 + + block/bfq-sched.c | 4 ++++ + block/bfq-sq-iosched.c | 64 +++++++++++++++++++++++++++++++++++++++++--------- + block/bfq.h | 2 ++ + 5 files changed, 113 insertions(+), 22 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index ff9776c8836a..8b17b25a3c30 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -119,6 +119,20 @@ static const int bfq_async_charge_factor = 10; + /* Default timeout values, in jiffies, approximating CFQ defaults. */ + static const int bfq_timeout = (HZ / 8); + ++/* ++ * Time limit for merging (see comments in bfq_setup_cooperator). Set ++ * to the slowest value that, in our tests, proved to be effective in ++ * removing false positives, while not causing true positives to miss ++ * queue merging. ++ * ++ * As can be deduced from the low time limit below, queue merging, if ++ * successful, happens at the very beggining of the I/O of the involved ++ * cooperating processes, as a consequence of the arrival of the very ++ * first requests from each cooperator. After that, there is very ++ * little chance to find cooperators. ++ */ ++static const unsigned long bfq_merge_time_limit = HZ/10; ++ + static struct kmem_cache *bfq_pool; + + /* Below this threshold (in ns), we consider thinktime immediate. */ +@@ -389,6 +403,13 @@ bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, + return bfqq; + } + ++static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) ++{ ++ return bfqq->service_from_backlogged > 0 && ++ time_is_before_jiffies(bfqq->first_IO_time + ++ bfq_merge_time_limit); ++} ++ + static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) + { + struct rb_node **p, *parent; +@@ -399,6 +420,14 @@ static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) + bfqq->pos_root = NULL; + } + ++ /* ++ * bfqq cannot be merged any longer (see comments in ++ * bfq_setup_cooperator): no point in adding bfqq into the ++ * position tree. ++ */ ++ if (bfq_too_late_for_merging(bfqq)) ++ return; ++ + if (bfq_class_idle(bfqq)) + return; + if (!bfqq->next_rq) +@@ -2081,6 +2110,13 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) + static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, + struct bfq_queue *new_bfqq) + { ++ if (bfq_too_late_for_merging(new_bfqq)) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "[%s] too late for bfq%d to be merged", ++ __func__, new_bfqq->pid); ++ return false; ++ } ++ + if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || + (bfqq->ioprio_class != new_bfqq->ioprio_class)) + return false; +@@ -2149,6 +2185,23 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + { + struct bfq_queue *in_service_bfqq, *new_bfqq; + ++ /* ++ * Prevent bfqq from being merged if it has been created too ++ * long ago. The idea is that true cooperating processes, and ++ * thus their associated bfq_queues, are supposed to be ++ * created shortly after each other. This is the case, e.g., ++ * for KVM/QEMU and dump I/O threads. Basing on this ++ * assumption, the following filtering greatly reduces the ++ * probability that two non-cooperating processes, which just ++ * happen to do close I/O for some short time interval, have ++ * their queues merged by mistake. ++ */ ++ if (bfq_too_late_for_merging(bfqq)) { ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have looked for coop, but too late"); ++ return NULL; ++ } ++ + if (bfqq->new_bfqq) + return bfqq->new_bfqq; + +@@ -3338,17 +3391,6 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd, + */ + slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); + +- /* +- * Increase service_from_backlogged before next statement, +- * because the possible next invocation of +- * bfq_bfqq_charge_time would likely inflate +- * entity->service. In contrast, service_from_backlogged must +- * contain real service, to enable the soft real-time +- * heuristic to correctly compute the bandwidth consumed by +- * bfqq. +- */ +- bfqq->service_from_backlogged += entity->service; +- + /* + * As above explained, charge slow (typically seeky) and + * timed-out queues with the time and not the service +diff --git a/block/bfq-mq.h b/block/bfq-mq.h +index 1cb05bb853d2..a5947b203ef2 100644 +--- a/block/bfq-mq.h ++++ b/block/bfq-mq.h +@@ -337,6 +337,7 @@ struct bfq_queue { + unsigned long wr_start_at_switch_to_srt; + + unsigned long split_time; /* time of last split */ ++ unsigned long first_IO_time; /* time of first I/O for this queue */ + }; + + /** +diff --git a/block/bfq-sched.c b/block/bfq-sched.c +index 616c0692335a..9d261dd428e4 100644 +--- a/block/bfq-sched.c ++++ b/block/bfq-sched.c +@@ -939,6 +939,10 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) + struct bfq_entity *entity = &bfqq->entity; + struct bfq_service_tree *st; + ++ if (!bfqq->service_from_backlogged) ++ bfqq->first_IO_time = jiffies; ++ ++ bfqq->service_from_backlogged += served; + for_each_entity(entity) { + st = bfq_entity_service_tree(entity); + +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index 3a2d764e760c..cd00a41ca35d 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -113,6 +113,20 @@ static const int bfq_async_charge_factor = 10; + /* Default timeout values, in jiffies, approximating CFQ defaults. */ + static const int bfq_timeout = (HZ / 8); + ++/* ++ * Time limit for merging (see comments in bfq_setup_cooperator). Set ++ * to the slowest value that, in our tests, proved to be effective in ++ * removing false positives, while not causing true positives to miss ++ * queue merging. ++ * ++ * As can be deduced from the low time limit below, queue merging, if ++ * successful, happens at the very beggining of the I/O of the involved ++ * cooperating processes, as a consequence of the arrival of the very ++ * first requests from each cooperator. After that, there is very ++ * little chance to find cooperators. ++ */ ++static const unsigned long bfq_merge_time_limit = HZ/10; ++ + static struct kmem_cache *bfq_pool; + + /* Below this threshold (in ns), we consider thinktime immediate. */ +@@ -351,6 +365,13 @@ bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, + return bfqq; + } + ++static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) ++{ ++ return bfqq->service_from_backlogged > 0 && ++ time_is_before_jiffies(bfqq->first_IO_time + ++ bfq_merge_time_limit); ++} ++ + static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) + { + struct rb_node **p, *parent; +@@ -361,6 +382,14 @@ static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) + bfqq->pos_root = NULL; + } + ++ /* ++ * bfqq cannot be merged any longer (see comments in ++ * bfq_setup_cooperator): no point in adding bfqq into the ++ * position tree. ++ */ ++ if (bfq_too_late_for_merging(bfqq)) ++ return; ++ + if (bfq_class_idle(bfqq)) + return; + if (!bfqq->next_rq) +@@ -1960,6 +1989,13 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) + static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, + struct bfq_queue *new_bfqq) + { ++ if (bfq_too_late_for_merging(new_bfqq)) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "[%s] too late for bfq%d to be merged", ++ __func__, new_bfqq->pid); ++ return false; ++ } ++ + if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || + (bfqq->ioprio_class != new_bfqq->ioprio_class)) + return false; +@@ -2028,6 +2064,23 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + { + struct bfq_queue *in_service_bfqq, *new_bfqq; + ++ /* ++ * Prevent bfqq from being merged if it has been created too ++ * long ago. The idea is that true cooperating processes, and ++ * thus their associated bfq_queues, are supposed to be ++ * created shortly after each other. This is the case, e.g., ++ * for KVM/QEMU and dump I/O threads. Basing on this ++ * assumption, the following filtering greatly reduces the ++ * probability that two non-cooperating processes, which just ++ * happen to do close I/O for some short time interval, have ++ * their queues merged by mistake. ++ */ ++ if (bfq_too_late_for_merging(bfqq)) { ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have looked for coop, but too late"); ++ return NULL; ++ } ++ + if (bfqq->new_bfqq) + return bfqq->new_bfqq; + +@@ -3226,17 +3279,6 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd, + */ + slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); + +- /* +- * Increase service_from_backlogged before next statement, +- * because the possible next invocation of +- * bfq_bfqq_charge_time would likely inflate +- * entity->service. In contrast, service_from_backlogged must +- * contain real service, to enable the soft real-time +- * heuristic to correctly compute the bandwidth consumed by +- * bfqq. +- */ +- bfqq->service_from_backlogged += entity->service; +- + /* + * As above explained, charge slow (typically seeky) and + * timed-out queues with the time and not the service +diff --git a/block/bfq.h b/block/bfq.h +index 47cd4d5a8c32..59539adc00a5 100644 +--- a/block/bfq.h ++++ b/block/bfq.h +@@ -329,6 +329,8 @@ struct bfq_queue { + unsigned long wr_start_at_switch_to_srt; + + unsigned long split_time; /* time of last split */ ++ ++ unsigned long first_IO_time; /* time of first I/O for this queue */ + }; + + /** + +From 157f39c43ab182280634cd4f6335d0187b3741a0 Mon Sep 17 00:00:00 2001 +From: Angelo Ruocco <angeloruocco90@gmail.com> +Date: Mon, 11 Dec 2017 14:19:54 +0100 +Subject: [PATCH 07/23] block, bfq-sq, bfq-mq: remove superfluous check in + queue-merging setup + +When two or more processes do I/O in a way that the their requests are +sequential in respect to one another, BFQ merges the bfq_queues associated +with the processes. This way the overall I/O pattern becomes sequential, +and thus there is a boost in througput. +These cooperating processes usually start or restart to do I/O shortly +after each other. So, in order to avoid merging non-cooperating processes, +BFQ ensures that none of these queues has been in weight raising for too +long. + +In this respect, from commit "block, bfq-sq, bfq-mq: let a queue be merged +only shortly after being created", BFQ checks whether any queue (and not +only weight-raised ones) is doing I/O continuously from too long to be +merged. + +This new additional check makes the first one useless: a queue doing +I/O from long enough, if being weight-raised, is also a queue in +weight raising for too long to be merged. Accordingly, this commit +removes the first check. + +Signed-off-by: Angelo Ruocco <angeloruocco90@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.com> +--- + block/bfq-mq-iosched.c | 53 ++++---------------------------------------------- + block/bfq-sq-iosched.c | 53 ++++---------------------------------------------- + 2 files changed, 8 insertions(+), 98 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 8b17b25a3c30..f5db8613a70f 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -2140,20 +2140,6 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, + return true; + } + +-/* +- * If this function returns true, then bfqq cannot be merged. The idea +- * is that true cooperation happens very early after processes start +- * to do I/O. Usually, late cooperations are just accidental false +- * positives. In case bfqq is weight-raised, such false positives +- * would evidently degrade latency guarantees for bfqq. +- */ +-static bool wr_from_too_long(struct bfq_queue *bfqq) +-{ +- return bfqq->wr_coeff > 1 && +- time_is_before_jiffies(bfqq->last_wr_start_finish + +- msecs_to_jiffies(100)); +-} +- + /* + * Attempt to schedule a merge of bfqq with the currently in-service + * queue or with a close queue among the scheduled queues. Return +@@ -2167,11 +2153,6 @@ static bool wr_from_too_long(struct bfq_queue *bfqq) + * to maintain. Besides, in such a critical condition as an out of memory, + * the benefits of queue merging may be little relevant, or even negligible. + * +- * Weight-raised queues can be merged only if their weight-raising +- * period has just started. In fact cooperating processes are usually +- * started together. Thus, with this filter we avoid false positives +- * that would jeopardize low-latency guarantees. +- * + * WARNING: queue merging may impair fairness among non-weight raised + * queues, for at least two reasons: 1) the original weight of a + * merged queue may change during the merged state, 2) even being the +@@ -2205,15 +2186,7 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + if (bfqq->new_bfqq) + return bfqq->new_bfqq; + +- if (io_struct && wr_from_too_long(bfqq) && +- likely(bfqq != &bfqd->oom_bfqq)) +- bfq_log_bfqq(bfqd, bfqq, +- "would have looked for coop, but bfq%d wr", +- bfqq->pid); +- +- if (!io_struct || +- wr_from_too_long(bfqq) || +- unlikely(bfqq == &bfqd->oom_bfqq)) ++ if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) + return NULL; + + /* If there is only one backlogged queue, don't search. */ +@@ -2223,17 +2196,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + in_service_bfqq = bfqd->in_service_queue; + + if (in_service_bfqq && in_service_bfqq != bfqq && +- wr_from_too_long(in_service_bfqq) +- && likely(in_service_bfqq == &bfqd->oom_bfqq)) +- bfq_log_bfqq(bfqd, bfqq, +- "would have tried merge with in-service-queue, but wr"); +- +- if (!in_service_bfqq || in_service_bfqq == bfqq +- || wr_from_too_long(in_service_bfqq) || +- unlikely(in_service_bfqq == &bfqd->oom_bfqq)) +- goto check_scheduled; +- +- if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && ++ likely(in_service_bfqq != &bfqd->oom_bfqq) && ++ bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && + bfqq->entity.parent == in_service_bfqq->entity.parent && + bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { + new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); +@@ -2245,21 +2209,12 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * queues. The only thing we need is that the bio/request is not + * NULL, as we need it to establish whether a cooperator exists. + */ +-check_scheduled: + new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, + bfq_io_struct_pos(io_struct, request)); + + BUG_ON(new_bfqq && bfqq->entity.parent != new_bfqq->entity.parent); + +- if (new_bfqq && wr_from_too_long(new_bfqq) && +- likely(new_bfqq != &bfqd->oom_bfqq) && +- bfq_may_be_close_cooperator(bfqq, new_bfqq)) +- bfq_log_bfqq(bfqd, bfqq, +- "would have merged with bfq%d, but wr", +- new_bfqq->pid); +- +- if (new_bfqq && !wr_from_too_long(new_bfqq) && +- likely(new_bfqq != &bfqd->oom_bfqq) && ++ if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq) && + bfq_may_be_close_cooperator(bfqq, new_bfqq)) + return bfq_setup_merge(bfqq, new_bfqq); + +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index cd00a41ca35d..d8a358e5e284 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -2019,20 +2019,6 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, + return true; + } + +-/* +- * If this function returns true, then bfqq cannot be merged. The idea +- * is that true cooperation happens very early after processes start +- * to do I/O. Usually, late cooperations are just accidental false +- * positives. In case bfqq is weight-raised, such false positives +- * would evidently degrade latency guarantees for bfqq. +- */ +-static bool wr_from_too_long(struct bfq_queue *bfqq) +-{ +- return bfqq->wr_coeff > 1 && +- time_is_before_jiffies(bfqq->last_wr_start_finish + +- msecs_to_jiffies(100)); +-} +- + /* + * Attempt to schedule a merge of bfqq with the currently in-service + * queue or with a close queue among the scheduled queues. Return +@@ -2046,11 +2032,6 @@ static bool wr_from_too_long(struct bfq_queue *bfqq) + * to maintain. Besides, in such a critical condition as an out of memory, + * the benefits of queue merging may be little relevant, or even negligible. + * +- * Weight-raised queues can be merged only if their weight-raising +- * period has just started. In fact cooperating processes are usually +- * started together. Thus, with this filter we avoid false positives +- * that would jeopardize low-latency guarantees. +- * + * WARNING: queue merging may impair fairness among non-weight raised + * queues, for at least two reasons: 1) the original weight of a + * merged queue may change during the merged state, 2) even being the +@@ -2084,15 +2065,7 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + if (bfqq->new_bfqq) + return bfqq->new_bfqq; + +- if (io_struct && wr_from_too_long(bfqq) && +- likely(bfqq != &bfqd->oom_bfqq)) +- bfq_log_bfqq(bfqd, bfqq, +- "would have looked for coop, but bfq%d wr", +- bfqq->pid); +- +- if (!io_struct || +- wr_from_too_long(bfqq) || +- unlikely(bfqq == &bfqd->oom_bfqq)) ++ if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) + return NULL; + + /* If there is only one backlogged queue, don't search. */ +@@ -2102,17 +2075,8 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + in_service_bfqq = bfqd->in_service_queue; + + if (in_service_bfqq && in_service_bfqq != bfqq && +- bfqd->in_service_bic && wr_from_too_long(in_service_bfqq) +- && likely(in_service_bfqq == &bfqd->oom_bfqq)) +- bfq_log_bfqq(bfqd, bfqq, +- "would have tried merge with in-service-queue, but wr"); +- +- if (!in_service_bfqq || in_service_bfqq == bfqq || +- !bfqd->in_service_bic || wr_from_too_long(in_service_bfqq) || +- unlikely(in_service_bfqq == &bfqd->oom_bfqq)) +- goto check_scheduled; +- +- if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && ++ likely(in_service_bfqq != &bfqd->oom_bfqq) && ++ bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && + bfqq->entity.parent == in_service_bfqq->entity.parent && + bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { + new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); +@@ -2124,21 +2088,12 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * queues. The only thing we need is that the bio/request is not + * NULL, as we need it to establish whether a cooperator exists. + */ +-check_scheduled: + new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, + bfq_io_struct_pos(io_struct, request)); + + BUG_ON(new_bfqq && bfqq->entity.parent != new_bfqq->entity.parent); + +- if (new_bfqq && wr_from_too_long(new_bfqq) && +- likely(new_bfqq != &bfqd->oom_bfqq) && +- bfq_may_be_close_cooperator(bfqq, new_bfqq)) +- bfq_log_bfqq(bfqd, bfqq, +- "would have merged with bfq%d, but wr", +- new_bfqq->pid); +- +- if (new_bfqq && !wr_from_too_long(new_bfqq) && +- likely(new_bfqq != &bfqd->oom_bfqq) && ++ if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq) && + bfq_may_be_close_cooperator(bfqq, new_bfqq)) + return bfq_setup_merge(bfqq, new_bfqq); + + +From b82eb91d87f172aba7eb5eb98e8d5e2a621adf51 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Thu, 30 Nov 2017 17:48:28 +0100 +Subject: [PATCH 08/23] block, bfq-sq, bfq-mq: increase threshold to deem I/O + as random + +If two processes do I/O close to each other, i.e., are cooperating +processes in BFQ (and CFQ'S) nomenclature, then BFQ merges their +associated bfq_queues, so as to get sequential I/O from the union of +the I/O requests of the processes, and thus reach a higher +throughput. A merged queue is then split if its I/O stops being +sequential. In this respect, BFQ deems the I/O of a bfq_queue as +(mostly) sequential only if less than 4 I/O requests are random, out +of the last 32 requests inserted into the queue. + +Unfortunately, extensive testing (with the interleaved_io benchmark of +the S suite [1], and with real applications spawning cooperating +processes) has clearly shown that, with such a low threshold, only a +rather low I/O throughput may be reached when several cooperating +processes do I/O. In particular, the outcome of each test run was +bimodal: if queue merging occurred and was stable during the test, +then the throughput was close to the peak rate of the storage device, +otherwise the throughput was arbitrarily low (usually around 1/10 of +the peak rate with a rotational device). The probability to get the +unlucky outcomes grew with the number of cooperating processes: it was +already significant with 5 processes, and close to one with 7 or more +processes. + +The cause of the low throughput in the unlucky runs was that the +merged queues containing the I/O of these cooperating processes were +soon split, because they contained more random I/O requests than those +tolerated by the 4/32 threshold, but +- that I/O would have however allowed the storage device to reach + peak throughput or almost peak throughput; +- in contrast, the I/O of these processes, if served individually + (from separate queues) yielded a rather low throughput. + +So we repeated our tests with increasing values of the threshold, +until we found the minimum value (19) for which we obtained maximum +throughput, reliably, with at least up to 9 cooperating +processes. Then we checked that the use of that higher threshold value +did not cause any regression for any other benchmark in the suite [1]. +This commit raises the threshold to such a higher value. + +[1] https://github.com/Algodev-github/S + +Signed-off-by: Angelo Ruocco <angeloruocco90@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 2 +- + block/bfq-sq-iosched.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index f5db8613a70f..cb5f49ddecb6 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -145,7 +145,7 @@ static struct kmem_cache *bfq_pool; + #define BFQQ_SEEK_THR (sector_t)(8 * 100) + #define BFQQ_SECT_THR_NONROT (sector_t)(2 * 32) + #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) +-#define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 32/8) ++#define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) + + /* Min number of samples required to perform peak-rate update */ + #define BFQ_RATE_MIN_SAMPLES 32 +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index d8a358e5e284..e1c6dc651be1 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -139,7 +139,7 @@ static struct kmem_cache *bfq_pool; + #define BFQQ_SEEK_THR (sector_t)(8 * 100) + #define BFQQ_SECT_THR_NONROT (sector_t)(2 * 32) + #define BFQQ_CLOSE_THR (sector_t)(8 * 1024) +-#define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 32/8) ++#define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) + + /* Min number of samples required to perform peak-rate update */ + #define BFQ_RATE_MIN_SAMPLES 32 + +From b739dda4e4b3a1cbbc905f86f9fbb0860b068ce7 Mon Sep 17 00:00:00 2001 +From: Chiara Bruschi <bruschi.chiara@outlook.it> +Date: Mon, 11 Dec 2017 18:55:26 +0100 +Subject: [PATCH 09/23] block, bfq-sq, bfq-mq: specify usage condition of + delta_us in bfq_log_bfqq call + +Inside the function bfq_completed_request the value of a variable +called delta_us is computed as current request completion time. +delta_us is used inside a call to the function bfq_log_bfqq as divisor +in a division operation to compute a rate value, but no check makes +sure that delta_us has non-zero value. A divisor with value 0 leads +to a division error that could result in a kernel oops (therefore +unstable/unreliable system state) and consequently cause kernel panic +if resources are unavailable after the system fault. + +This commit fixes this call to bfq_log_bfqq specifying the condition +that allows delta_us to be safely used as divisor. + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +Signed-off-by: Chiara Bruschi <bruschi.chiara@outlook.it> +--- + block/bfq-mq-iosched.c | 5 ++++- + block/bfq-sq-iosched.c | 5 ++++- + 2 files changed, 8 insertions(+), 2 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index cb5f49ddecb6..6ce2c0789046 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -4904,9 +4904,12 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) + bfq_log_bfqq(bfqd, bfqq, + "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu", + delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size, ++ delta_us > 0 ? + (USEC_PER_SEC* + (u64)((bfqd->last_rq_max_size<<BFQ_RATE_SHIFT)/delta_us)) +- >>BFQ_RATE_SHIFT, ++ >>BFQ_RATE_SHIFT : ++ (USEC_PER_SEC* ++ (u64)(bfqd->last_rq_max_size<<BFQ_RATE_SHIFT))>>BFQ_RATE_SHIFT, + (USEC_PER_SEC*(u64)(1UL<<(BFQ_RATE_SHIFT-10)))>>BFQ_RATE_SHIFT); + + /* +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index e1c6dc651be1..eff4c4edf5a0 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -4565,9 +4565,12 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq) + + bfq_log(bfqd, "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu", + delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size, ++ delta_us > 0 ? + (USEC_PER_SEC* + (u64)((bfqd->last_rq_max_size<<BFQ_RATE_SHIFT)/delta_us)) +- >>BFQ_RATE_SHIFT, ++ >>BFQ_RATE_SHIFT : ++ (USEC_PER_SEC* ++ (u64)(bfqd->last_rq_max_size<<BFQ_RATE_SHIFT))>>BFQ_RATE_SHIFT, + (USEC_PER_SEC*(u64)(1UL<<(BFQ_RATE_SHIFT-10)))>>BFQ_RATE_SHIFT); + + /* + +From ae4310c13eca762644734d53074d8456c85e2dec Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Tue, 19 Dec 2017 12:07:12 +0100 +Subject: [PATCH 10/23] block, bfq-mq: limit tags for writes and async I/O + +Asynchronous I/O can easily starve synchronous I/O (both sync reads +and sync writes), by consuming all request tags. Similarly, storms of +synchronous writes, such as those that sync(2) may trigger, can starve +synchronous reads. In their turn, these two problems may also cause +BFQ to loose control on latency for interactive and soft real-time +applications. For example, on a PLEXTOR PX-256M5S SSD, LibreOffice +Writer takes 0.6 seconds to start if the device is idle, but it takes +more than 45 seconds (!) if there are sequential writes in the +background. + +This commit addresses this issue by limiting the maximum percentage of +tags that asynchronous I/O requests and synchronous write requests can +consume. In particular, this commit grants a higher threshold to +synchronous writes, to prevent the latter from being starved by +asynchronous I/O. + +According to the above test, LibreOffice Writer now starts in about +1.2 seconds on average, regardless of the background workload, and +apart from some rare outlier. To check this improvement, run, e.g., +sudo ./comm_startup_lat.sh bfq-mq 5 5 seq 10 "lowriter --terminate_after_init" +for the comm_startup_lat benchmark in the S suite [1]. + +[1] https://github.com/Algodev-github/S + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++ + block/bfq-mq.h | 12 ++++++++ + 2 files changed, 89 insertions(+) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 6ce2c0789046..f384f5566672 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -362,6 +362,82 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd, + } + } + ++/* ++ * See the comments on bfq_limit_depth for the purpose of ++ * the depths set in the function. ++ */ ++static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt) ++{ ++ bfqd->sb_shift = bt->sb.shift; ++ ++ /* ++ * In-word depths if no bfq_queue is being weight-raised: ++ * leaving 25% of tags only for sync reads. ++ * ++ * In next formulas, right-shift the value ++ * (1U<<bfqd->sb_shift), instead of computing directly ++ * (1U<<(bfqd->sb_shift - something)), to be robust against ++ * any possible value of bfqd->sb_shift, without having to ++ * limit 'something'. ++ */ ++ /* no more than 50% of tags for async I/O */ ++ bfqd->word_depths[0][0] = max((1U<<bfqd->sb_shift)>>1, 1U); ++ /* ++ * no more than 75% of tags for sync writes (25% extra tags ++ * w.r.t. async I/O, to prevent async I/O from starving sync ++ * writes) ++ */ ++ bfqd->word_depths[0][1] = max(((1U<<bfqd->sb_shift) * 3)>>2, 1U); ++ ++ /* ++ * In-word depths in case some bfq_queue is being weight- ++ * raised: leaving ~63% of tags for sync reads. This is the ++ * highest percentage for which, in our tests, application ++ * start-up times didn't suffer from any regression due to tag ++ * shortage. ++ */ ++ /* no more than ~18% of tags for async I/O */ ++ bfqd->word_depths[1][0] = max(((1U<<bfqd->sb_shift) * 3)>>4, 1U); ++ /* no more than ~37% of tags for sync writes (~20% extra tags) */ ++ bfqd->word_depths[1][1] = max(((1U<<bfqd->sb_shift) * 6)>>4, 1U); ++} ++ ++/* ++ * Async I/O can easily starve sync I/O (both sync reads and sync ++ * writes), by consuming all tags. Similarly, storms of sync writes, ++ * such as those that sync(2) may trigger, can starve sync reads. ++ * Limit depths of async I/O and sync writes so as to counter both ++ * problems. ++ */ ++static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data) ++{ ++ struct blk_mq_tags *tags = blk_mq_tags_from_data(data); ++ struct bfq_data *bfqd = data->q->elevator->elevator_data; ++ struct sbitmap_queue *bt; ++ ++ if (op_is_sync(op) && !op_is_write(op)) ++ return; ++ ++ if (data->flags & BLK_MQ_REQ_RESERVED) { ++ if (unlikely(!tags->nr_reserved_tags)) { ++ WARN_ON_ONCE(1); ++ return; ++ } ++ bt = &tags->breserved_tags; ++ } else ++ bt = &tags->bitmap_tags; ++ ++ if (unlikely(bfqd->sb_shift != bt->sb.shift)) ++ bfq_update_depths(bfqd, bt); ++ ++ data->shallow_depth = ++ bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)]; ++ ++ bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", ++ __func__, bfqd->wr_busy_queues, op_is_sync(op), ++ data->shallow_depth); ++} ++ + static struct bfq_queue * + bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, + sector_t sector, struct rb_node **ret_parent, +@@ -5812,6 +5888,7 @@ static struct elv_fs_entry bfq_attrs[] = { + + static struct elevator_type iosched_bfq_mq = { + .ops.mq = { ++ .limit_depth = bfq_limit_depth, + .prepare_request = bfq_prepare_request, + .finish_request = bfq_finish_request, + .exit_icq = bfq_exit_icq, +diff --git a/block/bfq-mq.h b/block/bfq-mq.h +index a5947b203ef2..458099ee0308 100644 +--- a/block/bfq-mq.h ++++ b/block/bfq-mq.h +@@ -619,6 +619,18 @@ struct bfq_data { + struct bfq_queue *bio_bfqq; + /* Extra flag used only for TESTING */ + bool bio_bfqq_set; ++ ++ /* ++ * Cached sbitmap shift, used to compute depth limits in ++ * bfq_update_depths. ++ */ ++ unsigned int sb_shift; ++ ++ /* ++ * Depth limits used in bfq_limit_depth (see comments on the ++ * function) ++ */ ++ unsigned int word_depths[2][2]; + }; + + enum bfqq_state_flags { + +From 402e5f6b59662d290ab2b3c10b0016207a63ad21 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Thu, 21 Dec 2017 15:51:39 +0100 +Subject: [PATCH 11/23] bfq-sq, bfq-mq: limit sectors served with interactive + weight raising + +To maximise responsiveness, BFQ raises the weight, and performs device +idling, for bfq_queues associated with processes deemed as +interactive. In particular, weight raising has a maximum duration, +equal to the time needed to start a large application. If a +weight-raised process goes on doing I/O beyond this maximum duration, +it loses weight-raising. + +This mechanism is evidently vulnerable to the following false +positives: I/O-bound applications that will go on doing I/O for much +longer than the duration of weight-raising. These applications have +basically no benefit from being weight-raised at the beginning of +their I/O. On the opposite end, while being weight-raised, these +applications +a) unjustly steal throughput to applications that may truly need +low latency; +b) make BFQ uselessly perform device idling; device idling results +in loss of device throughput with most flash-based storage, and may +increase latencies when used purposelessly. + +This commit adds a countermeasure to reduce both the above +problems. To introduce this countermeasure, we provide the following +extra piece of information (full details in the comments added by this +commit). During the start-up of the large application used as a +reference to set the duration of weight-raising, involved processes +transfer at most ~110K sectors each. Accordingly, a process initially +deemed as interactive has no right to be weight-raised any longer, +once transferred 110K sectors or more. + +Basing on this consideration, this commit early-ends weight-raising +for a bfq_queue if the latter happens to have received an amount of +service at least equal to 110K sectors (actually, a little bit more, +to keep a safety margin). I/O-bound applications that reach a high +throughput, such as file copy, get to this threshold much before the +allowed weight-raising period finishes. Thus this early ending of +weight-raising reduces the amount of time during which these +applications cause the problems described above. + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 84 ++++++++++++++++++++++++++++++++++++++++++++------ + block/bfq-mq.h | 5 +++ + block/bfq-sched.c | 3 ++ + block/bfq-sq-iosched.c | 84 ++++++++++++++++++++++++++++++++++++++++++++------ + block/bfq.h | 5 +++ + 5 files changed, 163 insertions(+), 18 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index f384f5566672..63fdd16dec3c 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -162,15 +162,17 @@ static struct kmem_cache *bfq_pool; + * interactive applications automatically, using the following formula: + * duration = (R / r) * T, where r is the peak rate of the device, and + * R and T are two reference parameters. +- * In particular, R is the peak rate of the reference device (see below), +- * and T is a reference time: given the systems that are likely to be +- * installed on the reference device according to its speed class, T is +- * about the maximum time needed, under BFQ and while reading two files in +- * parallel, to load typical large applications on these systems. +- * In practice, the slower/faster the device at hand is, the more/less it +- * takes to load applications with respect to the reference device. +- * Accordingly, the longer/shorter BFQ grants weight raising to interactive +- * applications. ++ * In particular, R is the peak rate of the reference device (see ++ * below), and T is a reference time: given the systems that are ++ * likely to be installed on the reference device according to its ++ * speed class, T is about the maximum time needed, under BFQ and ++ * while reading two files in parallel, to load typical large ++ * applications on these systems (see the comments on ++ * max_service_from_wr below, for more details on how T is obtained). ++ * In practice, the slower/faster the device at hand is, the more/less ++ * it takes to load applications with respect to the reference device. ++ * Accordingly, the longer/shorter BFQ grants weight raising to ++ * interactive applications. + * + * BFQ uses four different reference pairs (R, T), depending on: + * . whether the device is rotational or non-rotational; +@@ -207,6 +209,60 @@ static int T_slow[2]; + static int T_fast[2]; + static int device_speed_thresh[2]; + ++/* ++ * BFQ uses the above-detailed, time-based weight-raising mechanism to ++ * privilege interactive tasks. This mechanism is vulnerable to the ++ * following false positives: I/O-bound applications that will go on ++ * doing I/O for much longer than the duration of weight ++ * raising. These applications have basically no benefit from being ++ * weight-raised at the beginning of their I/O. On the opposite end, ++ * while being weight-raised, these applications ++ * a) unjustly steal throughput to applications that may actually need ++ * low latency; ++ * b) make BFQ uselessly perform device idling; device idling results ++ * in loss of device throughput with most flash-based storage, and may ++ * increase latencies when used purposelessly. ++ * ++ * BFQ tries to reduce these problems, by adopting the following ++ * countermeasure. To introduce this countermeasure, we need first to ++ * finish explaining how the duration of weight-raising for ++ * interactive tasks is computed. ++ * ++ * For a bfq_queue deemed as interactive, the duration of weight ++ * raising is dynamically adjusted, as a function of the estimated ++ * peak rate of the device, so as to be equal to the time needed to ++ * execute the 'largest' interactive task we benchmarked so far. By ++ * largest task, we mean the task for which each involved process has ++ * to do more I/O than for any of the other tasks we benchmarked. This ++ * reference interactive task is the start-up of LibreOffice Writer, ++ * and in this task each process/bfq_queue needs to have at most ~110K ++ * sectors transferred. ++ * ++ * This last piece of information enables BFQ to reduce the actual ++ * duration of weight-raising for at least one class of I/O-bound ++ * applications: those doing sequential or quasi-sequential I/O. An ++ * example is file copy. In fact, once started, the main I/O-bound ++ * processes of these applications usually consume the above 110K ++ * sectors in much less time than the processes of an application that ++ * is starting, because these I/O-bound processes will greedily devote ++ * almost all their CPU cycles only to their target, ++ * throughput-friendly I/O operations. This is even more true if BFQ ++ * happens to be underestimating the device peak rate, and thus ++ * overestimating the duration of weight raising. But, according to ++ * our measurements, once transferred 110K sectors, these processes ++ * have no right to be weight-raised any longer. ++ * ++ * Basing on the last consideration, BFQ ends weight-raising for a ++ * bfq_queue if the latter happens to have received an amount of ++ * service at least equal to the following constant. The constant is ++ * set to slightly more than 110K, to have a minimum safety margin. ++ * ++ * This early ending of weight-raising reduces the amount of time ++ * during which interactive false positives cause the two problems ++ * described at the beginning of these comments. ++ */ ++static const unsigned long max_service_from_wr = 120000; ++ + #define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ + { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) + +@@ -1361,6 +1417,7 @@ static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, + if (old_wr_coeff == 1 && wr_or_deserves_wr) { + /* start a weight-raising period */ + if (interactive) { ++ bfqq->service_from_wr = 0; + bfqq->wr_coeff = bfqd->bfq_wr_coeff; + bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); + } else { +@@ -3980,6 +4037,15 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) + "back to interactive wr"); + } + } ++ if (bfqq->wr_coeff > 1 && ++ bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && ++ bfqq->service_from_wr > max_service_from_wr) { ++ /* see comments on max_service_from_wr */ ++ bfq_bfqq_end_wr(bfqq); ++ bfq_log_bfqq(bfqd, bfqq, ++ "[%s] too much service", ++ __func__); ++ } + } + /* + * To improve latency (for this or other queues), immediately +diff --git a/block/bfq-mq.h b/block/bfq-mq.h +index 458099ee0308..9a5ce1168ff5 100644 +--- a/block/bfq-mq.h ++++ b/block/bfq-mq.h +@@ -331,6 +331,11 @@ struct bfq_queue { + * last transition from idle to backlogged. + */ + unsigned long service_from_backlogged; ++ /* ++ * Cumulative service received from the @bfq_queue since its ++ * last transition to weight-raised state. ++ */ ++ unsigned long service_from_wr; + /* + * Value of wr start time when switching to soft rt + */ +diff --git a/block/bfq-sched.c b/block/bfq-sched.c +index 9d261dd428e4..4e6c5232e2fb 100644 +--- a/block/bfq-sched.c ++++ b/block/bfq-sched.c +@@ -942,6 +942,9 @@ static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) + if (!bfqq->service_from_backlogged) + bfqq->first_IO_time = jiffies; + ++ if (bfqq->wr_coeff > 1) ++ bfqq->service_from_wr += served; ++ + bfqq->service_from_backlogged += served; + for_each_entity(entity) { + st = bfq_entity_service_tree(entity); +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index eff4c4edf5a0..486493aafaf8 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -156,15 +156,17 @@ static struct kmem_cache *bfq_pool; + * interactive applications automatically, using the following formula: + * duration = (R / r) * T, where r is the peak rate of the device, and + * R and T are two reference parameters. +- * In particular, R is the peak rate of the reference device (see below), +- * and T is a reference time: given the systems that are likely to be +- * installed on the reference device according to its speed class, T is +- * about the maximum time needed, under BFQ and while reading two files in +- * parallel, to load typical large applications on these systems. +- * In practice, the slower/faster the device at hand is, the more/less it +- * takes to load applications with respect to the reference device. +- * Accordingly, the longer/shorter BFQ grants weight raising to interactive +- * applications. ++ * In particular, R is the peak rate of the reference device (see ++ * below), and T is a reference time: given the systems that are ++ * likely to be installed on the reference device according to its ++ * speed class, T is about the maximum time needed, under BFQ and ++ * while reading two files in parallel, to load typical large ++ * applications on these systems (see the comments on ++ * max_service_from_wr below, for more details on how T is obtained). ++ * In practice, the slower/faster the device at hand is, the more/less ++ * it takes to load applications with respect to the reference device. ++ * Accordingly, the longer/shorter BFQ grants weight raising to ++ * interactive applications. + * + * BFQ uses four different reference pairs (R, T), depending on: + * . whether the device is rotational or non-rotational; +@@ -201,6 +203,60 @@ static int T_slow[2]; + static int T_fast[2]; + static int device_speed_thresh[2]; + ++/* ++ * BFQ uses the above-detailed, time-based weight-raising mechanism to ++ * privilege interactive tasks. This mechanism is vulnerable to the ++ * following false positives: I/O-bound applications that will go on ++ * doing I/O for much longer than the duration of weight ++ * raising. These applications have basically no benefit from being ++ * weight-raised at the beginning of their I/O. On the opposite end, ++ * while being weight-raised, these applications ++ * a) unjustly steal throughput to applications that may actually need ++ * low latency; ++ * b) make BFQ uselessly perform device idling; device idling results ++ * in loss of device throughput with most flash-based storage, and may ++ * increase latencies when used purposelessly. ++ * ++ * BFQ tries to reduce these problems, by adopting the following ++ * countermeasure. To introduce this countermeasure, we need first to ++ * finish explaining how the duration of weight-raising for ++ * interactive tasks is computed. ++ * ++ * For a bfq_queue deemed as interactive, the duration of weight ++ * raising is dynamically adjusted, as a function of the estimated ++ * peak rate of the device, so as to be equal to the time needed to ++ * execute the 'largest' interactive task we benchmarked so far. By ++ * largest task, we mean the task for which each involved process has ++ * to do more I/O than for any of the other tasks we benchmarked. This ++ * reference interactive task is the start-up of LibreOffice Writer, ++ * and in this task each process/bfq_queue needs to have at most ~110K ++ * sectors transfered. ++ * ++ * This last piece of information enables BFQ to reduce the actual ++ * duration of weight-raising for at least one class of I/O-bound ++ * applications: those doing sequential or quasi-sequential I/O. An ++ * example is file copy. In fact, once started, the main I/O-bound ++ * processes of these applications usually consume the above 110K ++ * sectors in much less time than the processes of an application that ++ * is starting, because these I/O-bound processes will greedily devote ++ * almost all their CPU cycles only to their target, ++ * throughput-friendly I/O operations. This is even more true if BFQ ++ * happens to be underestimating the device peak rate, and thus ++ * overestimating the duration of weight raising. But, according to ++ * our measurements, once transferred 110K sectors, these processes ++ * have no right to be weight-raised any longer. ++ * ++ * Basing on the last consideration, BFQ ends weight-raising for a ++ * bfq_queue if the latter happens to have received an amount of ++ * service at least equal to the following constant. The constant is ++ * set to slightly more than 110K, to have a minimum safety margin. ++ * ++ * This early ending of weight-raising reduces the amount of time ++ * during which interactive false positives cause the two problems ++ * described at the beginning of these comments. ++ */ ++static const unsigned long max_service_from_wr = 120000; ++ + #define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ + { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) + +@@ -1246,6 +1302,7 @@ static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, + if (old_wr_coeff == 1 && wr_or_deserves_wr) { + /* start a weight-raising period */ + if (interactive) { ++ bfqq->service_from_wr = 0; + bfqq->wr_coeff = bfqd->bfq_wr_coeff; + bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); + } else { +@@ -3794,6 +3851,15 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) + "back to interactive wr"); + } + } ++ if (bfqq->wr_coeff > 1 && ++ bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && ++ bfqq->service_from_wr > max_service_from_wr) { ++ /* see comments on max_service_from_wr */ ++ bfq_bfqq_end_wr(bfqq); ++ bfq_log_bfqq(bfqd, bfqq, ++ "[%s] too much service", ++ __func__); ++ } + } + /* + * To improve latency (for this or other queues), immediately +diff --git a/block/bfq.h b/block/bfq.h +index 59539adc00a5..0cd7a3f251a7 100644 +--- a/block/bfq.h ++++ b/block/bfq.h +@@ -323,6 +323,11 @@ struct bfq_queue { + * last transition from idle to backlogged. + */ + unsigned long service_from_backlogged; ++ /* ++ * Cumulative service received from the @bfq_queue since its ++ * last transition to weight-raised state. ++ */ ++ unsigned long service_from_wr; + /* + * Value of wr start time when switching to soft rt + */ + +From 59efebb94b2f9bac653faf62dadb45b83bd27fa7 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Thu, 4 Jan 2018 16:29:58 +0100 +Subject: [PATCH 12/23] bfq-sq, bfq-mq: put async queues for root bfq groups + too +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +For each pair [device for which bfq is selected as I/O scheduler, +group in blkio/io], bfq maintains a corresponding bfq group. Each such +bfq group contains a set of async queues, with each async queue +created on demand, i.e., when some I/O request arrives for it. On +creation, an async queue gets an extra reference, to make sure that +the queue is not freed as long as its bfq group exists. Accordingly, +to allow the queue to be freed after the group exited, this extra +reference must released on group exit. + +The above holds also for a bfq root group, i.e., for the bfq group +corresponding to the root blkio/io root for a given device. Yet, by +mistake, the references to the existing async queues of a root group +are not released when the latter exits. This causes a memory leak when +the instance of bfq for a given device exits. In a similar vein, +bfqg_stats_xfer_dead is not executed for a root group. + +This commit fixes bfq_pd_offline so that the latter executes the above +missing operations for a root group too. + +Reported-by: Holger Hoffstätte <holger@applied-asynchrony.com> +Reported-by: Guoqing Jiang <gqjiang@suse.com> +Signed-off-by: Davide Ferrari <davideferrari8@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-cgroup-included.c | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/block/bfq-cgroup-included.c b/block/bfq-cgroup-included.c +index 562b0ce581a7..45fefb2e2d57 100644 +--- a/block/bfq-cgroup-included.c ++++ b/block/bfq-cgroup-included.c +@@ -885,13 +885,13 @@ static void bfq_pd_offline(struct blkg_policy_data *pd) + + entity = bfqg->my_entity; + +- if (!entity) /* root group */ +- return; +- + #ifdef BFQ_MQ + spin_lock_irqsave(&bfqd->lock, flags); + #endif + ++ if (!entity) /* root group */ ++ goto put_async_queues; ++ + /* + * Empty all service_trees belonging to this group before + * deactivating the group itself. +@@ -926,6 +926,8 @@ static void bfq_pd_offline(struct blkg_policy_data *pd) + BUG_ON(bfqg->sched_data.in_service_entity); + + __bfq_deactivate_entity(entity, false); ++ ++put_async_queues: + bfq_put_async_queues(bfqd, bfqg); + + #ifdef BFQ_MQ + +From 2dfbaaaf95054e2da3ededc0deb1ba5a4f589e53 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Mon, 8 Jan 2018 19:38:45 +0100 +Subject: [PATCH 13/23] bfq-sq, bfq-mq: release oom-queue ref to root group on + exit + +On scheduler init, a reference to the root group, and a reference to +its corresponding blkg are taken for the oom queue. Yet these +references are not released on scheduler exit, which prevents these +objects from be freed. This commit adds the missing reference +releases. + +Reported-by: Davide Ferrari <davideferrari8@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 3 +++ + block/bfq-sq-iosched.c | 3 +++ + 2 files changed, 6 insertions(+) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 63fdd16dec3c..b82c52fabf91 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -5507,6 +5507,9 @@ static void bfq_exit_queue(struct elevator_queue *e) + + BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); + ++ /* release oom-queue reference to root group */ ++ bfqg_and_blkg_put(bfqd->root_group); ++ + #ifdef BFQ_GROUP_IOSCHED_ENABLED + blkcg_deactivate_policy(bfqd->queue, &blkcg_policy_bfq); + #else +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index 486493aafaf8..851af055664d 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -5052,6 +5052,9 @@ static void bfq_exit_queue(struct elevator_queue *e) + + BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); + ++ /* release oom-queue reference to root group */ ++ bfqg_put(bfqd->root_group); ++ + #ifdef BFQ_GROUP_IOSCHED_ENABLED + blkcg_deactivate_policy(q, &blkcg_policy_bfq); + #else + +From 13efe00c8292d78d223e1090a7f36426e360eb38 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Mon, 8 Jan 2018 19:40:38 +0100 +Subject: [PATCH 14/23] block, bfq-sq, bfq-mq: trace get and put of bfq groups + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-cgroup-included.c | 15 +++++++++++++++ + block/bfq-mq-iosched.c | 3 ++- + 2 files changed, 17 insertions(+), 1 deletion(-) + +diff --git a/block/bfq-cgroup-included.c b/block/bfq-cgroup-included.c +index 45fefb2e2d57..f94743fb2e7d 100644 +--- a/block/bfq-cgroup-included.c ++++ b/block/bfq-cgroup-included.c +@@ -267,6 +267,8 @@ static struct bfq_group *bfqq_group(struct bfq_queue *bfqq) + + static void bfqg_get(struct bfq_group *bfqg) + { ++ trace_printk("bfqg %p\n", bfqg); ++ + #ifdef BFQ_MQ + bfqg->ref++; + #else +@@ -280,6 +282,9 @@ static void bfqg_put(struct bfq_group *bfqg) + bfqg->ref--; + + BUG_ON(bfqg->ref < 0); ++ trace_printk("putting bfqg %p %s\n", bfqg, ++ bfqg->ref == 0 ? "and freeing it" : ""); ++ + if (bfqg->ref == 0) + kfree(bfqg); + #else +@@ -293,6 +298,7 @@ static void bfqg_and_blkg_get(struct bfq_group *bfqg) + /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */ + bfqg_get(bfqg); + ++ trace_printk("getting blkg for bfqg %p\n", bfqg); + blkg_get(bfqg_to_blkg(bfqg)); + } + +@@ -300,6 +306,7 @@ static void bfqg_and_blkg_put(struct bfq_group *bfqg) + { + bfqg_put(bfqg); + ++ trace_printk("putting blkg for bfqg %p\n", bfqg); + blkg_put(bfqg_to_blkg(bfqg)); + } + #endif +@@ -382,6 +389,8 @@ static void bfq_init_entity(struct bfq_entity *entity, + * Make sure that bfqg and its associated blkg do not + * disappear before entity. + */ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] getting bfqg %p and blkg\n", __func__, bfqg); ++ + bfqg_and_blkg_get(bfqg); + #else + bfqg_get(bfqg); +@@ -475,6 +484,7 @@ static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node) + kfree(bfqg); + return NULL; + } ++ trace_printk("bfqg %p\n", bfqg); + + #ifdef BFQ_MQ + /* see comments in bfq_bic_update_cgroup for why refcounting */ +@@ -513,6 +523,7 @@ static void bfq_pd_init(struct blkg_policy_data *pd) + static void bfq_pd_free(struct blkg_policy_data *pd) + { + struct bfq_group *bfqg = pd_to_bfqg(pd); ++ trace_printk("bfqg %p\n", bfqg); + + bfqg_stats_exit(&bfqg->stats); + #ifdef BFQ_MQ +@@ -650,6 +661,8 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bfq_put_idle_entity(bfq_entity_service_tree(entity), entity); + } + #ifdef BFQ_MQ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] putting blkg and bfqg %p\n", __func__, bfqg); ++ + bfqg_and_blkg_put(bfqq_group(bfqq)); + #else + bfqg_put(bfqq_group(bfqq)); +@@ -658,6 +671,8 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, + entity->parent = bfqg->my_entity; + entity->sched_data = &bfqg->sched_data; + #ifdef BFQ_MQ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] getting blkg and bfqg %p\n", __func__, bfqg); ++ + /* pin down bfqg and its associated blkg */ + bfqg_and_blkg_get(bfqg); + #else +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index b82c52fabf91..d5b7a6b985d7 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -4385,10 +4385,11 @@ static void bfq_put_queue(struct bfq_queue *bfqq) + if (bfqq->bfqd) + bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p freed", bfqq); + +- kmem_cache_free(bfq_pool, bfqq); + #ifdef BFQ_GROUP_IOSCHED_ENABLED ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] putting blkg and bfqg %p\n", __func__, bfqg); + bfqg_and_blkg_put(bfqg); + #endif ++ kmem_cache_free(bfq_pool, bfqq); + } + + static void bfq_put_cooperator(struct bfq_queue *bfqq) + +From 816b77fba966171974eb5ee25d81bc4e19eaf1b4 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Wed, 10 Jan 2018 09:08:22 +0100 +Subject: [PATCH 15/23] bfq-sq, bfq-mq: compile group put for oom queue only if + BFQ_GROUP_IOSCHED is set + +Commit ("bfq-sq, bfq-mq: release oom-queue ref to root group on exit") +added a missing put of the root bfq group for the oom queue. That put +has to be, and can be, performed only if CONFIG_BFQ_GROUP_IOSCHED is +defined: the function doing the put is even not defined at all if +CONFIG_BFQ_GROUP_IOSCHED is not defined. But that commit makes that +put be invoked regardless of whether CONFIG_BFQ_GROUP_IOSCHED is +defined. This commit fixes this mistake, by making that invocation be +compiled only if CONFIG_BFQ_GROUP_IOSCHED is actually defined. + +Fixes ("block, bfq: release oom-queue ref to root group on exit") +Reported-by: Jan Alexander Steffens <jan.steffens@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 2 +- + block/bfq-sq-iosched.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index d5b7a6b985d7..2581fe0f6f2f 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -5508,10 +5508,10 @@ static void bfq_exit_queue(struct elevator_queue *e) + + BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); + ++#ifdef BFQ_GROUP_IOSCHED_ENABLED + /* release oom-queue reference to root group */ + bfqg_and_blkg_put(bfqd->root_group); + +-#ifdef BFQ_GROUP_IOSCHED_ENABLED + blkcg_deactivate_policy(bfqd->queue, &blkcg_policy_bfq); + #else + spin_lock_irq(&bfqd->lock); +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index 851af055664d..c4df156b1fb4 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -5052,10 +5052,10 @@ static void bfq_exit_queue(struct elevator_queue *e) + + BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); + ++#ifdef BFQ_GROUP_IOSCHED_ENABLED + /* release oom-queue reference to root group */ + bfqg_put(bfqd->root_group); + +-#ifdef BFQ_GROUP_IOSCHED_ENABLED + blkcg_deactivate_policy(q, &blkcg_policy_bfq); + #else + bfq_put_async_queues(bfqd, bfqd->root_group); + +From 643a89c659172b2c9ae16adfe03af4e3e88e1326 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Sat, 13 Jan 2018 18:48:41 +0100 +Subject: [PATCH 16/23] block, bfq-sq, bfq-mq: remove trace_printks + +Commit ("block, bfq-sq, bfq-mq: trace get and put of bfq groups") +unwisely added some invocations of the function trace_printk, which +is inappropriate in production kernels. This commit removes those +invocations. + +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-cgroup-included.c | 10 ---------- + 1 file changed, 10 deletions(-) + +diff --git a/block/bfq-cgroup-included.c b/block/bfq-cgroup-included.c +index f94743fb2e7d..a4f8a03edfc9 100644 +--- a/block/bfq-cgroup-included.c ++++ b/block/bfq-cgroup-included.c +@@ -267,8 +267,6 @@ static struct bfq_group *bfqq_group(struct bfq_queue *bfqq) + + static void bfqg_get(struct bfq_group *bfqg) + { +- trace_printk("bfqg %p\n", bfqg); +- + #ifdef BFQ_MQ + bfqg->ref++; + #else +@@ -282,9 +280,6 @@ static void bfqg_put(struct bfq_group *bfqg) + bfqg->ref--; + + BUG_ON(bfqg->ref < 0); +- trace_printk("putting bfqg %p %s\n", bfqg, +- bfqg->ref == 0 ? "and freeing it" : ""); +- + if (bfqg->ref == 0) + kfree(bfqg); + #else +@@ -298,7 +293,6 @@ static void bfqg_and_blkg_get(struct bfq_group *bfqg) + /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */ + bfqg_get(bfqg); + +- trace_printk("getting blkg for bfqg %p\n", bfqg); + blkg_get(bfqg_to_blkg(bfqg)); + } + +@@ -306,7 +300,6 @@ static void bfqg_and_blkg_put(struct bfq_group *bfqg) + { + bfqg_put(bfqg); + +- trace_printk("putting blkg for bfqg %p\n", bfqg); + blkg_put(bfqg_to_blkg(bfqg)); + } + #endif +@@ -484,8 +477,6 @@ static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node) + kfree(bfqg); + return NULL; + } +- trace_printk("bfqg %p\n", bfqg); +- + #ifdef BFQ_MQ + /* see comments in bfq_bic_update_cgroup for why refcounting */ + bfqg_get(bfqg); +@@ -523,7 +514,6 @@ static void bfq_pd_init(struct blkg_policy_data *pd) + static void bfq_pd_free(struct blkg_policy_data *pd) + { + struct bfq_group *bfqg = pd_to_bfqg(pd); +- trace_printk("bfqg %p\n", bfqg); + + bfqg_stats_exit(&bfqg->stats); + #ifdef BFQ_MQ + +From ce050275e24fecec800f346c09d9494563e9fc8a Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Mon, 15 Jan 2018 15:07:05 +0100 +Subject: [PATCH 17/23] block, bfq-mq: add requeue-request hook + +Commit 'a6a252e64914 ("blk-mq-sched: decide how to handle flush rq via +RQF_FLUSH_SEQ")' makes all non-flush re-prepared requests for a device +be re-inserted into the active I/O scheduler for that device. As a +consequence, I/O schedulers may get the same request inserted again, +even several times, without a finish_request invoked on that request +before each re-insertion. + +This fact is the cause of the failure reported in [1]. For an I/O +scheduler, every re-insertion of the same re-prepared request is +equivalent to the insertion of a new request. For schedulers like +mq-deadline or kyber, this fact causes no harm. In contrast, it +confuses a stateful scheduler like BFQ, which keeps state for an I/O +request, until the finish_request hook is invoked on the request. In +particular, BFQ may get stuck, waiting forever for the number of +request dispatches, of the same request, to be balanced by an equal +number of request completions (while there will be one completion for +that request). In this state, BFQ may refuse to serve I/O requests +from other bfq_queues. The hang reported in [1] then follows. + +However, the above re-prepared requests undergo a requeue, thus the +requeue_request hook of the active elevator is invoked for these +requests, if set. This commit then addresses the above issue by +properly implementing the hook requeue_request in BFQ. + +[1] https://marc.info/?l=linux-block&m=151211117608676 + +Reported-by: Ivan Kozik <ivan@ludios.org> +Reported-by: Alban Browaeys <alban.browaeys@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +Signed-off-by: Serena Ziviani <ziviani.serena@gmail.com> +--- + block/bfq-mq-iosched.c | 90 ++++++++++++++++++++++++++++++++++++++++---------- + 1 file changed, 73 insertions(+), 17 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 2581fe0f6f2f..bb7ccc2f1165 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -4162,9 +4162,9 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + * TESTING: reset DISP_LIST flag, because: 1) + * this rq this request has passed through + * bfq_prepare_request, 2) then it will have +- * bfq_finish_request invoked on it, and 3) in +- * bfq_finish_request we use this flag to check +- * that bfq_finish_request is not invoked on ++ * bfq_finish_requeue_request invoked on it, and 3) in ++ * bfq_finish_requeue_request we use this flag to check ++ * that bfq_finish_requeue_request is not invoked on + * requests for which bfq_prepare_request has + * been invoked. + */ +@@ -4173,8 +4173,8 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + } + + /* +- * We exploit the bfq_finish_request hook to decrement +- * rq_in_driver, but bfq_finish_request will not be ++ * We exploit the bfq_finish_requeue_request hook to decrement ++ * rq_in_driver, but bfq_finish_requeue_request will not be + * invoked on this request. So, to avoid unbalance, + * just start this request, without incrementing + * rq_in_driver. As a negative consequence, +@@ -4183,10 +4183,10 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + * bfq_schedule_dispatch to be invoked uselessly. + * + * As for implementing an exact solution, the +- * bfq_finish_request hook, if defined, is probably ++ * bfq_finish_requeue_request hook, if defined, is probably + * invoked also on this request. So, by exploiting + * this hook, we could 1) increment rq_in_driver here, +- * and 2) decrement it in bfq_finish_request. Such a ++ * and 2) decrement it in bfq_finish_requeue_request. Such a + * solution would let the value of the counter be + * always accurate, but it would entail using an extra + * interface function. This cost seems higher than the +@@ -4878,6 +4878,8 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) + return idle_timer_disabled; + } + ++static void bfq_prepare_request(struct request *rq, struct bio *bio); ++ + static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + bool at_head) + { +@@ -4919,6 +4921,20 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + BUG_ON(!(rq->rq_flags & RQF_GOT)); + rq->rq_flags &= ~RQF_GOT; + ++ if (!bfqq) { ++ /* ++ * This should never happen. Most likely rq is ++ * a requeued regular request, being ++ * re-inserted without being first ++ * re-prepared. Do a prepare, to avoid ++ * failure. ++ */ ++ pr_warn("Regular request associated with no queue"); ++ WARN_ON(1); ++ bfq_prepare_request(rq, rq->bio); ++ bfqq = RQ_BFQQ(rq); ++ } ++ + #if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) + idle_timer_disabled = __bfq_insert_request(bfqd, rq); + /* +@@ -5110,7 +5126,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) + } + } + +-static void bfq_finish_request_body(struct bfq_queue *bfqq) ++static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqq->bfqd, bfqq, + "put_request_body: allocated %d", bfqq->allocated); +@@ -5120,7 +5136,13 @@ static void bfq_finish_request_body(struct bfq_queue *bfqq) + bfq_put_queue(bfqq); + } + +-static void bfq_finish_request(struct request *rq) ++/* ++ * Handle either a requeue or a finish for rq. The things to do are ++ * the same in both cases: all references to rq are to be dropped. In ++ * particular, rq is considered completed from the point of view of ++ * the scheduler. ++ */ ++static void bfq_finish_requeue_request(struct request *rq) + { + struct bfq_queue *bfqq; + struct bfq_data *bfqd; +@@ -5128,11 +5150,27 @@ static void bfq_finish_request(struct request *rq) + + BUG_ON(!rq); + +- if (!rq->elv.icq) ++ bfqq = RQ_BFQQ(rq); ++ ++ /* ++ * Requeue and finish hooks are invoked in blk-mq without ++ * checking whether the involved request is actually still ++ * referenced in the scheduler. To handle this fact, the ++ * following two checks make this function exit in case of ++ * spurious invocations, for which there is nothing to do. ++ * ++ * First, check whether rq has nothing to do with an elevator. ++ */ ++ if (unlikely(!(rq->rq_flags & RQF_ELVPRIV))) + return; + +- bfqq = RQ_BFQQ(rq); +- BUG_ON(!bfqq); ++ /* ++ * rq either is not associated with any icq, or is an already ++ * requeued request that has not (yet) been re-inserted into ++ * a bfq_queue. ++ */ ++ if (!rq->elv.icq || !bfqq) ++ return; + + bic = RQ_BIC(rq); + BUG_ON(!bic); +@@ -5145,7 +5183,6 @@ static void bfq_finish_request(struct request *rq) + BUG(); + } + BUG_ON(rq->rq_flags & RQF_QUEUED); +- BUG_ON(!(rq->rq_flags & RQF_ELVPRIV)); + + bfq_log_bfqq(bfqd, bfqq, + "putting rq %p with %u sects left, STARTED %d", +@@ -5166,13 +5203,14 @@ static void bfq_finish_request(struct request *rq) + spin_lock_irqsave(&bfqd->lock, flags); + + bfq_completed_request(bfqq, bfqd); +- bfq_finish_request_body(bfqq); ++ bfq_finish_requeue_request_body(bfqq); + + spin_unlock_irqrestore(&bfqd->lock, flags); + } else { + /* + * Request rq may be still/already in the scheduler, +- * in which case we need to remove it. And we cannot ++ * in which case we need to remove it (this should ++ * never happen in case of requeue). And we cannot + * defer such a check and removal, to avoid + * inconsistencies in the time interval from the end + * of this function to the start of the deferred work. +@@ -5189,9 +5227,26 @@ static void bfq_finish_request(struct request *rq) + bfqg_stats_update_io_remove(bfqq_group(bfqq), + rq->cmd_flags); + } +- bfq_finish_request_body(bfqq); ++ bfq_finish_requeue_request_body(bfqq); + } + ++ /* ++ * Reset private fields. In case of a requeue, this allows ++ * this function to correctly do nothing if it is spuriously ++ * invoked again on this same request (see the check at the ++ * beginning of the function). Probably, a better general ++ * design would be to prevent blk-mq from invoking the requeue ++ * or finish hooks of an elevator, for a request that is not ++ * referred by that elevator. ++ * ++ * Resetting the following fields would break the ++ * request-insertion logic if rq is re-inserted into a bfq ++ * internal queue, without a re-preparation. Here we assume ++ * that re-insertions of requeued requests, without ++ * re-preparation, can happen only for pass_through or at_head ++ * requests (which are not re-inserted into bfq internal ++ * queues). ++ */ + rq->elv.priv[0] = NULL; + rq->elv.priv[1] = NULL; + } +@@ -5960,7 +6015,8 @@ static struct elevator_type iosched_bfq_mq = { + .ops.mq = { + .limit_depth = bfq_limit_depth, + .prepare_request = bfq_prepare_request, +- .finish_request = bfq_finish_request, ++ .requeue_request = bfq_finish_requeue_request, ++ .finish_request = bfq_finish_requeue_request, + .exit_icq = bfq_exit_icq, + .insert_requests = bfq_insert_requests, + .dispatch_request = bfq_dispatch_request, + +From 3e4f292191cc62b3844316b9741534c3f1b36f0a Mon Sep 17 00:00:00 2001 +From: Davide Paganelli <paga.david@gmail.com> +Date: Thu, 8 Feb 2018 12:19:24 +0100 +Subject: [PATCH 18/23] block, bfq-mq, bfq-sq: make log functions print names + of calling functions + +Add the macro __func__ as a parameter to the invocations of the functions +pr_crit, blk_add_trace_msg and blk_add_cgroup_trace_msg in bfq_log* +functions, in order to include automatically in the log messages +the names of the functions that call the log functions. +The programmer can then avoid doing it. + +Signed-off-by: Davide Paganelli <paga.david@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-cgroup-included.c | 9 +-- + block/bfq-mq-iosched.c | 167 ++++++++++++++++++++++---------------------- + block/bfq-mq.h | 33 ++++----- + block/bfq-sched.c | 54 +++++++------- + block/bfq-sq-iosched.c | 134 +++++++++++++++++------------------ + block/bfq.h | 33 ++++----- + 6 files changed, 214 insertions(+), 216 deletions(-) + +diff --git a/block/bfq-cgroup-included.c b/block/bfq-cgroup-included.c +index a4f8a03edfc9..613f154e9da5 100644 +--- a/block/bfq-cgroup-included.c ++++ b/block/bfq-cgroup-included.c +@@ -382,7 +382,8 @@ static void bfq_init_entity(struct bfq_entity *entity, + * Make sure that bfqg and its associated blkg do not + * disappear before entity. + */ +- bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] getting bfqg %p and blkg\n", __func__, bfqg); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "getting bfqg %p and blkg\n", ++ bfqg); + + bfqg_and_blkg_get(bfqg); + #else +@@ -651,7 +652,7 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bfq_put_idle_entity(bfq_entity_service_tree(entity), entity); + } + #ifdef BFQ_MQ +- bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] putting blkg and bfqg %p\n", __func__, bfqg); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "putting blkg and bfqg %p\n", bfqg); + + bfqg_and_blkg_put(bfqq_group(bfqq)); + #else +@@ -661,7 +662,7 @@ static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, + entity->parent = bfqg->my_entity; + entity->sched_data = &bfqg->sched_data; + #ifdef BFQ_MQ +- bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] getting blkg and bfqg %p\n", __func__, bfqg); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "getting blkg and bfqg %p\n", bfqg); + + /* pin down bfqg and its associated blkg */ + bfqg_and_blkg_get(bfqg); +@@ -721,7 +722,7 @@ static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd, + if (entity->sched_data != &bfqg->sched_data) { + bic_set_bfqq(bic, NULL, 0); + bfq_log_bfqq(bfqd, async_bfqq, +- "bic_change_group: %p %d", ++ "%p %d", + async_bfqq, + async_bfqq->ref); + bfq_put_queue(async_bfqq); +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index bb7ccc2f1165..edc93b6af186 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -310,7 +310,7 @@ static struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, + static void bfq_schedule_dispatch(struct bfq_data *bfqd) + { + if (bfqd->queued != 0) { +- bfq_log(bfqd, "schedule dispatch"); ++ bfq_log(bfqd, ""); + blk_mq_run_hw_queues(bfqd->queue, true); + } + } +@@ -489,8 +489,8 @@ static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data) + data->shallow_depth = + bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)]; + +- bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", +- __func__, bfqd->wr_busy_queues, op_is_sync(op), ++ bfq_log(bfqd, "wr_busy %d sync %d depth %u", ++ bfqd->wr_busy_queues, op_is_sync(op), + data->shallow_depth); + } + +@@ -528,7 +528,7 @@ bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, + if (rb_link) + *rb_link = p; + +- bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", ++ bfq_log(bfqd, "%llu: returning %d", + (unsigned long long) sector, + bfqq ? bfqq->pid : 0); + +@@ -749,7 +749,7 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq, + if (rq == last || ktime_get_ns() < rq->fifo_time) + return NULL; + +- bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "returned %p", rq); + BUG_ON(RB_EMPTY_NODE(&rq->rb_node)); + return rq; + } +@@ -842,7 +842,7 @@ static void bfq_updated_next_req(struct bfq_data *bfqd, + bfq_serv_to_charge(next_rq, bfqq)); + if (entity->budget != new_budget) { + entity->budget = new_budget; +- bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", ++ bfq_log_bfqq(bfqd, bfqq, "new budget %lu", + new_budget); + bfq_requeue_bfqq(bfqd, bfqq, false); + } +@@ -915,8 +915,7 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, + BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); + + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "[%s] bic %p wr_coeff %d start_finish %lu max_time %lu", +- __func__, ++ "bic %p wr_coeff %d start_finish %lu max_time %lu", + bic, bfqq->wr_coeff, bfqq->last_wr_start_finish, + bfqq->wr_cur_max_time); + +@@ -929,11 +928,11 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, + bfq_wr_duration(bfqd))) { + switch_back_to_interactive_wr(bfqq, bfqd); + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "resume state: switching back to interactive"); ++ "switching back to interactive"); + } else { + bfqq->wr_coeff = 1; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "resume state: switching off wr (%lu + %lu < %lu)", ++ "switching off wr (%lu + %lu < %lu)", + bfqq->last_wr_start_finish, bfqq->wr_cur_max_time, + jiffies); + } +@@ -985,7 +984,7 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + /* Increment burst size to take into account also bfqq */ + bfqd->burst_size++; + +- bfq_log_bfqq(bfqd, bfqq, "add_to_burst %d", bfqd->burst_size); ++ bfq_log_bfqq(bfqd, bfqq, "%d", bfqd->burst_size); + + BUG_ON(bfqd->burst_size > bfqd->bfq_large_burst_thresh); + +@@ -998,7 +997,7 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + * other to consider this burst as large. + */ + bfqd->large_burst = true; +- bfq_log_bfqq(bfqd, bfqq, "add_to_burst: large burst started"); ++ bfq_log_bfqq(bfqd, bfqq, "large burst started"); + + /* + * We can now mark all queues in the burst list as +@@ -1170,7 +1169,7 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + bfqd->large_burst = false; + bfq_reset_burst_list(bfqd, bfqq); + bfq_log_bfqq(bfqd, bfqq, +- "handle_burst: late activation or different group"); ++ "late activation or different group"); + goto end; + } + +@@ -1180,7 +1179,7 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + * bfqq as belonging to this large burst immediately. + */ + if (bfqd->large_burst) { +- bfq_log_bfqq(bfqd, bfqq, "handle_burst: marked in burst"); ++ bfq_log_bfqq(bfqd, bfqq, "marked in burst"); + bfq_mark_bfqq_in_large_burst(bfqq); + goto end; + } +@@ -1686,7 +1685,7 @@ static void bfq_add_request(struct request *rq) + unsigned int old_wr_coeff = bfqq->wr_coeff; + bool interactive = false; + +- bfq_log_bfqq(bfqd, bfqq, "add_request: size %u %s", ++ bfq_log_bfqq(bfqd, bfqq, "size %u %s", + blk_rq_sectors(rq), rq_is_sync(rq) ? "S" : "A"); + + if (bfqq->wr_coeff > 1) /* queue is being weight-raised */ +@@ -1952,7 +1951,7 @@ static int bfq_request_merge(struct request_queue *q, struct request **req, + __rq = bfq_find_rq_fmerge(bfqd, bio, q); + if (__rq && elv_bio_merge_ok(__rq, bio)) { + *req = __rq; +- bfq_log(bfqd, "request_merge: req %p", __rq); ++ bfq_log(bfqd, "req %p", __rq); + + return ELEVATOR_FRONT_MERGE; + } +@@ -1989,7 +1988,7 @@ static void bfq_request_merged(struct request_queue *q, struct request *req, + bfqq->next_rq = next_rq; + + bfq_log_bfqq(bfqd, bfqq, +- "request_merged: req %p prev %p next_rq %p bfqq %p", ++ "req %p prev %p next_rq %p bfqq %p", + req, prev, next_rq, bfqq); + + /* +@@ -2018,7 +2017,7 @@ static void bfq_requests_merged(struct request_queue *q, struct request *rq, + goto end; + + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "requests_merged: rq %p next %p bfqq %p next_bfqq %p", ++ "rq %p next %p bfqq %p next_bfqq %p", + rq, next, bfqq, next_bfqq); + + spin_lock_irq(&bfqq->bfqd->lock); +@@ -2069,10 +2068,10 @@ static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) + */ + bfqq->entity.prio_changed = 1; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "end_wr: wrais ending at %lu, rais_max_time %u", ++ "wrais ending at %lu, rais_max_time %u", + bfqq->last_wr_start_finish, + jiffies_to_msecs(bfqq->wr_cur_max_time)); +- bfq_log_bfqq(bfqq->bfqd, bfqq, "end_wr: wr_busy %d", ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "wr_busy %d", + bfqq->bfqd->wr_busy_queues); + } + +@@ -2245,8 +2244,8 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, + { + if (bfq_too_late_for_merging(new_bfqq)) { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "[%s] too late for bfq%d to be merged", +- __func__, new_bfqq->pid); ++ "too late for bfq%d to be merged", ++ new_bfqq->pid); + return false; + } + +@@ -2395,8 +2394,7 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq) + } + BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "[%s] bic %p wr_coeff %d start_finish %lu max_time %lu", +- __func__, ++ "bic %p wr_coeff %d start_finish %lu max_time %lu", + bic, bfqq->wr_coeff, bfqq->last_wr_start_finish, + bfqq->wr_cur_max_time); + } +@@ -2453,7 +2451,7 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, + + } + +- bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", ++ bfq_log_bfqq(bfqd, new_bfqq, "wr_busy %d", + bfqd->wr_busy_queues); + + /* +@@ -2554,7 +2552,7 @@ static void bfq_set_budget_timeout(struct bfq_data *bfqd, + bfqq->budget_timeout = jiffies + + bfqd->bfq_timeout * timeout_coeff; + +- bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u", ++ bfq_log_bfqq(bfqd, bfqq, "%u", + jiffies_to_msecs(bfqd->bfq_timeout * timeout_coeff)); + } + +@@ -2620,10 +2618,10 @@ static void __bfq_set_in_service_queue(struct bfq_data *bfqd, + + bfq_set_budget_timeout(bfqd, bfqq); + bfq_log_bfqq(bfqd, bfqq, +- "set_in_service_queue, cur-budget = %d", ++ "cur-budget = %d", + bfqq->entity.budget); + } else +- bfq_log(bfqd, "set_in_service_queue: NULL"); ++ bfq_log(bfqd, "NULL"); + + bfqd->in_service_queue = bfqq; + } +@@ -2746,7 +2744,7 @@ static void bfq_reset_rate_computation(struct bfq_data *bfqd, struct request *rq + bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ + + bfq_log(bfqd, +- "reset_rate_computation at end, sample %u/%u tot_sects %llu", ++ "at end, sample %u/%u tot_sects %llu", + bfqd->peak_rate_samples, bfqd->sequential_samples, + bfqd->tot_sectors_dispatched); + } +@@ -2766,7 +2764,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || + bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) { + bfq_log(bfqd, +- "update_rate_reset: only resetting, delta_first %lluus samples %d", ++ "only resetting, delta_first %lluus samples %d", + bfqd->delta_from_first>>10, bfqd->peak_rate_samples); + goto reset_computation; + } +@@ -2790,7 +2788,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + div_u64(bfqd->delta_from_first, NSEC_PER_USEC)); + + bfq_log(bfqd, +-"update_rate_reset: tot_sects %llu delta_first %lluus rate %llu sects/s (%d)", ++"tot_sects %llu delta_first %lluus rate %llu sects/s (%d)", + bfqd->tot_sectors_dispatched, bfqd->delta_from_first>>10, + ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), + rate > 20<<BFQ_RATE_SHIFT); +@@ -2805,14 +2803,14 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + rate <= bfqd->peak_rate) || + rate > 20<<BFQ_RATE_SHIFT) { + bfq_log(bfqd, +- "update_rate_reset: goto reset, samples %u/%u rate/peak %llu/%llu", ++ "goto reset, samples %u/%u rate/peak %llu/%llu", + bfqd->peak_rate_samples, bfqd->sequential_samples, + ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); + goto reset_computation; + } else { + bfq_log(bfqd, +- "update_rate_reset: do update, samples %u/%u rate/peak %llu/%llu", ++ "do update, samples %u/%u rate/peak %llu/%llu", + bfqd->peak_rate_samples, bfqd->sequential_samples, + ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); +@@ -2868,7 +2866,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + rate /= divisor; /* smoothing constant alpha = 1/divisor */ + + bfq_log(bfqd, +- "update_rate_reset: divisor %d tmp_peak_rate %llu tmp_rate %u", ++ "divisor %d tmp_peak_rate %llu tmp_rate %u", + divisor, + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT), + (u32)((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT)); +@@ -2922,7 +2920,7 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + + if (bfqd->peak_rate_samples == 0) { /* first dispatch */ + bfq_log(bfqd, +- "update_peak_rate: goto reset, samples %d", ++ "goto reset, samples %d", + bfqd->peak_rate_samples) ; + bfq_reset_rate_computation(bfqd, rq); + goto update_last_values; /* will add one sample */ +@@ -2943,7 +2941,7 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && + bfqd->rq_in_driver == 0) { + bfq_log(bfqd, +-"update_peak_rate: jumping to updating&resetting delta_last %lluus samples %d", ++"jumping to updating&resetting delta_last %lluus samples %d", + (now_ns - bfqd->last_dispatch)>>10, + bfqd->peak_rate_samples) ; + goto update_rate_and_reset; +@@ -2969,7 +2967,7 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + bfqd->delta_from_first = now_ns - bfqd->first_dispatch; + + bfq_log(bfqd, +- "update_peak_rate: added samples %u/%u tot_sects %llu delta_first %lluus", ++ "added samples %u/%u tot_sects %llu delta_first %lluus", + bfqd->peak_rate_samples, bfqd->sequential_samples, + bfqd->tot_sectors_dispatched, + bfqd->delta_from_first>>10); +@@ -2985,12 +2983,12 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + bfqd->last_dispatch = now_ns; + + bfq_log(bfqd, +- "update_peak_rate: delta_first %lluus last_pos %llu peak_rate %llu", ++ "delta_first %lluus last_pos %llu peak_rate %llu", + (now_ns - bfqd->first_dispatch)>>10, + (unsigned long long) bfqd->last_position, + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); + bfq_log(bfqd, +- "update_peak_rate: samples at end %d", bfqd->peak_rate_samples); ++ "samples at end %d", bfqd->peak_rate_samples); + } + + /* +@@ -3088,11 +3086,11 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, + */ + budget = 2 * min_budget; + +- bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", ++ bfq_log_bfqq(bfqd, bfqq, "last budg %d, budg left %d", + bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); +- bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", ++ bfq_log_bfqq(bfqd, bfqq, "last max_budg %d, min budg %d", + budget, bfq_min_budget(bfqd)); +- bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", ++ bfq_log_bfqq(bfqd, bfqq, "sync %d, seeky %d", + bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); + + if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { +@@ -3294,7 +3292,7 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, + else /* charge at least one seek */ + *delta_ms = bfq_slice_idle / NSEC_PER_MSEC; + +- bfq_log(bfqd, "bfq_bfqq_is_slow: too short %u", delta_usecs); ++ bfq_log(bfqd, "too short %u", delta_usecs); + + return slow; + } +@@ -3317,11 +3315,11 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * peak rate. + */ + slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; +- bfq_log(bfqd, "bfq_bfqq_is_slow: relative rate %d/%d", ++ bfq_log(bfqd, "relative rate %d/%d", + bfqq->entity.service, bfqd->bfq_max_budget); + } + +- bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); ++ bfq_log_bfqq(bfqd, bfqq, "slow %d", slow); + + return slow; + } +@@ -3423,7 +3421,7 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, + struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqd, bfqq, +-"softrt_next_start: service_blkg %lu soft_rate %u sects/sec interval %u", ++"service_blkg %lu soft_rate %u sects/sec interval %u", + bfqq->service_from_backlogged, + bfqd->bfq_wr_max_softrt_rate, + jiffies_to_msecs(HZ * bfqq->service_from_backlogged / +@@ -3602,7 +3600,7 @@ static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) + static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "may_budget_timeout: wait_request %d left %d timeout %d", ++ "wait_request %d left %d timeout %d", + bfq_bfqq_wait_request(bfqq), + bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, + bfq_bfqq_budget_timeout(bfqq)); +@@ -3863,11 +3861,11 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq) + * either boosts the throughput (without issues), or is + * necessary to preserve service guarantees. + */ +- bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d", ++ bfq_log_bfqq(bfqd, bfqq, "sync %d idling_boosts_thr %d", + bfq_bfqq_sync(bfqq), idling_boosts_thr); + + bfq_log_bfqq(bfqd, bfqq, +- "may_idle: wr_busy %d boosts %d IO-bound %d guar %d", ++ "wr_busy %d boosts %d IO-bound %d guar %d", + bfqd->wr_busy_queues, + idling_boosts_thr_without_issues, + bfq_bfqq_IO_bound(bfqq), +@@ -3907,7 +3905,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) + if (!bfqq) + goto new_queue; + +- bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); ++ bfq_log_bfqq(bfqd, bfqq, "already in-service queue"); + + if (bfq_may_expire_for_budg_timeout(bfqq) && + !bfq_bfqq_wait_request(bfqq) && +@@ -3983,14 +3981,14 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) + new_queue: + bfqq = bfq_set_in_service_queue(bfqd); + if (bfqq) { +- bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); ++ bfq_log_bfqq(bfqd, bfqq, "checking new queue"); + goto check_queue; + } + keep_queue: + if (bfqq) +- bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); ++ bfq_log_bfqq(bfqd, bfqq, "returned this queue"); + else +- bfq_log(bfqd, "select_queue: no queue returned"); ++ bfq_log(bfqd, "no queue returned"); + + return bfqq; + } +@@ -4043,8 +4041,7 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) + /* see comments on max_service_from_wr */ + bfq_bfqq_end_wr(bfqq); + bfq_log_bfqq(bfqd, bfqq, +- "[%s] too much service", +- __func__); ++ "too much service"); + } + } + /* +@@ -4122,7 +4119,7 @@ static bool bfq_has_work(struct blk_mq_hw_ctx *hctx) + { + struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; + +- bfq_log(bfqd, "has_work, dispatch_non_empty %d busy_queues %d", ++ bfq_log(bfqd, "dispatch_non_empty %d busy_queues %d", + !list_empty_careful(&bfqd->dispatch), bfqd->busy_queues > 0); + + /* +@@ -4146,7 +4143,7 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + rq->rq_flags &= ~RQF_DISP_LIST; + + bfq_log(bfqd, +- "dispatch requests: picked %p from dispatch list", rq); ++ "picked %p from dispatch list", rq); + bfqq = RQ_BFQQ(rq); + + if (bfqq) { +@@ -4196,7 +4193,7 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + goto start_rq; + } + +- bfq_log(bfqd, "dispatch requests: %d busy queues", bfqd->busy_queues); ++ bfq_log(bfqd, "%d busy queues", bfqd->busy_queues); + + if (bfqd->busy_queues == 0) + goto exit; +@@ -4236,13 +4233,13 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + rq->rq_flags |= RQF_STARTED; + if (bfqq) + bfq_log_bfqq(bfqd, bfqq, +- "dispatched %s request %p, rq_in_driver %d", ++ "%s request %p, rq_in_driver %d", + bfq_bfqq_sync(bfqq) ? "sync" : "async", + rq, + bfqd->rq_in_driver); + else + bfq_log(bfqd, +- "dispatched request %p from dispatch list, rq_in_driver %d", ++ "request %p from dispatch list, rq_in_driver %d", + rq, bfqd->rq_in_driver); + } else + bfq_log(bfqd, +@@ -4339,7 +4336,7 @@ static void bfq_put_queue(struct bfq_queue *bfqq) + BUG_ON(bfqq->ref <= 0); + + if (bfqq->bfqd) +- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "%p %d", bfqq, bfqq->ref); + + bfqq->ref--; + if (bfqq->ref) +@@ -4383,10 +4380,10 @@ static void bfq_put_queue(struct bfq_queue *bfqq) + } + + if (bfqq->bfqd) +- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p freed", bfqq); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "%p freed", bfqq); + + #ifdef BFQ_GROUP_IOSCHED_ENABLED +- bfq_log_bfqq(bfqq->bfqd, bfqq, "[%s] putting blkg and bfqg %p\n", __func__, bfqg); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "putting blkg and bfqg %p\n", bfqg); + bfqg_and_blkg_put(bfqg); + #endif + kmem_cache_free(bfq_pool, bfqq); +@@ -4418,7 +4415,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) + bfq_schedule_dispatch(bfqd); + } + +- bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqd, bfqq, "%p, %d", bfqq, bfqq->ref); + + bfq_put_cooperator(bfqq); + +@@ -4502,7 +4499,7 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, + bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); + bfqq->entity.prio_changed = 1; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "set_next_ioprio_data: bic_class %d prio %d class %d", ++ "bic_class %d prio %d class %d", + ioprio_class, bfqq->new_ioprio, bfqq->new_ioprio_class); + } + +@@ -4529,7 +4526,7 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) + bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); + bic_set_bfqq(bic, bfqq, false); + bfq_log_bfqq(bfqd, bfqq, +- "check_ioprio_change: bfqq %p %d", ++ "bfqq %p %d", + bfqq, bfqq->ref); + } + +@@ -4667,14 +4664,14 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, + * guarantee that this queue is not freed + * until its group goes away. + */ +- bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", ++ bfq_log_bfqq(bfqd, bfqq, "bfqq not in async: %p, %d", + bfqq, bfqq->ref); + *async_bfqq = bfqq; + } + + out: + bfqq->ref++; /* get a process reference to this queue */ +- bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqd, bfqq, "at end: %p, %d", bfqq, bfqq->ref); + rcu_read_unlock(); + return bfqq; + } +@@ -4733,7 +4730,7 @@ static void bfq_update_has_short_ttime(struct bfq_data *bfqd, + bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle)) + has_short_ttime = false; + +- bfq_log_bfqq(bfqd, bfqq, "update_has_short_ttime: has_short_ttime %d", ++ bfq_log_bfqq(bfqd, bfqq, "has_short_ttime %d", + has_short_ttime); + + if (has_short_ttime) +@@ -4759,7 +4756,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bfq_update_io_seektime(bfqd, bfqq, rq); + + bfq_log_bfqq(bfqd, bfqq, +- "rq_enqueued: has_short_ttime=%d (seeky %d)", ++ "has_short_ttime=%d (seeky %d)", + bfq_bfqq_has_short_ttime(bfqq), BFQQ_SEEKY(bfqq)); + + bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); +@@ -4818,7 +4815,7 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) + + assert_spin_locked(&bfqd->lock); + +- bfq_log_bfqq(bfqd, bfqq, "__insert_req: rq %p bfqq %p", rq, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, "rq %p bfqq %p", rq, bfqq); + + /* + * An unplug may trigger a requeue of a request from the device +@@ -4837,9 +4834,9 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) + new_bfqq->allocated++; + bfqq->allocated--; + bfq_log_bfqq(bfqd, bfqq, +- "insert_request: new allocated %d", bfqq->allocated); ++ "new allocated %d", bfqq->allocated); + bfq_log_bfqq(bfqd, new_bfqq, +- "insert_request: new_bfqq new allocated %d", ++ "new_bfqq new allocated %d", + bfqq->allocated); + + new_bfqq->ref++; +@@ -4911,11 +4908,11 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + rq->rq_flags |= RQF_DISP_LIST; + if (bfqq) + bfq_log_bfqq(bfqd, bfqq, +- "insert_request %p in disp: at_head %d", ++ "%p in disp: at_head %d", + rq, at_head); + else + bfq_log(bfqd, +- "insert_request %p in disp: at_head %d", ++ "%p in disp: at_head %d", + rq, at_head); + } else { + BUG_ON(!(rq->rq_flags & RQF_GOT)); +@@ -5033,7 +5030,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) + bfqq->dispatched--; + + bfq_log_bfqq(bfqd, bfqq, +- "completed_requests: new disp %d, new rq_in_driver %d", ++ "new disp %d, new rq_in_driver %d", + bfqq->dispatched, bfqd->rq_in_driver); + + if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { +@@ -5061,7 +5058,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) + delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC); + + bfq_log_bfqq(bfqd, bfqq, +- "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu", ++ "delta %uus/%luus max_size %u rate %llu/%llu", + delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size, + delta_us > 0 ? + (USEC_PER_SEC* +@@ -5129,7 +5126,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) + static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "put_request_body: allocated %d", bfqq->allocated); ++ "allocated %d", bfqq->allocated); + BUG_ON(!bfqq->allocated); + bfqq->allocated--; + +@@ -5406,10 +5403,10 @@ static void bfq_prepare_request(struct request *rq, struct bio *bio) + + bfqq->allocated++; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "get_request: new allocated %d", bfqq->allocated); ++ "new allocated %d", bfqq->allocated); + + bfqq->ref++; +- bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d", rq, bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqd, bfqq, "%p: bfqq %p, %d", rq, bfqq, bfqq->ref); + + rq->elv.priv[0] = bic; + rq->elv.priv[1] = bfqq; +@@ -5493,7 +5490,7 @@ static enum hrtimer_restart bfq_idle_slice_timer(struct hrtimer *timer) + idle_slice_timer); + struct bfq_queue *bfqq = bfqd->in_service_queue; + +- bfq_log(bfqd, "slice_timer expired"); ++ bfq_log(bfqd, "expired"); + + /* + * Theoretical race here: the in-service queue can be NULL or +@@ -5515,10 +5512,10 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd, + struct bfq_group *root_group = bfqd->root_group; + struct bfq_queue *bfqq = *bfqq_ptr; + +- bfq_log(bfqd, "put_async_bfqq: %p", bfqq); ++ bfq_log(bfqd, "%p", bfqq); + if (bfqq) { + bfq_bfqq_move(bfqd, bfqq, root_group); +- bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", ++ bfq_log_bfqq(bfqd, bfqq, "putting %p, %d", + bfqq, bfqq->ref); + bfq_put_queue(bfqq); + *bfqq_ptr = NULL; +@@ -5547,7 +5544,7 @@ static void bfq_exit_queue(struct elevator_queue *e) + struct bfq_data *bfqd = e->elevator_data; + struct bfq_queue *bfqq, *n; + +- bfq_log(bfqd, "exit_queue: starting ..."); ++ bfq_log(bfqd, "starting ..."); + + hrtimer_cancel(&bfqd->idle_slice_timer); + +@@ -5575,7 +5572,7 @@ static void bfq_exit_queue(struct elevator_queue *e) + spin_unlock_irq(&bfqd->lock); + #endif + +- bfq_log(bfqd, "exit_queue: finished ..."); ++ bfq_log(bfqd, "finished ..."); + kfree(bfqd); + } + +diff --git a/block/bfq-mq.h b/block/bfq-mq.h +index 9a5ce1168ff5..e2ae11bf8f76 100644 +--- a/block/bfq-mq.h ++++ b/block/bfq-mq.h +@@ -712,34 +712,34 @@ static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); + static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); + + #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ +- pr_crit("%s bfq%d%c %s " fmt "\n", \ ++ pr_crit("%s bfq%d%c %s [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ + (bfqq)->pid, \ + bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- bfqq_group(bfqq)->blkg_path, ##args); \ ++ bfqq_group(bfqq)->blkg_path, __func__, ##args); \ + } while (0) + + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \ +- pr_crit("%s %s " fmt "\n", \ ++ pr_crit("%s %s [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ +- bfqg->blkg_path, ##args); \ ++ bfqg->blkg_path, __func__, ##args); \ + } while (0) + + #else /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ +- pr_crit("%s bfq%d%c " fmt "\n", \ ++ pr_crit("%s bfq%d%c [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ + (bfqq)->pid, bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- ##args) ++ __func__, ##args) + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) + + #endif /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log(bfqd, fmt, args...) \ +- pr_crit("%s bfq " fmt "\n", \ ++ pr_crit("%s bfq [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ +- ##args) ++ __func__, ##args) + + #else /* CONFIG_BFQ_REDIRECT_TO_CONSOLE */ + +@@ -762,28 +762,29 @@ static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); + static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); + + #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ +- blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s " fmt, \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s [%s] " fmt, \ + (bfqq)->pid, \ + bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- bfqq_group(bfqq)->blkg_path, ##args); \ ++ bfqq_group(bfqq)->blkg_path, __func__, ##args); \ + } while (0) + + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \ +- blk_add_trace_msg((bfqd)->queue, "%s " fmt, bfqg->blkg_path, ##args);\ ++ blk_add_trace_msg((bfqd)->queue, "%s [%s] " fmt, bfqg->blkg_path, \ ++ __func__, ##args);\ + } while (0) + + #else /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ +- blk_add_trace_msg((bfqd)->queue, "bfq%d%c " fmt, (bfqq)->pid, \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c [%s] " fmt, (bfqq)->pid, \ + bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- ##args) ++ __func__, ##args) + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) + + #endif /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log(bfqd, fmt, args...) \ +- blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args) ++ blk_add_trace_msg((bfqd)->queue, "bfq [%s] " fmt, __func__, ##args) + + #endif /* CONFIG_BLK_DEV_IO_TRACE */ + #endif /* CONFIG_BFQ_REDIRECT_TO_CONSOLE */ +@@ -938,7 +939,7 @@ bfq_entity_service_tree(struct bfq_entity *entity) + + if (bfqq) + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "entity_service_tree %p %d", ++ "%p %d", + sched_data->service_tree + idx, idx); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + else { +@@ -946,7 +947,7 @@ bfq_entity_service_tree(struct bfq_entity *entity) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "entity_service_tree %p %d", ++ "%p %d", + sched_data->service_tree + idx, idx); + } + #endif +diff --git a/block/bfq-sched.c b/block/bfq-sched.c +index 4e6c5232e2fb..ead34c30a7c2 100644 +--- a/block/bfq-sched.c ++++ b/block/bfq-sched.c +@@ -119,7 +119,7 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, + + if (bfqq) + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "update_next_in_service: chose without lookup"); ++ "chose without lookup"); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + else { + struct bfq_group *bfqg = +@@ -127,7 +127,7 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, + struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data*)bfqg->bfqd, bfqg, +- "update_next_in_service: chose without lookup"); ++ "chose without lookup"); + } + #endif + } +@@ -148,7 +148,7 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, + bfqq = bfq_entity_to_bfqq(next_in_service); + if (bfqq) + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "update_next_in_service: chosen this queue"); ++ "chosen this queue"); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + else { + struct bfq_group *bfqg = +@@ -156,7 +156,7 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, + struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "update_next_in_service: chosen this entity"); ++ "chosen this entity"); + } + #endif + return parent_sched_may_change; +@@ -331,10 +331,10 @@ static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) + + if (bfqq) { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "calc_finish: serv %lu, w %d", ++ "serv %lu, w %d", + service, entity->weight); + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "calc_finish: start %llu, finish %llu, delta %llu", ++ "start %llu, finish %llu, delta %llu", + start, finish, delta); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + } else { +@@ -342,10 +342,10 @@ static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "calc_finish group: serv %lu, w %d", ++ "group: serv %lu, w %d", + service, entity->weight); + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "calc_finish group: start %llu, finish %llu, delta %llu", ++ "group: start %llu, finish %llu, delta %llu", + start, finish, delta); + #endif + } +@@ -484,7 +484,7 @@ static void bfq_update_active_node(struct rb_node *node) + + if (bfqq) { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "update_active_node: new min_start %llu", ++ "new min_start %llu", + ((entity->min_start>>10)*1000)>>12); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + } else { +@@ -492,7 +492,7 @@ static void bfq_update_active_node(struct rb_node *node) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "update_active_node: new min_start %llu", ++ "new min_start %llu", + ((entity->min_start>>10)*1000)>>12); + #endif + } +@@ -620,7 +620,7 @@ static void bfq_get_entity(struct bfq_entity *entity) + + if (bfqq) { + bfqq->ref++; +- bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d", ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "%p %d", + bfqq, bfqq->ref); + } + } +@@ -748,7 +748,7 @@ static void bfq_forget_entity(struct bfq_service_tree *st, + entity->on_st = false; + st->wsum -= entity->weight; + if (bfqq && !is_in_service) { +- bfq_log_bfqq(bfqq->bfqd, bfqq, "forget_entity (before): %p %d", ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "(before): %p %d", + bfqq, bfqq->ref); + bfq_put_queue(bfqq); + } +@@ -1008,7 +1008,7 @@ static void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, + tot_serv_to_charge = entity->service; + + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "charge_time: %lu/%u ms, %d/%d/%d sectors", ++ "%lu/%u ms, %d/%d/%d sectors", + time_ms, timeout_ms, entity->service, + tot_serv_to_charge, entity->budget); + +@@ -1080,7 +1080,7 @@ static void bfq_update_fin_time_enqueue(struct bfq_entity *entity, + + if (bfqq) { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "update_fin_time_enqueue: new queue finish %llu", ++ "new queue finish %llu", + ((entity->finish>>10)*1000)>>12); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + } else { +@@ -1088,7 +1088,7 @@ static void bfq_update_fin_time_enqueue(struct bfq_entity *entity, + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "update_fin_time_enqueue: new group finish %llu", ++ "new group finish %llu", + ((entity->finish>>10)*1000)>>12); + #endif + } +@@ -1098,7 +1098,7 @@ static void bfq_update_fin_time_enqueue(struct bfq_entity *entity, + + if (bfqq) { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "update_fin_time_enqueue: queue %seligible in st %p", ++ "queue %seligible in st %p", + entity->start <= st->vtime ? "" : "non ", st); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + } else { +@@ -1106,7 +1106,7 @@ static void bfq_update_fin_time_enqueue(struct bfq_entity *entity, + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "update_fin_time_enqueue: group %seligible in st %p", ++ "group %seligible in st %p", + entity->start <= st->vtime ? "" : "non ", st); + #endif + } +@@ -1550,7 +1550,7 @@ static u64 bfq_calc_vtime_jump(struct bfq_service_tree *st) + + if (bfqq) + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "calc_vtime_jump: new value %llu", ++ "new value %llu", + ((root_entity->min_start>>10)*1000)>>12); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + else { +@@ -1559,7 +1559,7 @@ static u64 bfq_calc_vtime_jump(struct bfq_service_tree *st) + entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "calc_vtime_jump: new value %llu", ++ "new value %llu", + ((root_entity->min_start>>10)*1000)>>12); + } + #endif +@@ -1677,7 +1677,7 @@ __bfq_lookup_next_entity(struct bfq_service_tree *st, bool in_service) + bfqq = bfq_entity_to_bfqq(entity); + if (bfqq) + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "__lookup_next: start %llu vtime %llu st %p", ++ "start %llu vtime %llu st %p", + ((entity->start>>10)*1000)>>12, + ((new_vtime>>10)*1000)>>12, st); + #ifdef BFQ_GROUP_IOSCHED_ENABLED +@@ -1686,7 +1686,7 @@ __bfq_lookup_next_entity(struct bfq_service_tree *st, bool in_service) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "__lookup_next: start %llu vtime %llu (%llu) st %p", ++ "start %llu vtime %llu (%llu) st %p", + ((entity->start>>10)*1000)>>12, + ((st->vtime>>10)*1000)>>12, + ((new_vtime>>10)*1000)>>12, st); +@@ -1821,14 +1821,14 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg(bfqd, bfqg, +- "get_next_queue: lookup in this group"); ++ "lookup in this group"); + if (!sd->next_in_service) +- pr_crit("get_next_queue: lookup in this group"); ++ pr_crit("lookup in this group"); + } else { + bfq_log_bfqg(bfqd, bfqd->root_group, +- "get_next_queue: lookup in root group"); ++ "lookup in root group"); + if (!sd->next_in_service) +- pr_crit("get_next_queue: lookup in root group"); ++ pr_crit("lookup in root group"); + } + #endif + +@@ -1903,7 +1903,7 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) + bfqq = bfq_entity_to_bfqq(entity); + if (bfqq) + bfq_log_bfqq(bfqd, bfqq, +- "get_next_queue: this queue, finish %llu", ++ "this queue, finish %llu", + (((entity->finish>>10)*1000)>>10)>>2); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + else { +@@ -1911,7 +1911,7 @@ static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg(bfqd, bfqg, +- "get_next_queue: this entity, finish %llu", ++ "this entity, finish %llu", + (((entity->finish>>10)*1000)>>10)>>2); + } + #endif +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index c4df156b1fb4..e49e8ac882b3 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -281,7 +281,7 @@ static void bfq_schedule_dispatch(struct bfq_data *bfqd); + static void bfq_schedule_dispatch(struct bfq_data *bfqd) + { + if (bfqd->queued != 0) { +- bfq_log(bfqd, "schedule dispatch"); ++ bfq_log(bfqd, ""); + kblockd_schedule_work(&bfqd->unplug_work); + } + } +@@ -414,7 +414,7 @@ bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, + if (rb_link) + *rb_link = p; + +- bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", ++ bfq_log(bfqd, "%llu: returning %d", + (unsigned long long) sector, + bfqq ? bfqq->pid : 0); + +@@ -635,7 +635,7 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq, + if (rq == last || ktime_get_ns() < rq->fifo_time) + return NULL; + +- bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "returned %p", rq); + BUG_ON(RB_EMPTY_NODE(&rq->rb_node)); + return rq; + } +@@ -728,7 +728,7 @@ static void bfq_updated_next_req(struct bfq_data *bfqd, + bfq_serv_to_charge(next_rq, bfqq)); + if (entity->budget != new_budget) { + entity->budget = new_budget; +- bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", ++ bfq_log_bfqq(bfqd, bfqq, "new budget %lu", + new_budget); + bfq_requeue_bfqq(bfqd, bfqq, false); + } +@@ -800,8 +800,7 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, + BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); + + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "[%s] bic %p wr_coeff %d start_finish %lu max_time %lu", +- __func__, ++ "bic %p wr_coeff %d start_finish %lu max_time %lu", + bic, bfqq->wr_coeff, bfqq->last_wr_start_finish, + bfqq->wr_cur_max_time); + +@@ -814,11 +813,11 @@ bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, + bfq_wr_duration(bfqd))) { + switch_back_to_interactive_wr(bfqq, bfqd); + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "resume state: switching back to interactive"); ++ "switching back to interactive"); + } else { + bfqq->wr_coeff = 1; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "resume state: switching off wr (%lu + %lu < %lu)", ++ "switching off wr (%lu + %lu < %lu)", + bfqq->last_wr_start_finish, bfqq->wr_cur_max_time, + jiffies); + } +@@ -870,7 +869,7 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + /* Increment burst size to take into account also bfqq */ + bfqd->burst_size++; + +- bfq_log_bfqq(bfqd, bfqq, "add_to_burst %d", bfqd->burst_size); ++ bfq_log_bfqq(bfqd, bfqq, "%d", bfqd->burst_size); + + BUG_ON(bfqd->burst_size > bfqd->bfq_large_burst_thresh); + +@@ -883,7 +882,7 @@ static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + * other to consider this burst as large. + */ + bfqd->large_burst = true; +- bfq_log_bfqq(bfqd, bfqq, "add_to_burst: large burst started"); ++ bfq_log_bfqq(bfqd, bfqq, "large burst started"); + + /* + * We can now mark all queues in the burst list as +@@ -1055,7 +1054,7 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + bfqd->large_burst = false; + bfq_reset_burst_list(bfqd, bfqq); + bfq_log_bfqq(bfqd, bfqq, +- "handle_burst: late activation or different group"); ++ "late activation or different group"); + goto end; + } + +@@ -1065,7 +1064,7 @@ static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) + * bfqq as belonging to this large burst immediately. + */ + if (bfqd->large_burst) { +- bfq_log_bfqq(bfqd, bfqq, "handle_burst: marked in burst"); ++ bfq_log_bfqq(bfqd, bfqq, "marked in burst"); + bfq_mark_bfqq_in_large_burst(bfqq); + goto end; + } +@@ -1572,7 +1571,7 @@ static void bfq_add_request(struct request *rq) + unsigned int old_wr_coeff = bfqq->wr_coeff; + bool interactive = false; + +- bfq_log_bfqq(bfqd, bfqq, "add_request: size %u %s", ++ bfq_log_bfqq(bfqd, bfqq, "size %u %s", + blk_rq_sectors(rq), rq_is_sync(rq) ? "S" : "A"); + + if (bfqq->wr_coeff > 1) /* queue is being weight-raised */ +@@ -1870,10 +1869,10 @@ static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) + */ + bfqq->entity.prio_changed = 1; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "end_wr: wrais ending at %lu, rais_max_time %u", ++ "wrais ending at %lu, rais_max_time %u", + bfqq->last_wr_start_finish, + jiffies_to_msecs(bfqq->wr_cur_max_time)); +- bfq_log_bfqq(bfqq->bfqd, bfqq, "end_wr: wr_busy %d", ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "wr_busy %d", + bfqq->bfqd->wr_busy_queues); + } + +@@ -2048,8 +2047,8 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, + { + if (bfq_too_late_for_merging(new_bfqq)) { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "[%s] too late for bfq%d to be merged", +- __func__, new_bfqq->pid); ++ "too late for bfq%d to be merged", ++ new_bfqq->pid); + return false; + } + +@@ -2258,7 +2257,7 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, + + } + +- bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", ++ bfq_log_bfqq(bfqd, new_bfqq, "wr_busy %d", + bfqd->wr_busy_queues); + + /* +@@ -2359,7 +2358,7 @@ static void bfq_set_budget_timeout(struct bfq_data *bfqd, + bfqq->budget_timeout = jiffies + + bfqd->bfq_timeout * timeout_coeff; + +- bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u", ++ bfq_log_bfqq(bfqd, bfqq, "%u", + jiffies_to_msecs(bfqd->bfq_timeout * timeout_coeff)); + } + +@@ -2427,10 +2426,10 @@ static void __bfq_set_in_service_queue(struct bfq_data *bfqd, + + bfq_set_budget_timeout(bfqd, bfqq); + bfq_log_bfqq(bfqd, bfqq, +- "set_in_service_queue, cur-budget = %d", ++ "cur-budget = %d", + bfqq->entity.budget); + } else +- bfq_log(bfqd, "set_in_service_queue: NULL"); ++ bfq_log(bfqd, "NULL"); + + bfqd->in_service_queue = bfqq; + } +@@ -2559,7 +2558,7 @@ static void bfq_reset_rate_computation(struct bfq_data *bfqd, struct request *rq + bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ + + bfq_log(bfqd, +- "reset_rate_computation at end, sample %u/%u tot_sects %llu", ++ "at end, sample %u/%u tot_sects %llu", + bfqd->peak_rate_samples, bfqd->sequential_samples, + bfqd->tot_sectors_dispatched); + } +@@ -2579,7 +2578,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || + bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) { + bfq_log(bfqd, +- "update_rate_reset: only resetting, delta_first %lluus samples %d", ++ "only resetting, delta_first %lluus samples %d", + bfqd->delta_from_first>>10, bfqd->peak_rate_samples); + goto reset_computation; + } +@@ -2603,7 +2602,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + div_u64(bfqd->delta_from_first, NSEC_PER_USEC)); + + bfq_log(bfqd, +-"update_rate_reset: tot_sects %llu delta_first %lluus rate %llu sects/s (%d)", ++"tot_sects %llu delta_first %lluus rate %llu sects/s (%d)", + bfqd->tot_sectors_dispatched, bfqd->delta_from_first>>10, + ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), + rate > 20<<BFQ_RATE_SHIFT); +@@ -2618,14 +2617,14 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + rate <= bfqd->peak_rate) || + rate > 20<<BFQ_RATE_SHIFT) { + bfq_log(bfqd, +- "update_rate_reset: goto reset, samples %u/%u rate/peak %llu/%llu", ++ "goto reset, samples %u/%u rate/peak %llu/%llu", + bfqd->peak_rate_samples, bfqd->sequential_samples, + ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); + goto reset_computation; + } else { + bfq_log(bfqd, +- "update_rate_reset: do update, samples %u/%u rate/peak %llu/%llu", ++ "do update, samples %u/%u rate/peak %llu/%llu", + bfqd->peak_rate_samples, bfqd->sequential_samples, + ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); +@@ -2681,7 +2680,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + rate /= divisor; /* smoothing constant alpha = 1/divisor */ + + bfq_log(bfqd, +- "update_rate_reset: divisor %d tmp_peak_rate %llu tmp_rate %u", ++ "divisor %d tmp_peak_rate %llu tmp_rate %u", + divisor, + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT), + (u32)((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT)); +@@ -2735,7 +2734,7 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + + if (bfqd->peak_rate_samples == 0) { /* first dispatch */ + bfq_log(bfqd, +- "update_peak_rate: goto reset, samples %d", ++ "goto reset, samples %d", + bfqd->peak_rate_samples) ; + bfq_reset_rate_computation(bfqd, rq); + goto update_last_values; /* will add one sample */ +@@ -2756,7 +2755,7 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && + bfqd->rq_in_driver == 0) { + bfq_log(bfqd, +-"update_peak_rate: jumping to updating&resetting delta_last %lluus samples %d", ++"jumping to updating&resetting delta_last %lluus samples %d", + (now_ns - bfqd->last_dispatch)>>10, + bfqd->peak_rate_samples) ; + goto update_rate_and_reset; +@@ -2782,7 +2781,7 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + bfqd->delta_from_first = now_ns - bfqd->first_dispatch; + + bfq_log(bfqd, +- "update_peak_rate: added samples %u/%u tot_sects %llu delta_first %lluus", ++ "added samples %u/%u tot_sects %llu delta_first %lluus", + bfqd->peak_rate_samples, bfqd->sequential_samples, + bfqd->tot_sectors_dispatched, + bfqd->delta_from_first>>10); +@@ -2798,12 +2797,12 @@ static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq) + bfqd->last_dispatch = now_ns; + + bfq_log(bfqd, +- "update_peak_rate: delta_first %lluus last_pos %llu peak_rate %llu", ++ "delta_first %lluus last_pos %llu peak_rate %llu", + (now_ns - bfqd->first_dispatch)>>10, + (unsigned long long) bfqd->last_position, + ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); + bfq_log(bfqd, +- "update_peak_rate: samples at end %d", bfqd->peak_rate_samples); ++ "samples at end %d", bfqd->peak_rate_samples); + } + + /* +@@ -2900,11 +2899,11 @@ static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, + */ + budget = 2 * min_budget; + +- bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", ++ bfq_log_bfqq(bfqd, bfqq, "last budg %d, budg left %d", + bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); +- bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", ++ bfq_log_bfqq(bfqd, bfqq, "last max_budg %d, min budg %d", + budget, bfq_min_budget(bfqd)); +- bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", ++ bfq_log_bfqq(bfqd, bfqq, "sync %d, seeky %d", + bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); + + if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { +@@ -3106,7 +3105,7 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, + else /* charge at least one seek */ + *delta_ms = bfq_slice_idle / NSEC_PER_MSEC; + +- bfq_log(bfqd, "bfq_bfqq_is_slow: too short %u", delta_usecs); ++ bfq_log(bfqd, "too short %u", delta_usecs); + + return slow; + } +@@ -3129,11 +3128,11 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, + * peak rate. + */ + slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; +- bfq_log(bfqd, "bfq_bfqq_is_slow: relative rate %d/%d", ++ bfq_log(bfqd, "relative rate %d/%d", + bfqq->entity.service, bfqd->bfq_max_budget); + } + +- bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); ++ bfq_log_bfqq(bfqd, bfqq, "slow %d", slow); + + return slow; + } +@@ -3235,7 +3234,7 @@ static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, + struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqd, bfqq, +-"softrt_next_start: service_blkg %lu soft_rate %u sects/sec interval %u", ++"service_blkg %lu soft_rate %u sects/sec interval %u", + bfqq->service_from_backlogged, + bfqd->bfq_wr_max_softrt_rate, + jiffies_to_msecs(HZ * bfqq->service_from_backlogged / +@@ -3414,7 +3413,7 @@ static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) + static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) + { + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "may_budget_timeout: wait_request %d left %d timeout %d", ++ "wait_request %d left %d timeout %d", + bfq_bfqq_wait_request(bfqq), + bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, + bfq_bfqq_budget_timeout(bfqq)); +@@ -3675,11 +3674,11 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq) + * either boosts the throughput (without issues), or is + * necessary to preserve service guarantees. + */ +- bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d", ++ bfq_log_bfqq(bfqd, bfqq, "sync %d idling_boosts_thr %d", + bfq_bfqq_sync(bfqq), idling_boosts_thr); + + bfq_log_bfqq(bfqd, bfqq, +- "may_idle: wr_busy %d boosts %d IO-bound %d guar %d", ++ "wr_busy %d boosts %d IO-bound %d guar %d", + bfqd->wr_busy_queues, + idling_boosts_thr_without_issues, + bfq_bfqq_IO_bound(bfqq), +@@ -3719,7 +3718,7 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) + if (!bfqq) + goto new_queue; + +- bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); ++ bfq_log_bfqq(bfqd, bfqq, "already in-service queue"); + + if (bfq_may_expire_for_budg_timeout(bfqq) && + !hrtimer_active(&bfqd->idle_slice_timer) && +@@ -3797,14 +3796,14 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) + new_queue: + bfqq = bfq_set_in_service_queue(bfqd); + if (bfqq) { +- bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); ++ bfq_log_bfqq(bfqd, bfqq, "checking new queue"); + goto check_queue; + } + keep_queue: + if (bfqq) +- bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); ++ bfq_log_bfqq(bfqd, bfqq, "returned this queue"); + else +- bfq_log(bfqd, "select_queue: no queue returned"); ++ bfq_log(bfqd, "no queue returned"); + + return bfqq; + } +@@ -3857,8 +3856,7 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) + /* see comments on max_service_from_wr */ + bfq_bfqq_end_wr(bfqq); + bfq_log_bfqq(bfqd, bfqq, +- "[%s] too much service", +- __func__); ++ "too much service"); + } + } + /* +@@ -3987,7 +3985,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force) + struct bfq_data *bfqd = q->elevator->elevator_data; + struct bfq_queue *bfqq; + +- bfq_log(bfqd, "dispatch requests: %d busy queues", bfqd->busy_queues); ++ bfq_log(bfqd, "%d busy queues", bfqd->busy_queues); + + if (bfqd->busy_queues == 0) + return 0; +@@ -4021,7 +4019,7 @@ static int bfq_dispatch_requests(struct request_queue *q, int force) + if (!bfq_dispatch_request(bfqd, bfqq)) + return 0; + +- bfq_log_bfqq(bfqd, bfqq, "dispatched %s request", ++ bfq_log_bfqq(bfqd, bfqq, "%s request", + bfq_bfqq_sync(bfqq) ? "sync" : "async"); + + BUG_ON(bfqq->next_rq == NULL && +@@ -4044,7 +4042,7 @@ static void bfq_put_queue(struct bfq_queue *bfqq) + + BUG_ON(bfqq->ref <= 0); + +- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "%p %d", bfqq, bfqq->ref); + bfqq->ref--; + if (bfqq->ref) + return; +@@ -4086,7 +4084,7 @@ static void bfq_put_queue(struct bfq_queue *bfqq) + bfqq->bfqd->burst_size--; + } + +- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p freed", bfqq); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "%p freed", bfqq); + + kmem_cache_free(bfq_pool, bfqq); + #ifdef BFQ_GROUP_IOSCHED_ENABLED +@@ -4120,7 +4118,7 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) + bfq_schedule_dispatch(bfqd); + } + +- bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqd, bfqq, "%p, %d", bfqq, bfqq->ref); + + bfq_put_cooperator(bfqq); + +@@ -4200,7 +4198,7 @@ static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, + bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); + bfqq->entity.prio_changed = 1; + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "set_next_ioprio_data: bic_class %d prio %d class %d", ++ "bic_class %d prio %d class %d", + ioprio_class, bfqq->new_ioprio, bfqq->new_ioprio_class); + } + +@@ -4227,7 +4225,7 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) + bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); + bic_set_bfqq(bic, bfqq, false); + bfq_log_bfqq(bfqd, bfqq, +- "check_ioprio_change: bfqq %p %d", ++ "bfqq %p %d", + bfqq, bfqq->ref); + } + +@@ -4362,14 +4360,14 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, + * guarantee that this queue is not freed + * until its group goes away. + */ +- bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", ++ bfq_log_bfqq(bfqd, bfqq, "bfqq not in async: %p, %d", + bfqq, bfqq->ref); + *async_bfqq = bfqq; + } + + out: + bfqq->ref++; /* get a process reference to this queue */ +- bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqd, bfqq, "at end: %p, %d", bfqq, bfqq->ref); + rcu_read_unlock(); + return bfqq; + } +@@ -4428,7 +4426,7 @@ static void bfq_update_has_short_ttime(struct bfq_data *bfqd, + bic->ttime.ttime_mean > bfqd->bfq_slice_idle)) + has_short_ttime = false; + +- bfq_log_bfqq(bfqd, bfqq, "update_has_short_ttime: has_short_ttime %d", ++ bfq_log_bfqq(bfqd, bfqq, "has_short_ttime %d", + has_short_ttime); + + if (has_short_ttime) +@@ -4454,7 +4452,7 @@ static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, + bfq_update_io_seektime(bfqd, bfqq, rq); + + bfq_log_bfqq(bfqd, bfqq, +- "rq_enqueued: has_short_ttime=%d (seeky %d)", ++ "has_short_ttime=%d (seeky %d)", + bfq_bfqq_has_short_ttime(bfqq), BFQQ_SEEKY(bfqq)); + + bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); +@@ -4629,7 +4627,7 @@ static void bfq_completed_request(struct request_queue *q, struct request *rq) + */ + delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC); + +- bfq_log(bfqd, "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu", ++ bfq_log(bfqd, "delta %uus/%luus max_size %u rate %llu/%llu", + delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size, + delta_us > 0 ? + (USEC_PER_SEC* +@@ -4750,7 +4748,7 @@ static void bfq_put_request(struct request *rq) + rq->elv.priv[0] = NULL; + rq->elv.priv[1] = NULL; + +- bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d", ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "%p, %d", + bfqq, bfqq->ref); + bfq_put_queue(bfqq); + } +@@ -4816,7 +4814,7 @@ static int bfq_set_request(struct request_queue *q, struct request *rq, + bic_set_bfqq(bic, bfqq, is_sync); + if (split && is_sync) { + bfq_log_bfqq(bfqd, bfqq, +- "set_request: was_in_list %d " ++ "was_in_list %d " + "was_in_large_burst %d " + "large burst in progress %d", + bic->was_in_burst_list, +@@ -4826,12 +4824,12 @@ static int bfq_set_request(struct request_queue *q, struct request *rq, + if ((bic->was_in_burst_list && bfqd->large_burst) || + bic->saved_in_large_burst) { + bfq_log_bfqq(bfqd, bfqq, +- "set_request: marking in " ++ "marking in " + "large burst"); + bfq_mark_bfqq_in_large_burst(bfqq); + } else { + bfq_log_bfqq(bfqd, bfqq, +- "set_request: clearing in " ++ "clearing in " + "large burst"); + bfq_clear_bfqq_in_large_burst(bfqq); + if (bic->was_in_burst_list) +@@ -4888,7 +4886,7 @@ static int bfq_set_request(struct request_queue *q, struct request *rq, + + bfqq->allocated[rw]++; + bfqq->ref++; +- bfq_log_bfqq(bfqd, bfqq, "set_request: bfqq %p, %d", bfqq, bfqq->ref); ++ bfq_log_bfqq(bfqd, bfqq, "bfqq %p, %d", bfqq, bfqq->ref); + + rq->elv.priv[0] = bic; + rq->elv.priv[1] = bfqq; +@@ -4962,7 +4960,7 @@ static enum hrtimer_restart bfq_idle_slice_timer(struct hrtimer *timer) + * case we just expire a queue too early. + */ + if (bfqq) { +- bfq_log_bfqq(bfqd, bfqq, "slice_timer expired"); ++ bfq_log_bfqq(bfqd, bfqq, "expired"); + bfq_clear_bfqq_wait_request(bfqq); + + if (bfq_bfqq_budget_timeout(bfqq)) +@@ -5005,10 +5003,10 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd, + struct bfq_group *root_group = bfqd->root_group; + struct bfq_queue *bfqq = *bfqq_ptr; + +- bfq_log(bfqd, "put_async_bfqq: %p", bfqq); ++ bfq_log(bfqd, "%p", bfqq); + if (bfqq) { + bfq_bfqq_move(bfqd, bfqq, root_group); +- bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", ++ bfq_log_bfqq(bfqd, bfqq, "putting %p, %d", + bfqq, bfqq->ref); + bfq_put_queue(bfqq); + *bfqq_ptr = NULL; +diff --git a/block/bfq.h b/block/bfq.h +index 0cd7a3f251a7..4d2fe7f77af1 100644 +--- a/block/bfq.h ++++ b/block/bfq.h +@@ -698,37 +698,37 @@ static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); + \ + assert_spin_locked((bfqd)->queue->queue_lock); \ + blkg_path(bfqg_to_blkg(bfqq_group(bfqq)), __pbuf, sizeof(__pbuf)); \ +- pr_crit("%s bfq%d%c %s " fmt "\n", \ ++ pr_crit("%s bfq%d%c %s [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ + (bfqq)->pid, \ + bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- __pbuf, ##args); \ ++ __pbuf, __func__, ##args); \ + } while (0) + + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \ + char __pbuf[128]; \ + \ + blkg_path(bfqg_to_blkg(bfqg), __pbuf, sizeof(__pbuf)); \ +- pr_crit("%s %s " fmt "\n", \ ++ pr_crit("%s %s [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ +- __pbuf, ##args); \ ++ __pbuf, __func__, ##args); \ + } while (0) + + #else /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ +- pr_crit("%s bfq%d%c " fmt "\n", \ ++ pr_crit("%s bfq%d%c [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ + (bfqq)->pid, bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- ##args) ++ __func__, ##args) + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) + + #endif /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log(bfqd, fmt, args...) \ +- pr_crit("%s bfq " fmt "\n", \ ++ pr_crit("%s bfq [%s] " fmt "\n", \ + checked_dev_name((bfqd)->queue->backing_dev_info->dev), \ +- ##args) ++ __func__, ##args) + + #else /* CONFIG_BFQ_REDIRECT_TO_CONSOLE */ + +@@ -755,31 +755,32 @@ static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); + \ + assert_spin_locked((bfqd)->queue->queue_lock); \ + blkg_path(bfqg_to_blkg(bfqq_group(bfqq)), __pbuf, sizeof(__pbuf)); \ +- blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s " fmt, \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s [%s] " fmt, \ + (bfqq)->pid, \ + bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- __pbuf, ##args); \ ++ __pbuf, __func__, ##args); \ + } while (0) + + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \ + char __pbuf[128]; \ + \ + blkg_path(bfqg_to_blkg(bfqg), __pbuf, sizeof(__pbuf)); \ +- blk_add_trace_msg((bfqd)->queue, "%s " fmt, __pbuf, ##args); \ ++ blk_add_trace_msg((bfqd)->queue, "%s [%s] " fmt, __pbuf, \ ++ __func__, ##args); \ + } while (0) + + #else /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ +- blk_add_trace_msg((bfqd)->queue, "bfq%d%c " fmt, (bfqq)->pid, \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c [%s] " fmt, (bfqq)->pid, \ + bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ +- ##args) ++ __func__, ##args) + #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) + + #endif /* BFQ_GROUP_IOSCHED_ENABLED */ + + #define bfq_log(bfqd, fmt, args...) \ +- blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args) ++ blk_add_trace_msg((bfqd)->queue, "bfq [%s] " fmt, __func__, ##args) + + #endif /* CONFIG_BLK_DEV_IO_TRACE */ + #endif /* CONFIG_BFQ_REDIRECT_TO_CONSOLE */ +@@ -928,7 +929,7 @@ bfq_entity_service_tree(struct bfq_entity *entity) + + if (bfqq) + bfq_log_bfqq(bfqq->bfqd, bfqq, +- "entity_service_tree %p %d", ++ "%p %d", + sched_data->service_tree + idx, idx); + #ifdef BFQ_GROUP_IOSCHED_ENABLED + else { +@@ -936,7 +937,7 @@ bfq_entity_service_tree(struct bfq_entity *entity) + container_of(entity, struct bfq_group, entity); + + bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, +- "entity_service_tree %p %d", ++ "%p %d", + sched_data->service_tree + idx, idx); + } + #endif + +From 673a457e8a54d1c4b66e61b1a50956ba0b8c6a60 Mon Sep 17 00:00:00 2001 +From: Davide Paganelli <paga.david@gmail.com> +Date: Thu, 8 Feb 2018 11:49:58 +0100 +Subject: [PATCH 19/23] block, bfq-mq, bfq-sq: make bfq_bfqq_expire print + expiration reason + +Improve readability of the log messages related to the expiration +reasons of the function bfq_bfqq_expire. +Change the printing of the number that represents the reason for +expiration with an actual textual description of the reason. + +Signed-off-by: Davide Paganelli <paga.david@gmail.com> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 10 ++++++++-- + block/bfq-sq-iosched.c | 10 ++++++++-- + 2 files changed, 16 insertions(+), 4 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index edc93b6af186..9268dd47a4e5 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -133,6 +133,12 @@ static const int bfq_timeout = (HZ / 8); + */ + static const unsigned long bfq_merge_time_limit = HZ/10; + ++#define MAX_LENGTH_REASON_NAME 25 ++ ++static const char reason_name[][MAX_LENGTH_REASON_NAME] = {"TOO_IDLE", ++"BUDGET_TIMEOUT", "BUDGET_EXHAUSTED", "NO_MORE_REQUESTS", ++"PREEMPTED"}; ++ + static struct kmem_cache *bfq_pool; + + /* Below this threshold (in ns), we consider thinktime immediate. */ +@@ -3553,8 +3559,8 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd, + } + + bfq_log_bfqq(bfqd, bfqq, +- "expire (%d, slow %d, num_disp %d, short_ttime %d, weight %d)", +- reason, slow, bfqq->dispatched, ++ "expire (%s, slow %d, num_disp %d, short_ttime %d, weight %d)", ++ reason_name[reason], slow, bfqq->dispatched, + bfq_bfqq_has_short_ttime(bfqq), entity->weight); + + /* +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index e49e8ac882b3..f95deaab49a1 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -127,6 +127,12 @@ static const int bfq_timeout = (HZ / 8); + */ + static const unsigned long bfq_merge_time_limit = HZ/10; + ++#define MAX_LENGTH_REASON_NAME 25 ++ ++static const char reason_name[][MAX_LENGTH_REASON_NAME] = {"TOO_IDLE", ++"BUDGET_TIMEOUT", "BUDGET_EXHAUSTED", "NO_MORE_REQUESTS", ++"PREEMPTED"}; ++ + static struct kmem_cache *bfq_pool; + + /* Below this threshold (in ns), we consider thinktime immediate. */ +@@ -3366,8 +3372,8 @@ static void bfq_bfqq_expire(struct bfq_data *bfqd, + } + + bfq_log_bfqq(bfqd, bfqq, +- "expire (%d, slow %d, num_disp %d, short_ttime %d, weight %d)", +- reason, slow, bfqq->dispatched, ++ "expire (%s, slow %d, num_disp %d, short_ttime %d, weight %d)", ++ reason_name[reason], slow, bfqq->dispatched, + bfq_bfqq_has_short_ttime(bfqq), entity->weight); + + /* + +From 62e80623fbb58367c3f667dab22fea0804001f3b Mon Sep 17 00:00:00 2001 +From: Melzani Alessandro <melzani.alessandro@gmail.com> +Date: Mon, 26 Feb 2018 22:21:59 +0100 +Subject: [PATCH 20/23] bfq-mq: port of "block, bfq: remove batches of + confusing ifdefs" + +Commit a33801e8b473 ("block, bfq: move debug blkio stats behind +CONFIG_DEBUG_BLK_CGROUP") introduced two batches of confusing ifdefs: +one reported in [1], plus a similar one in another function. This +commit removes both batches, in the way suggested in [1]. + +[1] https://www.spinics.net/lists/linux-block/msg20043.html + +Fixes: a33801e8b473 ("block, bfq: move debug blkio stats behind CONFIG_DEBUG_BLK_CGROUP") + +Signed-off-by: Alessandro Melzani <melzani.alessandro@gmail.com> +--- + block/bfq-mq-iosched.c | 128 ++++++++++++++++++++++++++++--------------------- + 1 file changed, 73 insertions(+), 55 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 9268dd47a4e5..5a211620f316 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -4256,35 +4256,17 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + return rq; + } + +-static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) +-{ +- struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; +- struct request *rq; +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) +- struct bfq_queue *in_serv_queue, *bfqq; +- bool waiting_rq, idle_timer_disabled; +-#endif + +- spin_lock_irq(&bfqd->lock); +- +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) +- in_serv_queue = bfqd->in_service_queue; +- waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue); +- +- rq = __bfq_dispatch_request(hctx); +- +- idle_timer_disabled = +- waiting_rq && !bfq_bfqq_wait_request(in_serv_queue); +- +-#else +- rq = __bfq_dispatch_request(hctx); +-#endif +- spin_unlock_irq(&bfqd->lock); ++#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) ++static void bfq_update_dispatch_stats(struct request_queue *q, ++ struct request *rq, ++ struct bfq_queue *in_serv_queue, ++ bool idle_timer_disabled) ++{ ++ struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL; + +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) +- bfqq = rq ? RQ_BFQQ(rq) : NULL; + if (!idle_timer_disabled && !bfqq) +- return rq; ++ return; + + /* + * rq and bfqq are guaranteed to exist until this function +@@ -4299,7 +4281,7 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + * In addition, the following queue lock guarantees that + * bfqq_group(bfqq) exists as well. + */ +- spin_lock_irq(hctx->queue->queue_lock); ++ spin_lock_irq(q->queue_lock); + if (idle_timer_disabled) + /* + * Since the idle timer has been disabled, +@@ -4318,8 +4300,35 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) + bfqg_stats_set_start_empty_time(bfqg); + bfqg_stats_update_io_remove(bfqg, rq->cmd_flags); + } +- spin_unlock_irq(hctx->queue->queue_lock); ++ spin_unlock_irq(q->queue_lock); ++} ++#else ++static inline void bfq_update_dispatch_stats(struct request_queue *q, ++ struct request *rq, ++ struct bfq_queue *in_serv_queue, ++ bool idle_timer_disabled) {} + #endif ++static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx) ++{ ++ struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; ++ struct request *rq; ++ struct bfq_queue *in_serv_queue; ++ bool waiting_rq, idle_timer_disabled; ++ ++ spin_lock_irq(&bfqd->lock); ++ ++ in_serv_queue = bfqd->in_service_queue; ++ waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue); ++ ++ rq = __bfq_dispatch_request(hctx); ++ ++ idle_timer_disabled = ++ waiting_rq && !bfq_bfqq_wait_request(in_serv_queue); ++ ++ spin_unlock_irq(&bfqd->lock); ++ ++ bfq_update_dispatch_stats(hctx->queue, rq, in_serv_queue, ++ idle_timer_disabled); + + return rq; + } +@@ -4881,6 +4890,38 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) + return idle_timer_disabled; + } + ++#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) ++static void bfq_update_insert_stats(struct request_queue *q, ++ struct bfq_queue *bfqq, ++ bool idle_timer_disabled, ++ unsigned int cmd_flags) ++{ ++ if (!bfqq) ++ return; ++ ++ /* ++ * bfqq still exists, because it can disappear only after ++ * either it is merged with another queue, or the process it ++ * is associated with exits. But both actions must be taken by ++ * the same process currently executing this flow of ++ * instructions. ++ * ++ * In addition, the following queue lock guarantees that ++ * bfqq_group(bfqq) exists as well. ++ */ ++ spin_lock_irq(q->queue_lock); ++ bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags); ++ if (idle_timer_disabled) ++ bfqg_stats_update_idle_time(bfqq_group(bfqq)); ++ spin_unlock_irq(q->queue_lock); ++} ++#else ++static inline void bfq_update_insert_stats(struct request_queue *q, ++ struct bfq_queue *bfqq, ++ bool idle_timer_disabled, ++ unsigned int cmd_flags) {} ++#endif ++ + static void bfq_prepare_request(struct request *rq, struct bio *bio); + + static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, +@@ -4889,10 +4930,8 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + struct request_queue *q = hctx->queue; + struct bfq_data *bfqd = q->elevator->elevator_data; + struct bfq_queue *bfqq = RQ_BFQQ(rq); +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) + bool idle_timer_disabled = false; + unsigned int cmd_flags; +-#endif + + spin_lock_irq(&bfqd->lock); + if (blk_mq_sched_try_insert_merge(q, rq)) { +@@ -4938,7 +4977,6 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + bfqq = RQ_BFQQ(rq); + } + +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) + idle_timer_disabled = __bfq_insert_request(bfqd, rq); + /* + * Update bfqq, because, if a queue merge has occurred +@@ -4946,9 +4984,6 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + * redirected into a new queue. + */ + bfqq = RQ_BFQQ(rq); +-#else +- __bfq_insert_request(bfqd, rq); +-#endif + + if (rq_mergeable(rq)) { + elv_rqhash_add(q, rq); +@@ -4956,34 +4991,17 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, + q->last_merge = rq; + } + } +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) ++ + /* + * Cache cmd_flags before releasing scheduler lock, because rq + * may disappear afterwards (for example, because of a request + * merge). + */ + cmd_flags = rq->cmd_flags; +-#endif ++ + spin_unlock_irq(&bfqd->lock); +-#if defined(BFQ_GROUP_IOSCHED_ENABLED) && defined(CONFIG_DEBUG_BLK_CGROUP) +- if (!bfqq) +- return; +- /* +- * bfqq still exists, because it can disappear only after +- * either it is merged with another queue, or the process it +- * is associated with exits. But both actions must be taken by +- * the same process currently executing this flow of +- * instruction. +- * +- * In addition, the following queue lock guarantees that +- * bfqq_group(bfqq) exists as well. +- */ +- spin_lock_irq(q->queue_lock); +- bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags); +- if (idle_timer_disabled) +- bfqg_stats_update_idle_time(bfqq_group(bfqq)); +- spin_unlock_irq(q->queue_lock); +-#endif ++ bfq_update_insert_stats(q, bfqq, idle_timer_disabled, ++ cmd_flags); + } + + static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx, + +From 0d0d05632872b226f4fae5e56af8736a4c24bf57 Mon Sep 17 00:00:00 2001 +From: Melzani Alessandro <melzani.alessandro@gmail.com> +Date: Mon, 26 Feb 2018 22:43:30 +0100 +Subject: [PATCH 21/23] bfq-sq, bfq-mq: port of "bfq: Use icq_to_bic() + consistently" + +Some code uses icq_to_bic() to convert an io_cq pointer to a +bfq_io_cq pointer while other code uses a direct cast. Convert +the code that uses a direct cast such that it uses icq_to_bic(). + +Signed-off-by: Alessandro Melzani <melzani.alessandro@gmail.com> +--- + block/bfq-mq-iosched.c | 2 +- + block/bfq-sq-iosched.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 5a211620f316..7b1269558c47 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -272,7 +272,7 @@ static const unsigned long max_service_from_wr = 120000; + #define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ + { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) + +-#define RQ_BIC(rq) ((struct bfq_io_cq *) (rq)->elv.priv[0]) ++#define RQ_BIC(rq) icq_to_bic((rq)->elv.priv[0]) + #define RQ_BFQQ(rq) ((rq)->elv.priv[1]) + + /** +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index f95deaab49a1..c4aff8d55fc4 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -266,7 +266,7 @@ static const unsigned long max_service_from_wr = 120000; + #define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ + { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) + +-#define RQ_BIC(rq) ((struct bfq_io_cq *) (rq)->elv.priv[0]) ++#define RQ_BIC(rq) icq_to_bic((rq)->elv.priv[0]) + #define RQ_BFQQ(rq) ((rq)->elv.priv[1]) + + static void bfq_schedule_dispatch(struct bfq_data *bfqd); + +From 4cb5de6add7d6ad0d25d73cb95dc871305db1522 Mon Sep 17 00:00:00 2001 +From: Melzani Alessandro <melzani.alessandro@gmail.com> +Date: Mon, 26 Feb 2018 22:59:30 +0100 +Subject: [PATCH 22/23] bfq-sq, bfq-mq: port of "block, bfq: fix error handle + in bfq_init" + +if elv_register fail, bfq_pool should be free. + +Signed-off-by: Alessandro Melzani <melzani.alessandro@gmail.com> +--- + block/bfq-mq-iosched.c | 4 +++- + block/bfq-sq-iosched.c | 4 +++- + 2 files changed, 6 insertions(+), 2 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 7b1269558c47..964e88c2ce59 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -6129,7 +6129,7 @@ static int __init bfq_init(void) + + ret = elv_register(&iosched_bfq_mq); + if (ret) +- goto err_pol_unreg; ++ goto slab_kill; + + #ifdef BFQ_GROUP_IOSCHED_ENABLED + strcat(msg, " (with cgroups support)"); +@@ -6138,6 +6138,8 @@ static int __init bfq_init(void) + + return 0; + ++slab_kill: ++ bfq_slab_kill(); + err_pol_unreg: + #ifdef BFQ_GROUP_IOSCHED_ENABLED + blkcg_policy_unregister(&blkcg_policy_bfq); +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index c4aff8d55fc4..7f0cf1f01ffc 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -5590,7 +5590,7 @@ static int __init bfq_init(void) + + ret = elv_register(&iosched_bfq); + if (ret) +- goto err_pol_unreg; ++ goto slab_kill; + + #ifdef BFQ_GROUP_IOSCHED_ENABLED + strcat(msg, " (with cgroups support)"); +@@ -5599,6 +5599,8 @@ static int __init bfq_init(void) + + return 0; + ++slab_kill: ++ bfq_slab_kill(); + err_pol_unreg: + #ifdef BFQ_GROUP_IOSCHED_ENABLED + blkcg_policy_unregister(&blkcg_policy_bfq); + +From 1f77c173aaa87ffb22c9f062a6449245d14311e4 Mon Sep 17 00:00:00 2001 +From: Paolo Valente <paolo.valente@linaro.org> +Date: Wed, 4 Apr 2018 11:28:16 +0200 +Subject: [PATCH 23/23] block, bfq-sq, bfq-mq: lower-bound the estimated peak + rate to 1 + +If a storage device handled by BFQ happens to be slower than 7.5 KB/s +for a certain amount of time (in the order of a second), then the +estimated peak rate of the device, maintained in BFQ, becomes equal to +0. The reason is the limited precision with which the rate is +represented (details on the range of representable values in the +comments introduced by this commit). This leads to a division-by-zero +error where the estimated peak rate is used as divisor. Such a type of +failure has been reported in [1]. + +This commit addresses this issue by: +1. Lower-bounding the estimated peak rate to 1 +2. Adding and improving comments on the range of rates representable + +[1] https://www.spinics.net/lists/kernel/msg2739205.html + +Signed-off-by: Konstantin Khlebnikov <khlebnikov@yandex-team.ru> +Signed-off-by: Paolo Valente <paolo.valente@linaro.org> +--- + block/bfq-mq-iosched.c | 25 ++++++++++++++++++++++++- + block/bfq-mq.h | 7 ++++++- + block/bfq-sq-iosched.c | 25 ++++++++++++++++++++++++- + block/bfq.h | 7 ++++++- + 4 files changed, 60 insertions(+), 4 deletions(-) + +diff --git a/block/bfq-mq-iosched.c b/block/bfq-mq-iosched.c +index 964e88c2ce59..03efd90c5d20 100644 +--- a/block/bfq-mq-iosched.c ++++ b/block/bfq-mq-iosched.c +@@ -160,7 +160,20 @@ static struct kmem_cache *bfq_pool; + /* Target observation time interval for a peak-rate update (ns) */ + #define BFQ_RATE_REF_INTERVAL NSEC_PER_SEC + +-/* Shift used for peak rate fixed precision calculations. */ ++/* ++ * Shift used for peak-rate fixed precision calculations. ++ * With ++ * - the current shift: 16 positions ++ * - the current type used to store rate: u32 ++ * - the current unit of measure for rate: [sectors/usec], or, more precisely, ++ * [(sectors/usec) / 2^BFQ_RATE_SHIFT] to take into account the shift, ++ * the range of rates that can be stored is ++ * [1 / 2^BFQ_RATE_SHIFT, 2^(32 - BFQ_RATE_SHIFT)] sectors/usec = ++ * [1 / 2^16, 2^16] sectors/usec = [15e-6, 65536] sectors/usec = ++ * [15, 65G] sectors/sec ++ * Which, assuming a sector size of 512B, corresponds to a range of ++ * [7.5K, 33T] B/sec ++ */ + #define BFQ_RATE_SHIFT 16 + + /* +@@ -2881,6 +2894,16 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + BUG_ON(bfqd->peak_rate > 20<<BFQ_RATE_SHIFT); + + bfqd->peak_rate += rate; ++ ++ /* ++ * For a very slow device, bfqd->peak_rate can reach 0 (see ++ * the minimum representable values reported in the comments ++ * on BFQ_RATE_SHIFT). Push to 1 if this happens, to avoid ++ * divisions by zero where bfqd->peak_rate is used as a ++ * divisor. ++ */ ++ bfqd->peak_rate = max_t(u32, 1, bfqd->peak_rate); ++ + update_thr_responsiveness_params(bfqd); + BUG_ON(bfqd->peak_rate > 20<<BFQ_RATE_SHIFT); + +diff --git a/block/bfq-mq.h b/block/bfq-mq.h +index e2ae11bf8f76..4a54e5076863 100644 +--- a/block/bfq-mq.h ++++ b/block/bfq-mq.h +@@ -490,7 +490,12 @@ struct bfq_data { + u32 last_rq_max_size; + /* time elapsed from first dispatch in current observ. interval (us) */ + u64 delta_from_first; +- /* current estimate of device peak rate */ ++ /* ++ * Current estimate of the device peak rate, measured in ++ * [(sectors/usec) / 2^BFQ_RATE_SHIFT]. The left-shift by ++ * BFQ_RATE_SHIFT is performed to increase precision in ++ * fixed-point calculations. ++ */ + u32 peak_rate; + + /* maximum budget allotted to a bfq_queue before rescheduling */ +diff --git a/block/bfq-sq-iosched.c b/block/bfq-sq-iosched.c +index 7f0cf1f01ffc..e96213865fc2 100644 +--- a/block/bfq-sq-iosched.c ++++ b/block/bfq-sq-iosched.c +@@ -154,7 +154,20 @@ static struct kmem_cache *bfq_pool; + /* Target observation time interval for a peak-rate update (ns) */ + #define BFQ_RATE_REF_INTERVAL NSEC_PER_SEC + +-/* Shift used for peak rate fixed precision calculations. */ ++/* ++ * Shift used for peak-rate fixed precision calculations. ++ * With ++ * - the current shift: 16 positions ++ * - the current type used to store rate: u32 ++ * - the current unit of measure for rate: [sectors/usec], or, more precisely, ++ * [(sectors/usec) / 2^BFQ_RATE_SHIFT] to take into account the shift, ++ * the range of rates that can be stored is ++ * [1 / 2^BFQ_RATE_SHIFT, 2^(32 - BFQ_RATE_SHIFT)] sectors/usec = ++ * [1 / 2^16, 2^16] sectors/usec = [15e-6, 65536] sectors/usec = ++ * [15, 65G] sectors/sec ++ * Which, assuming a sector size of 512B, corresponds to a range of ++ * [7.5K, 33T] B/sec ++ */ + #define BFQ_RATE_SHIFT 16 + + /* +@@ -2695,6 +2708,16 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) + BUG_ON(bfqd->peak_rate > 20<<BFQ_RATE_SHIFT); + + bfqd->peak_rate += rate; ++ ++ /* ++ * For a very slow device, bfqd->peak_rate can reach 0 (see ++ * the minimum representable values reported in the comments ++ * on BFQ_RATE_SHIFT). Push to 1 if this happens, to avoid ++ * divisions by zero where bfqd->peak_rate is used as a ++ * divisor. ++ */ ++ bfqd->peak_rate = max_t(u32, 1, bfqd->peak_rate); ++ + update_thr_responsiveness_params(bfqd); + BUG_ON(bfqd->peak_rate > 20<<BFQ_RATE_SHIFT); + +diff --git a/block/bfq.h b/block/bfq.h +index 4d2fe7f77af1..a25e76c906d9 100644 +--- a/block/bfq.h ++++ b/block/bfq.h +@@ -498,7 +498,12 @@ struct bfq_data { + u32 last_rq_max_size; + /* time elapsed from first dispatch in current observ. interval (us) */ + u64 delta_from_first; +- /* current estimate of device peak rate */ ++ /* ++ * Current estimate of the device peak rate, measured in ++ * [(sectors/usec) / 2^BFQ_RATE_SHIFT]. The left-shift by ++ * BFQ_RATE_SHIFT is performed to increase precision in ++ * fixed-point calculations. ++ */ + u32 peak_rate; + + /* maximum budget allotted to a bfq_queue before rescheduling */ |