2 * Block stat tracking code
4 * Copyright (C) 2016 Jens Axboe
6 #include <linux/kernel.h>
7 #include <linux/blk-mq.h>
12 static void blk_stat_flush_batch(struct blk_rq_stat *stat)
14 const s32 nr_batch = READ_ONCE(stat->nr_batch);
15 const s32 nr_samples = READ_ONCE(stat->nr_samples);
20 stat->mean = div64_s64(stat->batch, nr_batch);
22 stat->mean = div64_s64((stat->mean * nr_samples) +
24 nr_batch + nr_samples);
27 stat->nr_samples += nr_batch;
28 stat->nr_batch = stat->batch = 0;
31 static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
36 blk_stat_flush_batch(src);
38 dst->min = min(dst->min, src->min);
39 dst->max = max(dst->max, src->max);
42 dst->mean = src->mean;
44 dst->mean = div64_s64((src->mean * src->nr_samples) +
45 (dst->mean * dst->nr_samples),
46 dst->nr_samples + src->nr_samples);
48 dst->nr_samples += src->nr_samples;
51 static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
53 struct blk_mq_hw_ctx *hctx;
54 struct blk_mq_ctx *ctx;
58 blk_stat_init(&dst[BLK_STAT_READ]);
59 blk_stat_init(&dst[BLK_STAT_WRITE]);
65 queue_for_each_hw_ctx(q, hctx, i) {
66 hctx_for_each_ctx(hctx, ctx, j) {
67 blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
68 blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
70 if (!ctx->stat[BLK_STAT_READ].nr_samples &&
71 !ctx->stat[BLK_STAT_WRITE].nr_samples)
73 if (ctx->stat[BLK_STAT_READ].time > newest)
74 newest = ctx->stat[BLK_STAT_READ].time;
75 if (ctx->stat[BLK_STAT_WRITE].time > newest)
76 newest = ctx->stat[BLK_STAT_WRITE].time;
89 queue_for_each_hw_ctx(q, hctx, i) {
90 hctx_for_each_ctx(hctx, ctx, j) {
91 if (ctx->stat[BLK_STAT_READ].time == newest) {
92 blk_stat_sum(&dst[BLK_STAT_READ],
93 &ctx->stat[BLK_STAT_READ]);
96 if (ctx->stat[BLK_STAT_WRITE].time == newest) {
97 blk_stat_sum(&dst[BLK_STAT_WRITE],
98 &ctx->stat[BLK_STAT_WRITE]);
104 * If we race on finding an entry, just loop back again.
105 * Should be very rare.
109 dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest;
112 void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
115 blk_mq_stat_get(q, dst);
117 blk_stat_flush_batch(&q->rq_stats[BLK_STAT_READ]);
118 blk_stat_flush_batch(&q->rq_stats[BLK_STAT_WRITE]);
119 memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ],
120 sizeof(struct blk_rq_stat));
121 memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE],
122 sizeof(struct blk_rq_stat));
126 void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst)
128 struct blk_mq_ctx *ctx;
135 hctx_for_each_ctx(hctx, ctx, i) {
136 blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
137 blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
139 if (!ctx->stat[BLK_STAT_READ].nr_samples &&
140 !ctx->stat[BLK_STAT_WRITE].nr_samples)
143 if (ctx->stat[BLK_STAT_READ].time > newest)
144 newest = ctx->stat[BLK_STAT_READ].time;
145 if (ctx->stat[BLK_STAT_WRITE].time > newest)
146 newest = ctx->stat[BLK_STAT_WRITE].time;
152 hctx_for_each_ctx(hctx, ctx, i) {
153 if (ctx->stat[BLK_STAT_READ].time == newest) {
154 blk_stat_sum(&dst[BLK_STAT_READ],
155 &ctx->stat[BLK_STAT_READ]);
158 if (ctx->stat[BLK_STAT_WRITE].time == newest) {
159 blk_stat_sum(&dst[BLK_STAT_WRITE],
160 &ctx->stat[BLK_STAT_WRITE]);
165 * If we race on finding an entry, just loop back again.
166 * Should be very rare, as the window is only updated
172 static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now)
175 stat->max = stat->nr_samples = stat->mean = 0;
176 stat->batch = stat->nr_batch = 0;
177 stat->time = time_now & BLK_STAT_NSEC_MASK;
180 void blk_stat_init(struct blk_rq_stat *stat)
182 __blk_stat_init(stat, ktime_to_ns(ktime_get()));
185 static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now)
187 return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK);
190 bool blk_stat_is_current(struct blk_rq_stat *stat)
192 return __blk_stat_is_current(stat, ktime_to_ns(ktime_get()));
195 void blk_stat_add(struct blk_rq_stat *stat, struct request *rq)
199 now = __blk_stat_time(ktime_to_ns(ktime_get()));
200 if (now < blk_stat_time(&rq->issue_stat))
203 if (!__blk_stat_is_current(stat, now))
204 __blk_stat_init(stat, now);
206 value = now - blk_stat_time(&rq->issue_stat);
207 if (value > stat->max)
209 if (value < stat->min)
212 if (stat->batch + value < stat->batch ||
213 stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)
214 blk_stat_flush_batch(stat);
216 stat->batch += value;
220 void blk_stat_clear(struct request_queue *q)
223 struct blk_mq_hw_ctx *hctx;
224 struct blk_mq_ctx *ctx;
227 queue_for_each_hw_ctx(q, hctx, i) {
228 hctx_for_each_ctx(hctx, ctx, j) {
229 blk_stat_init(&ctx->stat[BLK_STAT_READ]);
230 blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
234 blk_stat_init(&q->rq_stats[BLK_STAT_READ]);
235 blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]);
239 void blk_stat_set_issue_time(struct blk_issue_stat *stat)
241 stat->time = (stat->time & BLK_STAT_MASK) |
242 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK);
246 * Enable stat tracking, return whether it was enabled
248 bool blk_stat_enable(struct request_queue *q)
250 if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
251 set_bit(QUEUE_FLAG_STATS, &q->queue_flags);