#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
+#include "blk-mq-debugfs.h"
#include "blk-mq-tag.h"
#include "blk-stat.h"
#include "blk-wbt.h"
static void blk_mq_poll_stats_start(struct request_queue *q);
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
+static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync);
static int blk_mq_poll_stats_bkt(const struct request *rq)
{
unsigned int i;
bool rcu = false;
- blk_mq_stop_hw_queues(q);
+ __blk_mq_stop_hw_queues(q, true);
queue_for_each_hw_ctx(q, hctx, i) {
if (hctx->flags & BLK_MQ_F_BLOCKING)
}
EXPORT_SYMBOL(blk_mq_queue_stopped);
-void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
+static void __blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx, bool sync)
{
- cancel_delayed_work_sync(&hctx->run_work);
+ if (sync)
+ cancel_delayed_work_sync(&hctx->run_work);
+ else
+ cancel_delayed_work(&hctx->run_work);
+
set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
+
+void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
+{
+ __blk_mq_stop_hw_queue(hctx, false);
+}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);
-void blk_mq_stop_hw_queues(struct request_queue *q)
+static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync)
{
struct blk_mq_hw_ctx *hctx;
int i;
queue_for_each_hw_ctx(q, hctx, i)
- blk_mq_stop_hw_queue(hctx);
+ __blk_mq_stop_hw_queue(hctx, sync);
+}
+
+void blk_mq_stop_hw_queues(struct request_queue *q)
+{
+ __blk_mq_stop_hw_queues(q, false);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);
blk_queue_bounce(q, &bio);
+ blk_queue_split(q, &bio, q->bio_split);
+
if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
bio_io_error(bio);
return BLK_QC_T_NONE;
}
- blk_queue_split(q, &bio, q->bio_split);
-
if (!is_flush_fua && !blk_queue_nomerges(q) &&
blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
return BLK_QC_T_NONE;
if (!rq)
continue;
- set->ops->exit_request(set->driver_data, rq,
- hctx_idx, i);
+ set->ops->exit_request(set, rq, hctx_idx);
tags->static_rqs[i] = NULL;
}
}
tags->static_rqs[i] = rq;
if (set->ops->init_request) {
- if (set->ops->init_request(set->driver_data,
- rq, hctx_idx, i,
+ if (set->ops->init_request(set, rq, hctx_idx,
node)) {
tags->static_rqs[i] = NULL;
goto fail;
struct blk_mq_tag_set *set,
struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
- unsigned flush_start_tag = set->queue_depth;
+ blk_mq_debugfs_unregister_hctx(hctx);
blk_mq_tag_idle(hctx);
if (set->ops->exit_request)
- set->ops->exit_request(set->driver_data,
- hctx->fq->flush_rq, hctx_idx,
- flush_start_tag + hctx_idx);
+ set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
blk_mq_sched_exit_hctx(q, hctx, hctx_idx);
struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
{
int node;
- unsigned flush_start_tag = set->queue_depth;
node = hctx->numa_node;
if (node == NUMA_NO_NODE)
goto sched_exit_hctx;
if (set->ops->init_request &&
- set->ops->init_request(set->driver_data,
- hctx->fq->flush_rq, hctx_idx,
- flush_start_tag + hctx_idx, node))
+ set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx,
+ node))
goto free_fq;
if (hctx->flags & BLK_MQ_F_BLOCKING)
init_srcu_struct(&hctx->queue_rq_srcu);
+ blk_mq_debugfs_register_hctx(q, hctx);
+
return 0;
free_fq:
{
WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
+ blk_mq_debugfs_unregister_hctxs(q);
blk_mq_sysfs_unregister(q);
/*
blk_mq_map_swqueue(q, online_mask);
blk_mq_sysfs_register(q);
+ blk_mq_debugfs_register_hctxs(q);
}
/*
return -EINVAL;
blk_mq_freeze_queue(q);
- blk_mq_quiesce_queue(q);
ret = 0;
queue_for_each_hw_ctx(q, hctx, i) {
q->nr_requests = nr;
blk_mq_unfreeze_queue(q);
- blk_mq_start_stopped_hw_queues(q, true);
return ret;
}