+static int nvme_find_closest_node(int node)
+{
+ int n, val, min_val = INT_MAX, best_node = node;
+
+ for_each_online_node(n) {
+ if (n == node)
+ continue;
+ val = node_distance(node, n);
+ if (val < min_val) {
+ min_val = val;
+ best_node = n;
+ }
+ }
+ return best_node;
+}
+
+static void nvme_set_queue_cpus(cpumask_t *qmask, struct nvme_queue *nvmeq,
+ int count)
+{
+ int cpu;
+ for_each_cpu(cpu, qmask) {
+ if (cpumask_weight(nvmeq->cpu_mask) >= count)
+ break;
+ if (!cpumask_test_and_set_cpu(cpu, nvmeq->cpu_mask))
+ *per_cpu_ptr(nvmeq->dev->io_queue, cpu) = nvmeq->qid;
+ }
+}
+
+static void nvme_add_cpus(cpumask_t *mask, const cpumask_t *unassigned_cpus,
+ const cpumask_t *new_mask, struct nvme_queue *nvmeq, int cpus_per_queue)
+{
+ int next_cpu;
+ for_each_cpu(next_cpu, new_mask) {
+ cpumask_or(mask, mask, get_cpu_mask(next_cpu));
+ cpumask_or(mask, mask, topology_thread_cpumask(next_cpu));
+ cpumask_and(mask, mask, unassigned_cpus);
+ nvme_set_queue_cpus(mask, nvmeq, cpus_per_queue);
+ }
+}
+
+static void nvme_create_io_queues(struct nvme_dev *dev)
+{
+ unsigned i, max;
+
+ max = min(dev->max_qid, num_online_cpus());
+ for (i = dev->queue_count; i <= max; i++)
+ if (!nvme_alloc_queue(dev, i, dev->q_depth, i - 1))
+ break;
+
+ max = min(dev->queue_count - 1, num_online_cpus());
+ for (i = dev->online_queues; i <= max; i++)
+ if (nvme_create_queue(raw_nvmeq(dev, i), i))
+ break;
+}
+
+/*
+ * If there are fewer queues than online cpus, this will try to optimally
+ * assign a queue to multiple cpus by grouping cpus that are "close" together:
+ * thread siblings, core, socket, closest node, then whatever else is
+ * available.
+ */
+static void nvme_assign_io_queues(struct nvme_dev *dev)
+{
+ unsigned cpu, cpus_per_queue, queues, remainder, i;
+ cpumask_var_t unassigned_cpus;
+
+ nvme_create_io_queues(dev);
+
+ queues = min(dev->online_queues - 1, num_online_cpus());
+ if (!queues)
+ return;
+
+ cpus_per_queue = num_online_cpus() / queues;
+ remainder = queues - (num_online_cpus() - queues * cpus_per_queue);
+
+ if (!alloc_cpumask_var(&unassigned_cpus, GFP_KERNEL))
+ return;
+
+ cpumask_copy(unassigned_cpus, cpu_online_mask);
+ cpu = cpumask_first(unassigned_cpus);
+ for (i = 1; i <= queues; i++) {
+ struct nvme_queue *nvmeq = lock_nvmeq(dev, i);
+ cpumask_t mask;
+
+ cpumask_clear(nvmeq->cpu_mask);
+ if (!cpumask_weight(unassigned_cpus)) {
+ unlock_nvmeq(nvmeq);
+ break;
+ }
+
+ mask = *get_cpu_mask(cpu);
+ nvme_set_queue_cpus(&mask, nvmeq, cpus_per_queue);
+ if (cpus_weight(mask) < cpus_per_queue)
+ nvme_add_cpus(&mask, unassigned_cpus,
+ topology_thread_cpumask(cpu),
+ nvmeq, cpus_per_queue);
+ if (cpus_weight(mask) < cpus_per_queue)
+ nvme_add_cpus(&mask, unassigned_cpus,
+ topology_core_cpumask(cpu),
+ nvmeq, cpus_per_queue);
+ if (cpus_weight(mask) < cpus_per_queue)
+ nvme_add_cpus(&mask, unassigned_cpus,
+ cpumask_of_node(cpu_to_node(cpu)),
+ nvmeq, cpus_per_queue);
+ if (cpus_weight(mask) < cpus_per_queue)
+ nvme_add_cpus(&mask, unassigned_cpus,
+ cpumask_of_node(
+ nvme_find_closest_node(
+ cpu_to_node(cpu))),
+ nvmeq, cpus_per_queue);
+ if (cpus_weight(mask) < cpus_per_queue)
+ nvme_add_cpus(&mask, unassigned_cpus,
+ unassigned_cpus,
+ nvmeq, cpus_per_queue);
+
+ WARN(cpumask_weight(nvmeq->cpu_mask) != cpus_per_queue,
+ "nvme%d qid:%d mis-matched queue-to-cpu assignment\n",
+ dev->instance, i);
+
+ irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
+ nvmeq->cpu_mask);
+ cpumask_andnot(unassigned_cpus, unassigned_cpus,
+ nvmeq->cpu_mask);
+ cpu = cpumask_next(cpu, unassigned_cpus);
+ if (remainder && !--remainder)
+ cpus_per_queue++;
+ unlock_nvmeq(nvmeq);
+ }
+ WARN(cpumask_weight(unassigned_cpus), "nvme%d unassigned online cpus\n",
+ dev->instance);
+ i = 0;
+ cpumask_andnot(unassigned_cpus, cpu_possible_mask, cpu_online_mask);
+ for_each_cpu(cpu, unassigned_cpus)
+ *per_cpu_ptr(dev->io_queue, cpu) = (i++ % queues) + 1;
+ free_cpumask_var(unassigned_cpus);
+}
+