scheduler: replace __get_cpu_var with this_cpu_ptr

Convert all uses of __get_cpu_var for address calculation to use
this_cpu_ptr instead.

Signed-off-by: Christoph Lameter <cl@linux.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index 51c72be4a7c3f1782a5d5f21801b06bea1720f57..a67e40bd068462973794ae9616af6338c03d99b0 100644 (file)
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -44,8 +44,8 @@ DECLARE_PER_CPU(struct kernel_stat, kstat);
 DECLARE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
 
 /* Must have preemption disabled for this to be meaningful. */
-#define kstat_this_cpu (&__get_cpu_var(kstat))
-#define kcpustat_this_cpu (&__get_cpu_var(kernel_cpustat))
+#define kstat_this_cpu this_cpu_ptr(&kstat)
+#define kcpustat_this_cpu this_cpu_ptr(&kernel_cpustat)
 #define kstat_cpu(cpu) per_cpu(kstat, cpu)
 #define kcpustat_cpu(cpu) per_cpu(kernel_cpustat, cpu)
 

diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c
index 97b67df8fbfed0fe7d3cdbcec0ed213b5df80e6b..c4f63e68a35cd877d6932ce43d36a6d5c68d662c 100644 (file)
--- a/kernel/events/callchain.c
+++ b/kernel/events/callchain.c
@@ -137,7 +137,7 @@ static struct perf_callchain_entry *get_callchain_entry(int *rctx)
        int cpu;
        struct callchain_cpus_entries *entries;
 
-       *rctx = get_recursion_context(__get_cpu_var(callchain_recursion));
+       *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
        if (*rctx == -1)
                return NULL;
 
@@ -153,7 +153,7 @@ static struct perf_callchain_entry *get_callchain_entry(int *rctx)
 static void
 put_callchain_entry(int rctx)
 {
-       put_recursion_context(__get_cpu_var(callchain_recursion), rctx);
+       put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
 }
 
 struct perf_callchain_entry *

diff --git a/kernel/events/core.c b/kernel/events/core.c
index f83a71a3e46d75547e540ed317f99531838f408b..5dbfac3a894f877e6925653acaca35dfb4b4e0e3 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -259,10 +259,10 @@ void perf_sample_event_took(u64 sample_len_ns)
                return;
 
        /* decay the counter by 1 average sample */
-       local_samples_len = __get_cpu_var(running_sample_length);
+       local_samples_len = __this_cpu_read(running_sample_length);
        local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
        local_samples_len += sample_len_ns;
-       __get_cpu_var(running_sample_length) = local_samples_len;
+       __this_cpu_write(running_sample_length, local_samples_len);
 
        /*
         * note: this will be biased artifically low until we have
@@ -874,7 +874,7 @@ static DEFINE_PER_CPU(struct list_head, rotation_list);
 static void perf_pmu_rotate_start(struct pmu *pmu)
 {
        struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
-       struct list_head *head = &__get_cpu_var(rotation_list);
+       struct list_head *head = this_cpu_ptr(&rotation_list);
 
        WARN_ON(!irqs_disabled());
 
@@ -2370,7 +2370,7 @@ void __perf_event_task_sched_out(struct task_struct *task,
         * to check if we have to switch out PMU state.
         * cgroup event are system-wide mode only
         */
-       if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
+       if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
                perf_cgroup_sched_out(task, next);
 }
 
@@ -2613,11 +2613,11 @@ void __perf_event_task_sched_in(struct task_struct *prev,
         * to check if we have to switch in PMU state.
         * cgroup event are system-wide mode only
         */
-       if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
+       if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
                perf_cgroup_sched_in(prev, task);
 
        /* check for system-wide branch_stack events */
-       if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
+       if (atomic_read(this_cpu_ptr(&perf_branch_stack_events)))
                perf_branch_stack_sched_in(prev, task);
 }
 
@@ -2872,7 +2872,7 @@ bool perf_event_can_stop_tick(void)
 
 void perf_event_task_tick(void)
 {
-       struct list_head *head = &__get_cpu_var(rotation_list);
+       struct list_head *head = this_cpu_ptr(&rotation_list);
        struct perf_cpu_context *cpuctx, *tmp;
        struct perf_event_context *ctx;
        int throttled;
@@ -5586,7 +5586,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
                                    struct perf_sample_data *data,
                                    struct pt_regs *regs)
 {
-       struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
+       struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
        struct perf_event *event;
        struct hlist_head *head;
 
@@ -5605,7 +5605,7 @@ end:
 
 int perf_swevent_get_recursion_context(void)
 {
-       struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
+       struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
 
        return get_recursion_context(swhash->recursion);
 }
@@ -5613,7 +5613,7 @@ EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
 
 inline void perf_swevent_put_recursion_context(int rctx)
 {
-       struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
+       struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
 
        put_recursion_context(swhash->recursion, rctx);
 }
@@ -5642,7 +5642,7 @@ static void perf_swevent_read(struct perf_event *event)
 
 static int perf_swevent_add(struct perf_event *event, int flags)
 {
-       struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
+       struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
        struct hw_perf_event *hwc = &event->hw;
        struct hlist_head *head;
 

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d8482e1c575e6e5dd36920b8508e5c1a75ffe316..f5d367d2067a5d493480c31dfc6f872e316dd443 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6412,7 +6412,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
        struct sched_group *group;
        struct rq *busiest;
        unsigned long flags;
-       struct cpumask *cpus = __get_cpu_var(load_balance_mask);
+       struct cpumask *cpus = this_cpu_ptr(load_balance_mask);
 
        struct lb_env env = {
                .sd             = sd,

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index facc824334fb0f2894faac43d4b755beab19c3ed..8e04ba8244d0f3cd9c2084c0855738e35a0733ec 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1448,7 +1448,7 @@ static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
 static int find_lowest_rq(struct task_struct *task)
 {
        struct sched_domain *sd;
-       struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);
+       struct cpumask *lowest_mask = this_cpu_ptr(local_cpu_mask);
        int this_cpu = smp_processor_id();
        int cpu      = task_cpu(task);
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 378bff76267f7c11e5cabcef28c4c7b96a4aad3b..09a7500a86e63bca5466fb22f5cc1bbb479b398f 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -645,10 +645,10 @@ static inline int cpu_of(struct rq *rq)
 DECLARE_PER_CPU(struct rq, runqueues);
 
 #define cpu_rq(cpu)            (&per_cpu(runqueues, (cpu)))
-#define this_rq()              (&__get_cpu_var(runqueues))
+#define this_rq()              this_cpu_ptr(&runqueues)
 #define task_rq(p)             cpu_rq(task_cpu(p))
 #define cpu_curr(cpu)          (cpu_rq(cpu)->curr)
-#define raw_rq()               (&__raw_get_cpu_var(runqueues))
+#define raw_rq()               raw_cpu_ptr(&runqueues)
 
 static inline u64 rq_clock(struct rq *rq)
 {

diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 13d2f7cd65dbfd851eaa61cf08645994c9fd0037..b312fcc730243054a295e717460b3d61e710a26b 100644 (file)
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -638,7 +638,7 @@ void taskstats_exit(struct task_struct *tsk, int group_dead)
                fill_tgid_exit(tsk);
        }
 
-       listeners = __this_cpu_ptr(&listener_array);
+       listeners = raw_cpu_ptr(&listener_array);
        if (list_empty(&listeners->list))
                return;
 

diff --git a/kernel/user-return-notifier.c b/kernel/user-return-notifier.c
index 394f70b17162ef1131c0c0eb04dcb49e55ac717f..9586b670a5b27fef9d4e4f49d8e515059f02d6e4 100644 (file)
--- a/kernel/user-return-notifier.c
+++ b/kernel/user-return-notifier.c
@@ -14,7 +14,7 @@ static DEFINE_PER_CPU(struct hlist_head, return_notifier_list);
 void user_return_notifier_register(struct user_return_notifier *urn)
 {
        set_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY);
-       hlist_add_head(&urn->link, &__get_cpu_var(return_notifier_list));
+       hlist_add_head(&urn->link, this_cpu_ptr(&return_notifier_list));
 }
 EXPORT_SYMBOL_GPL(user_return_notifier_register);
 
@@ -25,7 +25,7 @@ EXPORT_SYMBOL_GPL(user_return_notifier_register);
 void user_return_notifier_unregister(struct user_return_notifier *urn)
 {
        hlist_del(&urn->link);
-       if (hlist_empty(&__get_cpu_var(return_notifier_list)))
+       if (hlist_empty(this_cpu_ptr(&return_notifier_list)))
                clear_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY);
 }
 EXPORT_SYMBOL_GPL(user_return_notifier_unregister);