thp: reduce usage of huge zero page's atomic counter

[karo-tx-linux.git] / include / linux / sched.h

diff --git a/include/linux/sched.h b/include/linux/sched.h
index abb795afc8237ad5bef0cb8d61f3eafebaf0617d..348f51b0ec92ed02e72a2060eedb03f37cd0995f 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -448,6 +448,8 @@ static inline void io_schedule(void)
        io_schedule_timeout(MAX_SCHEDULE_TIMEOUT);
 }
 
+void __noreturn do_task_dead(void);
+
 struct nsproxy;
 struct user_namespace;
 
@@ -522,8 +524,9 @@ static inline int get_dumpable(struct mm_struct *mm)
 
 #define MMF_HAS_UPROBES                19      /* has uprobes */
 #define MMF_RECALC_UPROBES     20      /* MMF_HAS_UPROBES can be wrong */
-#define MMF_OOM_REAPED         21      /* mm has been already reaped */
-#define MMF_OOM_NOT_REAPABLE   22      /* mm couldn't be reaped */
+#define MMF_OOM_SKIP           21      /* mm is of no interest for the OOM killer */
+#define MMF_UNSTABLE           22      /* mm is unstable for copy_from_user */
+#define MMF_HUGE_ZERO_PAGE     23      /* mm has ever used the global huge zero page */
 
 #define MMF_INIT_MASK          (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
 
@@ -671,7 +674,6 @@ struct signal_struct {
        atomic_t                sigcnt;
        atomic_t                live;
        int                     nr_threads;
-       atomic_t oom_victims; /* # of TIF_MEDIE threads in this thread group */
        struct list_head        thread_head;
 
        wait_queue_head_t       wait_chldexit;  /* for wait4() */
@@ -804,6 +806,8 @@ struct signal_struct {
        short oom_score_adj;            /* OOM kill score adjustment */
        short oom_score_adj_min;        /* OOM kill score adjustment min value.
                                         * Only settable by CAP_SYS_RESOURCE. */
+       struct mm_struct *oom_mm;       /* recorded mm when the thread group got
+                                        * killed by the oom killer */
 
        struct mutex cred_guard_mutex;  /* guard against foreign influences on
                                         * credential calculations
@@ -1022,7 +1026,8 @@ extern void wake_up_q(struct wake_q_head *head);
 #define SD_BALANCE_FORK                0x0008  /* Balance on fork, clone */
 #define SD_BALANCE_WAKE                0x0010  /* Balance on wakeup */
 #define SD_WAKE_AFFINE         0x0020  /* Wake task to waking CPU */
-#define SD_SHARE_CPUCAPACITY   0x0080  /* Domain members share cpu power */
+#define SD_ASYM_CPUCAPACITY    0x0040  /* Groups have different max cpu capacities */
+#define SD_SHARE_CPUCAPACITY   0x0080  /* Domain members share cpu capacity */
 #define SD_SHARE_POWERDOMAIN   0x0100  /* Domain members share power domain */
 #define SD_SHARE_PKG_RESOURCES 0x0200  /* Domain members share cpu pkg resources */
 #define SD_SERIALIZE           0x0400  /* Only a single load balancing instance */
@@ -1064,6 +1069,12 @@ extern int sched_domain_level_max;
 
 struct sched_group;
 
+struct sched_domain_shared {
+       atomic_t        ref;
+       atomic_t        nr_busy_cpus;
+       int             has_idle_cores;
+};
+
 struct sched_domain {
        /* These fields must be setup */
        struct sched_domain *parent;    /* top domain must be null terminated */
@@ -1094,6 +1105,8 @@ struct sched_domain {
        u64 max_newidle_lb_cost;
        unsigned long next_decay_max_lb_cost;
 
+       u64 avg_scan_cost;              /* select_idle_sibling */
+
 #ifdef CONFIG_SCHEDSTATS
        /* load_balance() stats */
        unsigned int lb_count[CPU_MAX_IDLE_TYPES];
@@ -1132,6 +1145,7 @@ struct sched_domain {
                void *private;          /* used during construction */
                struct rcu_head rcu;    /* used during destruction */
        };
+       struct sched_domain_shared *shared;
 
        unsigned int span_weight;
        /*
@@ -1165,6 +1179,7 @@ typedef int (*sched_domain_flags_f)(void);
 
 struct sd_data {
        struct sched_domain **__percpu sd;
+       struct sched_domain_shared **__percpu sds;
        struct sched_group **__percpu sg;
        struct sched_group_capacity **__percpu sgc;
 };
@@ -2597,7 +2612,7 @@ static inline bool is_idle_task(const struct task_struct *p)
        return p->pid == 0;
 }
 extern struct task_struct *curr_task(int cpu);
-extern void set_curr_task(int cpu, struct task_struct *p);
+extern void ia64_set_curr_task(int cpu, struct task_struct *p);
 
 void yield(void);
 
@@ -2863,6 +2878,20 @@ static inline void mmdrop(struct mm_struct *mm)
                __mmdrop(mm);
 }
 
+static inline void mmdrop_async_fn(struct work_struct *work)
+{
+       struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
+       __mmdrop(mm);
+}
+
+static inline void mmdrop_async(struct mm_struct *mm)
+{
+       if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
+               INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
+               schedule_work(&mm->async_put_work);
+       }
+}
+
 static inline bool mmget_not_zero(struct mm_struct *mm)
 {
        return atomic_inc_not_zero(&mm->mm_users);
@@ -3279,7 +3308,11 @@ static inline int signal_pending_state(long state, struct task_struct *p)
  * cond_resched_lock() will drop the spinlock before scheduling,
  * cond_resched_softirq() will enable bhs before scheduling.
  */
+#ifndef CONFIG_PREEMPT
 extern int _cond_resched(void);
+#else
+static inline int _cond_resched(void) { return 0; }
+#endif
 
 #define cond_resched() ({                      \
        ___might_sleep(__FILE__, __LINE__, 0);  \
@@ -3309,6 +3342,15 @@ static inline void cond_resched_rcu(void)
 #endif
 }
 
+static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+       return p->preempt_disable_ip;
+#else
+       return 0;
+#endif
+}
+
 /*
  * Does a critical section need to be broken due to another
  * task waiting?: (technically does not depend on CONFIG_PREEMPT,
@@ -3546,15 +3588,20 @@ static inline unsigned long rlimit_max(unsigned int limit)
        return task_rlimit_max(current, limit);
 }
 
+#define SCHED_CPUFREQ_RT       (1U << 0)
+#define SCHED_CPUFREQ_DL       (1U << 1)
+#define SCHED_CPUFREQ_IOWAIT   (1U << 2)
+
+#define SCHED_CPUFREQ_RT_DL    (SCHED_CPUFREQ_RT | SCHED_CPUFREQ_DL)
+
 #ifdef CONFIG_CPU_FREQ
 struct update_util_data {
-       void (*func)(struct update_util_data *data,
-                    u64 time, unsigned long util, unsigned long max);
+       void (*func)(struct update_util_data *data, u64 time, unsigned int flags);
 };
 
 void cpufreq_add_update_util_hook(int cpu, struct update_util_data *data,
-                       void (*func)(struct update_util_data *data, u64 time,
-                                    unsigned long util, unsigned long max));
+                       void (*func)(struct update_util_data *data, u64 time,
+                                   unsigned int flags));
 void cpufreq_remove_update_util_hook(int cpu);
 #endif /* CONFIG_CPU_FREQ */