sched.h 102 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H

4
#include <uapi/linux/sched.h>
5

6
7
#include <linux/sched/prio.h>

8
9
10
11
12

struct sched_param {
	int sched_priority;
};

Linus Torvalds's avatar
Linus Torvalds committed
13
14
15
16
17
18
19
20
#include <asm/param.h>	/* for HZ */

#include <linux/capability.h>
#include <linux/threads.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/timex.h>
#include <linux/jiffies.h>
21
#include <linux/plist.h>
Linus Torvalds's avatar
Linus Torvalds committed
22
23
24
25
26
#include <linux/rbtree.h>
#include <linux/thread_info.h>
#include <linux/cpumask.h>
#include <linux/errno.h>
#include <linux/nodemask.h>
27
#include <linux/mm_types.h>
28
#include <linux/preempt.h>
Linus Torvalds's avatar
Linus Torvalds committed
29
30
31

#include <asm/page.h>
#include <asm/ptrace.h>
32
#include <linux/cputime.h>
Linus Torvalds's avatar
Linus Torvalds committed
33
34
35

#include <linux/smp.h>
#include <linux/sem.h>
36
#include <linux/shm.h>
Linus Torvalds's avatar
Linus Torvalds committed
37
38
39
40
41
42
43
#include <linux/signal.h>
#include <linux/compiler.h>
#include <linux/completion.h>
#include <linux/pid.h>
#include <linux/percpu.h>
#include <linux/topology.h>
#include <linux/seccomp.h>
Ingo Molnar's avatar
Ingo Molnar committed
44
#include <linux/rcupdate.h>
45
#include <linux/rculist.h>
Ingo Molnar's avatar
Ingo Molnar committed
46
#include <linux/rtmutex.h>
Linus Torvalds's avatar
Linus Torvalds committed
47

48
49
50
51
52
#include <linux/time.h>
#include <linux/param.h>
#include <linux/resource.h>
#include <linux/timer.h>
#include <linux/hrtimer.h>
Dmitry Vyukov's avatar
Dmitry Vyukov committed
53
#include <linux/kcov.h>
54
#include <linux/task_io_accounting.h>
Arjan van de Ven's avatar
Arjan van de Ven committed
55
#include <linux/latencytop.h>
56
#include <linux/cred.h>
57
#include <linux/llist.h>
58
#include <linux/uidgid.h>
59
#include <linux/gfp.h>
60
#include <linux/magic.h>
61
#include <linux/cgroup-defs.h>
62
63

#include <asm/processor.h>
H. J. Lu's avatar
H. J. Lu committed
64

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#define SCHED_ATTR_SIZE_VER0	48	/* sizeof first published struct */

/*
 * Extended scheduling parameters data structure.
 *
 * This is needed because the original struct sched_param can not be
 * altered without introducing ABI issues with legacy applications
 * (e.g., in sched_getparam()).
 *
 * However, the possibility of specifying more than just a priority for
 * the tasks may be useful for a wide variety of application fields, e.g.,
 * multimedia, streaming, automation and control, and many others.
 *
 * This variant (sched_attr) is meant at describing a so-called
 * sporadic time-constrained task. In such model a task is specified by:
 *  - the activation period or minimum instance inter-arrival time;
 *  - the maximum (or average, depending on the actual scheduling
 *    discipline) computation time of all instances, a.k.a. runtime;
 *  - the deadline (relative to the actual activation time) of each
 *    instance.
 * Very briefly, a periodic (sporadic) task asks for the execution of
 * some specific computation --which is typically called an instance--
 * (at most) every period. Moreover, each instance typically lasts no more
 * than the runtime and must be completed by time instant t equal to
 * the instance activation time + the deadline.
 *
 * This is reflected by the actual fields of the sched_attr structure:
 *
 *  @size		size of the structure, for fwd/bwd compat.
 *
 *  @sched_policy	task's scheduling policy
 *  @sched_flags	for customizing the scheduler behaviour
 *  @sched_nice		task's nice value      (SCHED_NORMAL/BATCH)
 *  @sched_priority	task's static priority (SCHED_FIFO/RR)
 *  @sched_deadline	representative of the task's deadline
 *  @sched_runtime	representative of the task's runtime
 *  @sched_period	representative of the task's period
 *
 * Given this task model, there are a multiplicity of scheduling algorithms
 * and policies, that can be used to ensure all the tasks will make their
 * timing constraints.
106
107
108
109
 *
 * As of now, the SCHED_DEADLINE policy (sched_dl scheduling class) is the
 * only user of this new interface. More information about the algorithm
 * available in the scheduling class file or in Documentation/.
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
 */
struct sched_attr {
	u32 size;

	u32 sched_policy;
	u64 sched_flags;

	/* SCHED_NORMAL, SCHED_BATCH */
	s32 sched_nice;

	/* SCHED_FIFO, SCHED_RR */
	u32 sched_priority;

	/* SCHED_DEADLINE */
	u64 sched_runtime;
	u64 sched_deadline;
	u64 sched_period;
};

129
struct futex_pi_state;
130
struct robust_list_head;
131
struct bio_list;
132
struct fs_struct;
133
struct perf_event_context;
134
struct blk_plug;
135
struct filename;
136
struct nameidata;
Linus Torvalds's avatar
Linus Torvalds committed
137

Davidlohr Bueso's avatar
Davidlohr Bueso committed
138
139
140
141
#define VMACACHE_BITS 2
#define VMACACHE_SIZE (1U << VMACACHE_BITS)
#define VMACACHE_MASK (VMACACHE_SIZE - 1)

Linus Torvalds's avatar
Linus Torvalds committed
142
143
144
145
146
147
148
149
150
151
152
/*
 * These are the constant used to fake the fixed-point load-average
 * counting. Some notes:
 *  - 11 bit fractions expand to 22 bits by the multiplies: this gives
 *    a load-average precision of 10 bits integer + 11 bits fractional
 *  - if you want to count load-averages more often, you need more
 *    precision, or rounding will get you. With 2-second counting freq,
 *    the EXP_n values would be 1981, 2034 and 2043 if still using only
 *    11 bit fractions.
 */
extern unsigned long avenrun[];		/* Load averages */
153
extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift);
Linus Torvalds's avatar
Linus Torvalds committed
154
155
156

#define FSHIFT		11		/* nr of bits of precision */
#define FIXED_1		(1<<FSHIFT)	/* 1.0 as fixed-point */
157
#define LOAD_FREQ	(5*HZ+1)	/* 5 sec intervals */
Linus Torvalds's avatar
Linus Torvalds committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
#define EXP_1		1884		/* 1/exp(5sec/1min) as fixed-point */
#define EXP_5		2014		/* 1/exp(5sec/5min) */
#define EXP_15		2037		/* 1/exp(5sec/15min) */

#define CALC_LOAD(load,exp,n) \
	load *= exp; \
	load += n*(FIXED_1-exp); \
	load >>= FSHIFT;

extern unsigned long total_forks;
extern int nr_threads;
DECLARE_PER_CPU(unsigned long, process_counts);
extern int nr_processes(void);
extern unsigned long nr_running(void);
172
extern bool single_task_running(void);
Linus Torvalds's avatar
Linus Torvalds committed
173
extern unsigned long nr_iowait(void);
174
extern unsigned long nr_iowait_cpu(int cpu);
175
extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
176

177
extern void calc_global_load(unsigned long ticks);
178
179

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
180
181
extern void cpu_load_update_nohz_start(void);
extern void cpu_load_update_nohz_stop(void);
182
#else
183
184
static inline void cpu_load_update_nohz_start(void) { }
static inline void cpu_load_update_nohz_stop(void) { }
185
#endif
Linus Torvalds's avatar
Linus Torvalds committed
186

187
188
extern void dump_cpu_task(int cpu);

Ingo Molnar's avatar
Ingo Molnar committed
189
190
struct seq_file;
struct cfs_rq;
191
struct task_group;
Ingo Molnar's avatar
Ingo Molnar committed
192
193
194
195
#ifdef CONFIG_SCHED_DEBUG
extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m);
extern void proc_sched_set_task(struct task_struct *p);
#endif
Linus Torvalds's avatar
Linus Torvalds committed
196

197
198
199
200
201
202
203
204
205
206
/*
 * Task state bitmask. NOTE! These bits are also
 * encoded in fs/proc/array.c: get_task_state().
 *
 * We have two separate sets of flags: task->state
 * is about runnability, while task->exit_state are
 * about the task exiting. Confusing, but this way
 * modifying one set can't modify the other one by
 * mistake.
 */
Linus Torvalds's avatar
Linus Torvalds committed
207
208
209
#define TASK_RUNNING		0
#define TASK_INTERRUPTIBLE	1
#define TASK_UNINTERRUPTIBLE	2
Matthew Wilcox's avatar
Matthew Wilcox committed
210
211
#define __TASK_STOPPED		4
#define __TASK_TRACED		8
212
/* in tsk->exit_state */
213
214
#define EXIT_DEAD		16
#define EXIT_ZOMBIE		32
215
#define EXIT_TRACE		(EXIT_ZOMBIE | EXIT_DEAD)
216
/* in tsk->state again */
217
#define TASK_DEAD		64
Matthew Wilcox's avatar
Matthew Wilcox committed
218
#define TASK_WAKEKILL		128
Peter Zijlstra's avatar
Peter Zijlstra committed
219
#define TASK_WAKING		256
220
#define TASK_PARKED		512
221
#define TASK_NOLOAD		1024
222
223
#define TASK_NEW		2048
#define TASK_STATE_MAX		4096
Matthew Wilcox's avatar
Matthew Wilcox committed
224

225
#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn"
226

227
228
extern char ___assert_task_state[1 - 2*!!(
		sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
Matthew Wilcox's avatar
Matthew Wilcox committed
229
230
231
232
233

/* Convenience macros for the sake of set_task_state */
#define TASK_KILLABLE		(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED		(TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED		(TASK_WAKEKILL | __TASK_TRACED)
Linus Torvalds's avatar
Linus Torvalds committed
234

235
236
#define TASK_IDLE		(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)

237
238
/* Convenience macros for the sake of wake_up */
#define TASK_NORMAL		(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
Matthew Wilcox's avatar
Matthew Wilcox committed
239
#define TASK_ALL		(TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
240
241
242

/* get_task_state() */
#define TASK_REPORT		(TASK_RUNNING | TASK_INTERRUPTIBLE | \
Matthew Wilcox's avatar
Matthew Wilcox committed
243
				 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
244
				 __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
245

Matthew Wilcox's avatar
Matthew Wilcox committed
246
247
#define task_is_traced(task)	((task->state & __TASK_TRACED) != 0)
#define task_is_stopped(task)	((task->state & __TASK_STOPPED) != 0)
248
#define task_is_stopped_or_traced(task)	\
Matthew Wilcox's avatar
Matthew Wilcox committed
249
			((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
250
#define task_contributes_to_load(task)	\
251
				((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
252
253
				 (task->flags & PF_FROZEN) == 0 && \
				 (task->state & TASK_NOLOAD) == 0)
Linus Torvalds's avatar
Linus Torvalds committed
254

Peter Zijlstra's avatar
Peter Zijlstra committed
255
256
257
258
259
260
261
262
263
264
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP

#define __set_task_state(tsk, state_value)			\
	do {							\
		(tsk)->task_state_change = _THIS_IP_;		\
		(tsk)->state = (state_value);			\
	} while (0)
#define set_task_state(tsk, state_value)			\
	do {							\
		(tsk)->task_state_change = _THIS_IP_;		\
265
		smp_store_mb((tsk)->state, (state_value));		\
Peter Zijlstra's avatar
Peter Zijlstra committed
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
	} while (0)

/*
 * set_current_state() includes a barrier so that the write of current->state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
 *	set_current_state(TASK_UNINTERRUPTIBLE);
 *	if (do_i_need_to_sleep())
 *		schedule();
 *
 * If the caller does not need such serialisation then use __set_current_state()
 */
#define __set_current_state(state_value)			\
	do {							\
		current->task_state_change = _THIS_IP_;		\
		current->state = (state_value);			\
	} while (0)
#define set_current_state(state_value)				\
	do {							\
		current->task_state_change = _THIS_IP_;		\
287
		smp_store_mb(current->state, (state_value));		\
Peter Zijlstra's avatar
Peter Zijlstra committed
288
289
290
291
	} while (0)

#else

Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
#define __set_task_state(tsk, state_value)		\
	do { (tsk)->state = (state_value); } while (0)
#define set_task_state(tsk, state_value)		\
295
	smp_store_mb((tsk)->state, (state_value))
Linus Torvalds's avatar
Linus Torvalds committed
296

297
298
299
300
301
302
303
304
305
306
307
/*
 * set_current_state() includes a barrier so that the write of current->state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
 *	set_current_state(TASK_UNINTERRUPTIBLE);
 *	if (do_i_need_to_sleep())
 *		schedule();
 *
 * If the caller does not need such serialisation then use __set_current_state()
 */
Peter Zijlstra's avatar
Peter Zijlstra committed
308
#define __set_current_state(state_value)		\
Linus Torvalds's avatar
Linus Torvalds committed
309
	do { current->state = (state_value); } while (0)
Peter Zijlstra's avatar
Peter Zijlstra committed
310
#define set_current_state(state_value)			\
311
	smp_store_mb(current->state, (state_value))
Linus Torvalds's avatar
Linus Torvalds committed
312

Peter Zijlstra's avatar
Peter Zijlstra committed
313
314
#endif

Linus Torvalds's avatar
Linus Torvalds committed
315
316
317
318
319
320
321
322
323
324
325
326
327
328
/* Task command name length */
#define TASK_COMM_LEN 16

#include <linux/spinlock.h>

/*
 * This serializes "schedule()" and also protects
 * the run-queue from deletions/modifications (but
 * _adding_ to the beginning of the run-queue has
 * a separate lock).
 */
extern rwlock_t tasklist_lock;
extern spinlock_t mmlist_lock;

329
struct task_struct;
Linus Torvalds's avatar
Linus Torvalds committed
330

331
332
333
334
#ifdef CONFIG_PROVE_RCU
extern int lockdep_tasklist_lock_is_held(void);
#endif /* #ifdef CONFIG_PROVE_RCU */

Linus Torvalds's avatar
Linus Torvalds committed
335
336
extern void sched_init(void);
extern void sched_init_smp(void);
337
extern asmlinkage void schedule_tail(struct task_struct *prev);
338
extern void init_idle(struct task_struct *idle, int cpu);
339
extern void init_idle_bootup_task(struct task_struct *idle);
Linus Torvalds's avatar
Linus Torvalds committed
340

341
342
extern cpumask_var_t cpu_isolated_map;

343
extern int runqueue_is_locked(int cpu);
Ingo Molnar's avatar
Ingo Molnar committed
344

345
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
346
extern void nohz_balance_enter_idle(int cpu);
347
extern void set_cpu_sd_state_idle(void);
348
extern int get_nohz_timer_target(void);
349
#else
350
static inline void nohz_balance_enter_idle(int cpu) { }
351
static inline void set_cpu_sd_state_idle(void) { }
352
#endif
Linus Torvalds's avatar
Linus Torvalds committed
353

354
/*
355
 * Only dump TASK_* tasks. (0 for all tasks)
356
357
358
359
360
 */
extern void show_state_filter(unsigned long state_filter);

static inline void show_state(void)
{
361
	show_state_filter(0);
362
363
}

Linus Torvalds's avatar
Linus Torvalds committed
364
365
366
367
368
369
370
371
372
373
374
375
376
extern void show_regs(struct pt_regs *);

/*
 * TASK is a pointer to the task whose backtrace we want to see (or NULL for current
 * task), SP is the stack pointer of the first frame that should be shown in the back
 * trace (or NULL if the entire call-chain of the task should be shown).
 */
extern void show_stack(struct task_struct *task, unsigned long *sp);

extern void cpu_init (void);
extern void trap_init(void);
extern void update_process_times(int user);
extern void scheduler_tick(void);
377
extern int sched_cpu_starting(unsigned int cpu);
378
379
extern int sched_cpu_activate(unsigned int cpu);
extern int sched_cpu_deactivate(unsigned int cpu);
Linus Torvalds's avatar
Linus Torvalds committed
380

381
382
383
384
385
#ifdef CONFIG_HOTPLUG_CPU
extern int sched_cpu_dying(unsigned int cpu);
#else
# define sched_cpu_dying	NULL
#endif
Linus Torvalds's avatar
Linus Torvalds committed
386

387
388
extern void sched_show_task(struct task_struct *p);

389
#ifdef CONFIG_LOCKUP_DETECTOR
390
extern void touch_softlockup_watchdog_sched(void);
Ingo Molnar's avatar
Ingo Molnar committed
391
extern void touch_softlockup_watchdog(void);
392
extern void touch_softlockup_watchdog_sync(void);
393
extern void touch_all_softlockup_watchdogs(void);
394
395
396
extern int proc_dowatchdog_thresh(struct ctl_table *table, int write,
				  void __user *buffer,
				  size_t *lenp, loff_t *ppos);
397
extern unsigned int  softlockup_panic;
398
extern unsigned int  hardlockup_panic;
399
void lockup_detector_init(void);
Ingo Molnar's avatar
Ingo Molnar committed
400
#else
401
402
403
static inline void touch_softlockup_watchdog_sched(void)
{
}
Ingo Molnar's avatar
Ingo Molnar committed
404
405
406
static inline void touch_softlockup_watchdog(void)
{
}
407
408
409
static inline void touch_softlockup_watchdog_sync(void)
{
}
410
411
412
static inline void touch_all_softlockup_watchdogs(void)
{
}
413
414
415
static inline void lockup_detector_init(void)
{
}
Ingo Molnar's avatar
Ingo Molnar committed
416
417
#endif

418
419
420
421
422
423
424
425
#ifdef CONFIG_DETECT_HUNG_TASK
void reset_hung_task_detector(void);
#else
static inline void reset_hung_task_detector(void)
{
}
#endif

Linus Torvalds's avatar
Linus Torvalds committed
426
427
/* Attach to any functions which should be ignored in wchan output. */
#define __sched		__attribute__((__section__(".sched.text")))
428
429
430
431

/* Linker adds these: start and end of __sched functions */
extern char __sched_text_start[], __sched_text_end[];

Linus Torvalds's avatar
Linus Torvalds committed
432
433
434
435
/* Is this address in the __sched functions? */
extern int in_sched_functions(unsigned long addr);

#define	MAX_SCHEDULE_TIMEOUT	LONG_MAX
436
extern signed long schedule_timeout(signed long timeout);
437
extern signed long schedule_timeout_interruptible(signed long timeout);
Matthew Wilcox's avatar
Matthew Wilcox committed
438
extern signed long schedule_timeout_killable(signed long timeout);
439
extern signed long schedule_timeout_uninterruptible(signed long timeout);
440
extern signed long schedule_timeout_idle(signed long timeout);
Linus Torvalds's avatar
Linus Torvalds committed
441
asmlinkage void schedule(void);
442
extern void schedule_preempt_disabled(void);
Linus Torvalds's avatar
Linus Torvalds committed
443

444
445
446
447
448
449
450
extern long io_schedule_timeout(long timeout);

static inline void io_schedule(void)
{
	io_schedule_timeout(MAX_SCHEDULE_TIMEOUT);
}

451
452
void __noreturn do_task_dead(void);

453
struct nsproxy;
454
struct user_namespace;
Linus Torvalds's avatar
Linus Torvalds committed
455

456
457
#ifdef CONFIG_MMU
extern void arch_pick_mmap_layout(struct mm_struct *mm);
Linus Torvalds's avatar
Linus Torvalds committed
458
459
460
461
462
463
464
extern unsigned long
arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
		       unsigned long, unsigned long);
extern unsigned long
arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
			  unsigned long len, unsigned long pgoff,
			  unsigned long flags);
465
466
467
#else
static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
#endif
Linus Torvalds's avatar
Linus Torvalds committed
468

469
470
471
472
#define SUID_DUMP_DISABLE	0	/* No setuid dumping */
#define SUID_DUMP_USER		1	/* Dump as user of process */
#define SUID_DUMP_ROOT		2	/* Dump as root */

473
/* mm flags */
Hugh Dickins's avatar
Hugh Dickins committed
474

475
/* for SUID_DUMP_* above */
476
#define MMF_DUMPABLE_BITS 2
Hugh Dickins's avatar
Hugh Dickins committed
477
#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
478

479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
extern void set_dumpable(struct mm_struct *mm, int value);
/*
 * This returns the actual value of the suid_dumpable flag. For things
 * that are using this for checking for privilege transitions, it must
 * test against SUID_DUMP_USER rather than treating it as a boolean
 * value.
 */
static inline int __get_dumpable(unsigned long mm_flags)
{
	return mm_flags & MMF_DUMPABLE_MASK;
}

static inline int get_dumpable(struct mm_struct *mm)
{
	return __get_dumpable(mm->flags);
}

496
497
498
499
500
/* coredump filter bits */
#define MMF_DUMP_ANON_PRIVATE	2
#define MMF_DUMP_ANON_SHARED	3
#define MMF_DUMP_MAPPED_PRIVATE	4
#define MMF_DUMP_MAPPED_SHARED	5
Roland McGrath's avatar
Roland McGrath committed
501
#define MMF_DUMP_ELF_HEADERS	6
502
503
#define MMF_DUMP_HUGETLB_PRIVATE 7
#define MMF_DUMP_HUGETLB_SHARED  8
504
505
#define MMF_DUMP_DAX_PRIVATE	9
#define MMF_DUMP_DAX_SHARED	10
Hugh Dickins's avatar
Hugh Dickins committed
506

507
#define MMF_DUMP_FILTER_SHIFT	MMF_DUMPABLE_BITS
508
#define MMF_DUMP_FILTER_BITS	9
509
510
511
#define MMF_DUMP_FILTER_MASK \
	(((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
#define MMF_DUMP_FILTER_DEFAULT \
512
	((1 << MMF_DUMP_ANON_PRIVATE) |	(1 << MMF_DUMP_ANON_SHARED) |\
513
514
515
516
517
518
519
	 (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)

#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
# define MMF_DUMP_MASK_DEFAULT_ELF	(1 << MMF_DUMP_ELF_HEADERS)
#else
# define MMF_DUMP_MASK_DEFAULT_ELF	0
#endif
Hugh Dickins's avatar
Hugh Dickins committed
520
521
					/* leave room for more dump flags */
#define MMF_VM_MERGEABLE	16	/* KSM may merge identical pages */
Andrea Arcangeli's avatar
Andrea Arcangeli committed
522
#define MMF_VM_HUGEPAGE		17	/* set when VM_HUGEPAGE is set on vma */
523
#define MMF_EXE_FILE_CHANGED	18	/* see prctl_set_mm_exe_file() */
Hugh Dickins's avatar
Hugh Dickins committed
524

525
526
#define MMF_HAS_UPROBES		19	/* has uprobes */
#define MMF_RECALC_UPROBES	20	/* MMF_HAS_UPROBES can be wrong */
527
#define MMF_OOM_SKIP		21	/* mm is of no interest for the OOM killer */
528
#define MMF_UNSTABLE		22	/* mm is unstable for copy_from_user */
529
#define MMF_HUGE_ZERO_PAGE	23      /* mm has ever used the global huge zero page */
530

Hugh Dickins's avatar
Hugh Dickins committed
531
#define MMF_INIT_MASK		(MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
532

Linus Torvalds's avatar
Linus Torvalds committed
533
534
535
536
struct sighand_struct {
	atomic_t		count;
	struct k_sigaction	action[_NSIG];
	spinlock_t		siglock;
Davide Libenzi's avatar
Davide Libenzi committed
537
	wait_queue_head_t	signalfd_wqh;
Linus Torvalds's avatar
Linus Torvalds committed
538
539
};

540
struct pacct_struct {
541
542
	int			ac_flag;
	long			ac_exitcode;
543
	unsigned long		ac_mem;
544
545
	cputime_t		ac_utime, ac_stime;
	unsigned long		ac_minflt, ac_majflt;
546
547
};

548
549
550
struct cpu_itimer {
	cputime_t expires;
	cputime_t incr;
551
552
	u32 error;
	u32 incr_error;
553
554
};

555
/**
556
 * struct prev_cputime - snaphsot of system and user cputime
557
558
 * @utime: time spent in user mode
 * @stime: time spent in system mode
559
 * @lock: protects the above two fields
560
 *
561
562
 * Stores previous user/system time values such that we can guarantee
 * monotonicity.
563
 */
564
565
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
566
567
	cputime_t utime;
	cputime_t stime;
568
569
	raw_spinlock_t lock;
#endif
570
571
};

572
573
574
575
576
577
578
579
static inline void prev_cputime_init(struct prev_cputime *prev)
{
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
	prev->utime = prev->stime = 0;
	raw_spin_lock_init(&prev->lock);
#endif
}

580
581
582
583
584
/**
 * struct task_cputime - collected CPU time counts
 * @utime:		time spent in user mode, in &cputime_t units
 * @stime:		time spent in kernel mode, in &cputime_t units
 * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
585
 *
586
587
588
 * This structure groups together three kinds of CPU time that are tracked for
 * threads and thread groups.  Most things considering CPU time want to group
 * these counts together and treat all three of them in parallel.
589
590
591
592
593
594
 */
struct task_cputime {
	cputime_t utime;
	cputime_t stime;
	unsigned long long sum_exec_runtime;
};
595

596
597
/* Alternate field names when used to cache expirations. */
#define virt_exp	utime
598
#define prof_exp	stime
599
600
#define sched_exp	sum_exec_runtime

601
602
#define INIT_CPUTIME	\
	(struct task_cputime) {					\
603
604
		.utime = 0,					\
		.stime = 0,					\
605
606
607
		.sum_exec_runtime = 0,				\
	}

608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
/*
 * This is the atomic variant of task_cputime, which can be used for
 * storing and updating task_cputime statistics without locking.
 */
struct task_cputime_atomic {
	atomic64_t utime;
	atomic64_t stime;
	atomic64_t sum_exec_runtime;
};

#define INIT_CPUTIME_ATOMIC \
	(struct task_cputime_atomic) {				\
		.utime = ATOMIC64_INIT(0),			\
		.stime = ATOMIC64_INIT(0),			\
		.sum_exec_runtime = ATOMIC64_INIT(0),		\
	}

625
#define PREEMPT_DISABLED	(PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
626

Peter Zijlstra's avatar
Peter Zijlstra committed
627
/*
628
629
 * Disable preemption until the scheduler is running -- use an unconditional
 * value so that it also works on !PREEMPT_COUNT kernels.
630
 *
631
 * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
Peter Zijlstra's avatar
Peter Zijlstra committed
632
 */
633
#define INIT_PREEMPT_COUNT	PREEMPT_OFFSET
634

Peter Zijlstra's avatar
Peter Zijlstra committed
635
/*
636
637
 * Initial preempt_count value; reflects the preempt_count schedule invariant
 * which states that during context switches:
638
 *
639
640
641
642
 *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
 *
 * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
 * Note: See finish_task_switch().
Peter Zijlstra's avatar
Peter Zijlstra committed
643
 */
644
#define FORK_PREEMPT_COUNT	(2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
Peter Zijlstra's avatar
Peter Zijlstra committed
645

646
/**
647
 * struct thread_group_cputimer - thread group interval timer counts
648
 * @cputime_atomic:	atomic thread group interval timers.
649
650
 * @running:		true when there are timers running and
 *			@cputime_atomic receives updates.
651
652
 * @checking_timer:	true when a thread in the group is in the
 *			process of checking for thread group timers.
653
654
 *
 * This structure contains the version of task_cputime, above, that is
655
 * used for thread group CPU timer calculations.
656
 */
657
struct thread_group_cputimer {
658
	struct task_cputime_atomic cputime_atomic;
659
	bool running;
660
	bool checking_timer;
661
662
};

663
#include <linux/rwsem.h>
664
665
struct autogroup;

Linus Torvalds's avatar
Linus Torvalds committed
666
/*
667
 * NOTE! "signal_struct" does not have its own
Linus Torvalds's avatar
Linus Torvalds committed
668
669
670
671
672
673
 * locking, because a shared signal_struct always
 * implies a shared sighand_struct, so locking
 * sighand_struct is always a proper superset of
 * the locking of signal_struct.
 */
struct signal_struct {
674
	atomic_t		sigcnt;
Linus Torvalds's avatar
Linus Torvalds committed
675
	atomic_t		live;
676
	int			nr_threads;
677
	struct list_head	thread_head;
Linus Torvalds's avatar
Linus Torvalds committed
678
679
680
681

	wait_queue_head_t	wait_chldexit;	/* for wait4() */

	/* current thread group signal load-balancing target: */
682
	struct task_struct	*curr_target;
Linus Torvalds's avatar
Linus Torvalds committed
683
684
685
686
687
688
689
690
691
692
693
694

	/* shared signal handling: */
	struct sigpending	shared_pending;

	/* thread group exit support */
	int			group_exit_code;
	/* overloaded:
	 * - notify group_exit_task when ->count is equal to notify_count
	 * - everyone except group_exit_task is stopped during signal delivery
	 *   of fatal signals, group_exit_task processes the signal.
	 */
	int			notify_count;
695
	struct task_struct	*group_exit_task;
Linus Torvalds's avatar
Linus Torvalds committed
696
697
698
699
700

	/* thread group stop support, overloads group_exit_code too */
	int			group_stop_count;
	unsigned int		flags; /* see SIGNAL_* flags below */

701
702
703
704
705
706
707
708
709
710
711
712
	/*
	 * PR_SET_CHILD_SUBREAPER marks a process, like a service
	 * manager, to re-parent orphan (double-forking) child processes
	 * to this process instead of 'init'. The service manager is
	 * able to receive SIGCHLD signals and is able to investigate
	 * the process until it calls wait(). All children of this
	 * process will inherit a flag if they should look for a
	 * child_subreaper process at exit.
	 */
	unsigned int		is_child_subreaper:1;
	unsigned int		has_child_subreaper:1;

Linus Torvalds's avatar
Linus Torvalds committed
713
	/* POSIX.1b Interval Timers */
714
715
	int			posix_timer_id;
	struct list_head	posix_timers;
Linus Torvalds's avatar
Linus Torvalds committed
716
717

	/* ITIMER_REAL timer for the process */
718
	struct hrtimer real_timer;
719
	struct pid *leader_pid;
720
	ktime_t it_real_incr;
Linus Torvalds's avatar
Linus Torvalds committed
721

722
723
724
725
726
727
	/*
	 * ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
	 * CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
	 * values are defined to 0 and 1 respectively
	 */
	struct cpu_itimer it[2];
Linus Torvalds's avatar
Linus Torvalds committed
728

729
	/*
730
731
	 * Thread group totals for process CPU timers.
	 * See thread_group_cputimer(), et al, for details.
732
	 */
733
	struct thread_group_cputimer cputimer;
734
735
736
737

	/* Earliest-expiration cache. */
	struct task_cputime cputime_expires;

738
#ifdef CONFIG_NO_HZ_FULL
739
	atomic_t tick_dep_mask;
740
741
#endif

742
743
	struct list_head cpu_timers[3];

744
	struct pid *tty_old_pgrp;
745

Linus Torvalds's avatar
Linus Torvalds committed
746
747
748
749
750
	/* boolean value for session group leader */
	int leader;

	struct tty_struct *tty; /* NULL if no tty */

751
752
753
#ifdef CONFIG_SCHED_AUTOGROUP
	struct autogroup *autogroup;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
754
755
756
757
758
759
	/*
	 * Cumulative resource counters for dead threads in the group,
	 * and for reaped dead child processes forked by this group.
	 * Live threads maintain their own counters and add to these
	 * in __exit_signal, except for the group leader.
	 */
760
	seqlock_t stats_lock;
761
	cputime_t utime, stime, cutime, cstime;
762
763
	cputime_t gtime;
	cputime_t cgtime;
764
	struct prev_cputime prev_cputime;
Linus Torvalds's avatar
Linus Torvalds committed
765
766
	unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
	unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
767
	unsigned long inblock, oublock, cinblock, coublock;
Jiri Pirko's avatar
Jiri Pirko committed
768
	unsigned long maxrss, cmaxrss;
769
	struct task_io_accounting ioac;
Linus Torvalds's avatar
Linus Torvalds committed
770

771
772
773
774
775
776
777
778
	/*
	 * Cumulative ns of schedule CPU time fo dead threads in the
	 * group, not including a zombie group leader, (This only differs
	 * from jiffies_to_ns(utime + stime) if sched_clock uses something
	 * other than jiffies.)
	 */
	unsigned long long sum_sched_runtime;

Linus Torvalds's avatar
Linus Torvalds committed
779
780
781
782
783
784
785
786
787
788
789
	/*
	 * We don't bother to synchronize most readers of this at all,
	 * because there is no reader checking a limit that actually needs
	 * to get both rlim_cur and rlim_max atomically, and either one
	 * alone is a single word that can safely be read normally.
	 * getrlimit/setrlimit use task_lock(current->group_leader) to
	 * protect this instead of the siglock, because they really
	 * have no need to disable irqs.
	 */
	struct rlimit rlim[RLIM_NLIMITS];

790
791
792
#ifdef CONFIG_BSD_PROCESS_ACCT
	struct pacct_struct pacct;	/* per-process accounting information */
#endif
793
794
795
#ifdef CONFIG_TASKSTATS
	struct taskstats *stats;
#endif
Miloslav Trmac's avatar
Miloslav Trmac committed
796
797
798
799
#ifdef CONFIG_AUDIT
	unsigned audit_tty;
	struct tty_audit_buf *tty_audit_buf;
#endif
800

Tetsuo Handa's avatar
Tetsuo Handa committed
801
802
803
804
805
	/*
	 * Thread is the potential origin of an oom condition; kill first on
	 * oom
	 */
	bool oom_flag_origin;
806
807
808
	short oom_score_adj;		/* OOM kill score adjustment */
	short oom_score_adj_min;	/* OOM kill score adjustment min value.
					 * Only settable by CAP_SYS_RESOURCE. */
809
810
	struct mm_struct *oom_mm;	/* recorded mm when the thread group got
					 * killed by the oom killer */
811
812
813
814

	struct mutex cred_guard_mutex;	/* guard against foreign influences on
					 * credential calculations
					 * (notably. ptrace) */
Linus Torvalds's avatar
Linus Torvalds committed
815
816
817
818
819
820
};

/*
 * Bits in flags field of signal_struct.
 */
#define SIGNAL_STOP_STOPPED	0x00000001 /* job control stop in effect */
821
822
#define SIGNAL_STOP_CONTINUED	0x00000002 /* SIGCONT since WCONTINUED reap */
#define SIGNAL_GROUP_EXIT	0x00000004 /* group exit in progress */
823
#define SIGNAL_GROUP_COREDUMP	0x00000008 /* coredump in progress */
824
825
826
827
828
829
/*
 * Pending notifications to parent.
 */
#define SIGNAL_CLD_STOPPED	0x00000010
#define SIGNAL_CLD_CONTINUED	0x00000020
#define SIGNAL_CLD_MASK		(SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
Linus Torvalds's avatar
Linus Torvalds committed
830

831
832
#define SIGNAL_UNKILLABLE	0x00000040 /* for init: ignore fatal signals */

833
834
835
836
837
838
839
/* If true, all threads except ->group_exit_task have pending SIGKILL */
static inline int signal_group_exit(const struct signal_struct *sig)
{
	return	(sig->flags & SIGNAL_GROUP_EXIT) ||
		(sig->group_exit_task != NULL);
}

Linus Torvalds's avatar
Linus Torvalds committed
840
841
842
843
844
845
846
/*
 * Some day this will be a full-fledged user tracking system..
 */
struct user_struct {
	atomic_t __count;	/* reference count */
	atomic_t processes;	/* How many processes does this user have? */
	atomic_t sigpending;	/* How many pending signals does this user have? */
847
#ifdef CONFIG_INOTIFY_USER
Robert Love's avatar
Robert Love committed
848
849
850
	atomic_t inotify_watches; /* How many inotify watches does this user have? */
	atomic_t inotify_devs;	/* How many inotify devs does this user have opened? */
#endif
851
852
853
#ifdef CONFIG_FANOTIFY
	atomic_t fanotify_listeners;
#endif
854
#ifdef CONFIG_EPOLL
855
	atomic_long_t epoll_watches; /* The number of file descriptors currently watched */
856
#endif
Alexey Dobriyan's avatar
Alexey Dobriyan committed
857
#ifdef CONFIG_POSIX_MQUEUE
Linus Torvalds's avatar
Linus Torvalds committed
858
859
	/* protected by mq_lock	*/
	unsigned long mq_bytes;	/* How many bytes can be allocated to mqueue? */
Alexey Dobriyan's avatar
Alexey Dobriyan committed
860
#endif
Linus Torvalds's avatar
Linus Torvalds committed
861
	unsigned long locked_shm; /* How many pages of mlocked shm ? */
862
	unsigned long unix_inflight;	/* How many files in flight in unix sockets */
863
	atomic_long_t pipe_bufs;  /* how many pages are allocated in pipe buffers */
Linus Torvalds's avatar
Linus Torvalds committed
864
865
866
867
868
869
870

#ifdef CONFIG_KEYS
	struct key *uid_keyring;	/* UID specific keyring */
	struct key *session_keyring;	/* UID's default session keyring */
#endif

	/* Hash table maintenance information */
Pavel Emelyanov's avatar
Pavel Emelyanov committed
871
	struct hlist_node uidhash_node;
872
	kuid_t uid;
873

874
#if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL)
875
876
	atomic_long_t locked_vm;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
877
878
};

879
extern int uids_sysfs_init(void);
880

881
extern struct user_struct *find_user(kuid_t);
Linus Torvalds's avatar
Linus Torvalds committed
882
883
884
885

extern struct user_struct root_user;
#define INIT_USER (&root_user)

886

Linus Torvalds's avatar
Linus Torvalds committed
887
888
889
struct backing_dev_info;
struct reclaim_state;

890
#ifdef CONFIG_SCHED_INFO
Linus Torvalds's avatar
Linus Torvalds committed
891
892
struct sched_info {
	/* cumulative counters */
893
	unsigned long pcount;	      /* # of times run on this cpu */
894
	unsigned long long run_delay; /* time spent waiting on a runqueue */
Linus Torvalds's avatar
Linus Torvalds committed
895
896

	/* timestamps */
897
898
	unsigned long long last_arrival,/* when we last ran on a cpu */
			   last_queued;	/* when we were last queued to run */
Linus Torvalds's avatar
Linus Torvalds committed
899
};
900
#endif /* CONFIG_SCHED_INFO */
Linus Torvalds's avatar
Linus Torvalds committed
901

902
903
904
905
906
907
908
909
910
911
912
913
914
915
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info {
	spinlock_t	lock;
	unsigned int	flags;	/* Private per-task flags */

	/* For each stat XXX, add following, aligned appropriately
	 *
	 * struct timespec XXX_start, XXX_end;
	 * u64 XXX_delay;
	 * u32 XXX_count;
	 *
	 * Atomicity of updates to XXX_delay, XXX_count protected by
	 * single lock above (split into XXX_lock if contention is an issue).
	 */
916
917
918
919
920
921

	/*
	 * XXX_count is incremented on every XXX operation, the delay
	 * associated with the operation is added to XXX_delay.
	 * XXX_delay contains the accumulated delay time in nanoseconds.
	 */
922
	u64 blkio_start;	/* Shared by blkio, swapin */
923
924
925
926
927
928
	u64 blkio_delay;	/* wait for sync block io completion */
	u64 swapin_delay;	/* wait for swapin block io completion */
	u32 blkio_count;	/* total count of the number of sync block */
				/* io operations performed */
	u32 swapin_count;	/* total count of the number of swapin block */
				/* io operations performed */
929

930
	u64 freepages_start;
931
932
	u64 freepages_delay;	/* wait for memory reclaim */
	u32 freepages_count;	/* total count of memory reclaim */
933
};
934
935
936
937
938
939
940
941
942
943
944
#endif	/* CONFIG_TASK_DELAY_ACCT */

static inline int sched_info_on(void)
{
#ifdef CONFIG_SCHEDSTATS
	return 1;
#elif defined(CONFIG_TASK_DELAY_ACCT)
	extern int delayacct_on;
	return delayacct_on;
#else
	return 0;
945
#endif
946
}
947

948
949
950
951
#ifdef CONFIG_SCHEDSTATS
void force_schedstat_enabled(void);
#endif

952
953
954
955
956
enum cpu_idle_type {
	CPU_IDLE,
	CPU_NOT_IDLE,
	CPU_NEWLY_IDLE,
	CPU_MAX_IDLE_TYPES
Linus Torvalds's avatar
Linus Torvalds committed
957
958
};

959
960
961
962
963
964
965
966
967
968
/*
 * Integer metrics need fixed point arithmetic, e.g., sched/fair
 * has a few: load, load_avg, util_avg, freq, and capacity.
 *
 * We define a basic fixed point arithmetic range, and then formalize
 * all these metrics based on that basic range.
 */
# define SCHED_FIXEDPOINT_SHIFT	10
# define SCHED_FIXEDPOINT_SCALE	(1L << SCHED_FIXEDPOINT_SHIFT)

969
/*
970
 * Increase resolution of cpu_capacity calculations
971
 */
972
#define SCHED_CAPACITY_SHIFT	SCHED_FIXEDPOINT_SHIFT
973
#define SCHED_CAPACITY_SCALE	(1L << SCHED_CAPACITY_SHIFT)
Linus Torvalds's avatar
Linus Torvalds committed
974

975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
/*
 * Wake-queues are lists of tasks with a pending wakeup, whose
 * callers have already marked the task as woken internally,
 * and can thus carry on. A common use case is being able to
 * do the wakeups once the corresponding user lock as been
 * released.
 *
 * We hold reference to each task in the list across the wakeup,
 * thus guaranteeing that the memory is still valid by the time
 * the actual wakeups are performed in wake_up_q().
 *
 * One per task suffices, because there's never a need for a task to be
 * in two wake queues simultaneously; it is forbidden to abandon a task
 * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is
 * already in a wake queue, the wakeup will happen soon and the second
 * waker can just skip it.
 *
 * The WAKE_Q macro declares and initializes the list head.
 * wake_up_q() does NOT reinitialize the list; it's expected to be
 * called near the end of a function, where the fact that the queue is
 * not used again will be easy to see by inspection.
 *
 * Note that this can cause spurious wakeups. schedule() callers
 * must ensure the call is done inside a loop, confirming that the
 * wakeup condition has in fact occurred.
 */
struct wake_q_node {
	struct wake_q_node *next;
};

struct wake_q_head {
	struct wake_q_node *first;
	struct wake_q_node **lastp;
};

#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01)

#define WAKE_Q(name)					\
	struct wake_q_head name = { WAKE_Q_TAIL, &name.first }

extern void wake_q_add(struct wake_q_head *head,
		       struct task_struct *task);
extern void wake_up_q(struct wake_q_head *head);

1019
1020
1021
/*
 * sched-domains (multiprocessor balancing) declarations:
 */
1022
#ifdef CONFIG_SMP
Peter Zijlstra's avatar
Peter Zijlstra committed
1023
1024
1025
1026
#define SD_LOAD_BALANCE		0x0001	/* Do load balancing on this domain. */
#define SD_BALANCE_NEWIDLE	0x0002	/* Balance when about to become idle */
#define SD_BALANCE_EXEC		0x0004	/* Balance on exec */
#define SD_BALANCE_FORK		0x0008	/* Balance on fork, clone */
1027
#define SD_BALANCE_WAKE		0x0010  /* Balance on wakeup */
Peter Zijlstra's avatar
Peter Zijlstra committed
1028
#define SD_WAKE_AFFINE		0x0020	/* Wake task to waking CPU */
1029
#define SD_ASYM_CPUCAPACITY	0x0040  /* Groups have different max cpu capacities */
1030
#define SD_SHARE_CPUCAPACITY	0x0080	/* Domain members share cpu capacity */
1031
#define SD_SHARE_POWERDOMAIN	0x0100	/* Domain members share power domain */
Peter Zijlstra's avatar
Peter Zijlstra committed
1032
1033
#define SD_SHARE_PKG_RESOURCES	0x0200	/* Domain members share cpu pkg resources */
#define SD_SERIALIZE		0x0400	/* Only a single load balancing instance */
1034
#define SD_ASYM_PACKING		0x0800  /* Place busy groups earlier in the domain */
Peter Zijlstra's avatar
Peter Zijlstra committed
1035
#define SD_PREFER_SIBLING	0x1000	/* Prefer to place tasks in a sibling domain */
1036
#define SD_OVERLAP		0x2000	/* sched_domains of this level overlap */
1037
#define SD_NUMA			0x4000	/* cross-node balancing */
1038

1039
#ifdef CONFIG_SCHED_SMT
Guenter Roeck's avatar
Guenter Roeck committed
1040
static inline int cpu_smt_flags(void)
1041
{
1042
	return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
1043
1044
1045
1046
}
#endif

#ifdef CONFIG_SCHED_MC
Guenter Roeck's avatar
Guenter Roeck committed
1047
static inline int cpu_core_flags(void)
1048
1049
1050
1051
1052
1053
{
	return SD_SHARE_PKG_RESOURCES;
}
#endif

#ifdef CONFIG_NUMA
Guenter Roeck's avatar
Guenter Roeck committed
1054
static inline int cpu_numa_flags(void)
1055
1056
1057
1058
{
	return SD_NUMA;
}
#endif
1059

1060
1061
1062
1063
1064
1065
1066
1067
struct sched_domain_attr {
	int relax_domain_level;
};

#define SD_ATTR_INIT	(struct sched_domain_attr) {	\
	.relax_domain_level = -1,			\
}

1068
1069
extern int sched_domain_level_max;

1070
1071
struct sched_group;

1072
1073
struct sched_domain_shared {
	atomic_t	ref;
1074
	atomic_t	nr_busy_cpus;
1075
	int		has_idle_cores;
1076
1077
};

Linus Torvalds's avatar
Linus Torvalds committed
1078
1079
1080
struct sched_domain {
	/* These fields must be setup */
	struct sched_domain *parent;	/* top domain must be null terminated */
1081
	struct sched_domain *child;	/* bottom domain must be null terminated */
Linus Torvalds's avatar
Linus Torvalds committed
1082
1083
1084
1085
1086
1087
	struct sched_group *groups;	/* the balancing groups of the domain */
	unsigned long min_interval;	/* Minimum balance interval ms */
	unsigned long max_interval;	/* Maximum balance interval ms */
	unsigned int busy_factor;	/* less balancing by factor if busy */
	unsigned int imbalance_pct;	/* No balance until over watermark */
	unsigned int cache_nice_tries;	/* Leave cache hot tasks for # tries */
Nick Piggin's avatar
Nick Piggin committed
1088
1089
1090
1091
	unsigned int busy_idx;
	unsigned int idle_idx;
	unsigned int newidle_idx;
	unsigned int wake_idx;
Nick Piggin's avatar
Nick Piggin committed
1092
	unsigned int forkexec_idx;
Peter Zijlstra's avatar
Peter Zijlstra committed
1093
	unsigned int smt_gain;
1094
1095

	int nohz_idle;			/* NOHZ IDLE status */
Linus Torvalds's avatar
Linus Torvalds committed
1096
	int flags;			/* See SD_* */
1097
	int level;
Linus Torvalds's avatar
Linus Torvalds committed
1098
1099
1100
1101
1102
1103

	/* Runtime fields. */
	unsigned long last_balance;	/* init to jiffies. units in jiffies */
	unsigned int balance_interval;	/* initialise to 1. units in ms. */
	unsigned int nr_balance_failed; /* initialise to 0 */

1104
	/* idle_balance() stats */
1105
	u64 max_newidle_lb_cost;
1106
	unsigned long next_decay_max_lb_cost;
1107

1108
1109
	u64 avg_scan_cost;		/* select_idle_sibling */

Linus Torvalds's avatar
Linus Torvalds committed
1110
1111
#ifdef CONFIG_SCHEDSTATS
	/* load_balance() stats */
1112
1113
1114
1115
1116
1117
1118
1119
	unsigned int lb_count[CPU_MAX_IDLE_TYPES];
	unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
	unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
	unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
	unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
	unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
	unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
	unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
Linus Torvalds's avatar
Linus Torvalds committed
1120
1121

	/* Active load balancing */
1122
1123
1124
	unsigned int alb_count;
	unsigned int alb_failed;
	unsigned int alb_pushed;
Linus Torvalds's avatar
Linus Torvalds committed
1125

1126
	/* SD_BALANCE_EXEC stats */
1127
1128
1129
	unsigned int sbe_count;
	unsigned int sbe_balanced;
	unsigned int sbe_pushed;
Linus Torvalds's avatar
Linus Torvalds committed
1130

1131
	/* SD_BALANCE_FORK stats */