cpuidle.c 11.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
/*
 * cpuidle.c - core cpuidle infrastructure
 *
 * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
 *               Shaohua Li <shaohua.li@intel.com>
 *               Adam Belay <abelay@novell.com>
 *
 * This code is licenced under the GPL.
 */

#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/sched.h>
#include <linux/notifier.h>
15
#include <linux/pm_qos.h>
16
17
#include <linux/cpu.h>
#include <linux/cpuidle.h>
18
#include <linux/ktime.h>
19
#include <linux/hrtimer.h>
20
#include <linux/module.h>
21
#include <trace/events/power.h>
22
23
24
25
26
27
28
29
30

#include "cpuidle.h"

DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);

DEFINE_MUTEX(cpuidle_lock);
LIST_HEAD(cpuidle_detected_devices);

static int enabled_devices;
31
static int off __read_mostly;
Len Brown's avatar
Len Brown committed
32
static int initialized __read_mostly;
33
34
35
36
37

int cpuidle_disabled(void)
{
	return off;
}
38
39
40
41
void disable_cpuidle(void)
{
	off = 1;
}
42

43
44
static int __cpuidle_register_device(struct cpuidle_device *dev);

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
static inline int cpuidle_enter(struct cpuidle_device *dev,
				struct cpuidle_driver *drv, int index)
{
	struct cpuidle_state *target_state = &drv->states[index];
	return target_state->enter(dev, drv, index);
}

static inline int cpuidle_enter_tk(struct cpuidle_device *dev,
			       struct cpuidle_driver *drv, int index)
{
	return cpuidle_wrap_enter(dev, drv, index, cpuidle_enter);
}

typedef int (*cpuidle_enter_t)(struct cpuidle_device *dev,
			       struct cpuidle_driver *drv, int index);

static cpuidle_enter_t cpuidle_enter_ops;

63
64
65
/**
 * cpuidle_play_dead - cpu off-lining
 *
66
 * Returns in case of an error or no driver
67
68
69
70
71
72
73
74
 */
int cpuidle_play_dead(void)
{
	struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
	struct cpuidle_driver *drv = cpuidle_get_driver();
	int i, dead_state = -1;
	int power_usage = -1;

75
76
77
	if (!drv)
		return -ENODEV;

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
	/* Find lowest-power state that supports long-term idle */
	for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) {
		struct cpuidle_state *s = &drv->states[i];

		if (s->power_usage < power_usage && s->enter_dead) {
			power_usage = s->power_usage;
			dead_state = i;
		}
	}

	if (dead_state != -1)
		return drv->states[dead_state].enter_dead(dev, dead_state);

	return -ENODEV;
}

94
95
96
97
/**
 * cpuidle_idle_call - the main idle loop
 *
 * NOTE: no locks or semaphores should be used here
Len Brown's avatar
Len Brown committed
98
 * return non-zero on failure
99
 */
Len Brown's avatar
Len Brown committed
100
int cpuidle_idle_call(void)
101
{
102
	struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
103
	struct cpuidle_driver *drv = cpuidle_get_driver();
104
	int next_state, entered_state;
105

Len Brown's avatar
Len Brown committed
106
107
108
109
110
111
	if (off)
		return -ENODEV;

	if (!initialized)
		return -ENODEV;

112
	/* check if the device is ready */
Len Brown's avatar
Len Brown committed
113
114
	if (!dev || !dev->enabled)
		return -EBUSY;
115

116
117
#if 0
	/* shows regressions, re-enable for 2.6.29 */
118
119
120
121
122
	/*
	 * run any timers that can be run now, at this point
	 * before calculating the idle duration etc.
	 */
	hrtimer_peek_ahead_timers();
123
#endif
124

125
	/* ask the governor for the next state */
126
	next_state = cpuidle_curr_governor->select(drv, dev);
127
128
	if (need_resched()) {
		local_irq_enable();
Len Brown's avatar
Len Brown committed
129
		return 0;
130
131
	}

132
133
	trace_power_start_rcuidle(POWER_CSTATE, next_state, dev->cpu);
	trace_cpu_idle_rcuidle(next_state, dev->cpu);
134

135
	entered_state = cpuidle_enter_ops(dev, drv, next_state);
136

137
138
	trace_power_end_rcuidle(dev->cpu);
	trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
139

140
141
142
143
144
	if (entered_state >= 0) {
		/* Update cpuidle counters */
		/* This can be moved to within driver enter routine
		 * but that results in multiple copies of same code.
		 */
145
		dev->states_usage[entered_state].time +=
146
				(unsigned long long)dev->last_residency;
147
		dev->states_usage[entered_state].usage++;
148
149
	} else {
		dev->last_residency = 0;
150
	}
151
152
153

	/* give the governor an opportunity to reflect on the outcome */
	if (cpuidle_curr_governor->reflect)
154
		cpuidle_curr_governor->reflect(dev, entered_state);
Len Brown's avatar
Len Brown committed
155
156

	return 0;
157
158
159
160
161
162
163
}

/**
 * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
 */
void cpuidle_install_idle_handler(void)
{
Len Brown's avatar
Len Brown committed
164
	if (enabled_devices) {
165
166
		/* Make sure all changes finished before we switch to new idle */
		smp_wmb();
Len Brown's avatar
Len Brown committed
167
		initialized = 1;
168
169
170
171
172
173
174
175
	}
}

/**
 * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
 */
void cpuidle_uninstall_idle_handler(void)
{
Len Brown's avatar
Len Brown committed
176
177
	if (enabled_devices) {
		initialized = 0;
178
		kick_all_cpus_sync();
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
	}
}

/**
 * cpuidle_pause_and_lock - temporarily disables CPUIDLE
 */
void cpuidle_pause_and_lock(void)
{
	mutex_lock(&cpuidle_lock);
	cpuidle_uninstall_idle_handler();
}

EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);

/**
 * cpuidle_resume_and_unlock - resumes CPUIDLE operation
 */
void cpuidle_resume_and_unlock(void)
{
	cpuidle_install_idle_handler();
	mutex_unlock(&cpuidle_lock);
}

EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);

204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
/* Currently used in suspend/resume path to suspend cpuidle */
void cpuidle_pause(void)
{
	mutex_lock(&cpuidle_lock);
	cpuidle_uninstall_idle_handler();
	mutex_unlock(&cpuidle_lock);
}

/* Currently used in suspend/resume path to resume cpuidle */
void cpuidle_resume(void)
{
	mutex_lock(&cpuidle_lock);
	cpuidle_install_idle_handler();
	mutex_unlock(&cpuidle_lock);
}

220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
/**
 * cpuidle_wrap_enter - performs timekeeping and irqen around enter function
 * @dev: pointer to a valid cpuidle_device object
 * @drv: pointer to a valid cpuidle_driver object
 * @index: index of the target cpuidle state.
 */
int cpuidle_wrap_enter(struct cpuidle_device *dev,
				struct cpuidle_driver *drv, int index,
				int (*enter)(struct cpuidle_device *dev,
					struct cpuidle_driver *drv, int index))
{
	ktime_t time_start, time_end;
	s64 diff;

	time_start = ktime_get();

	index = enter(dev, drv, index);

	time_end = ktime_get();

	local_irq_enable();

	diff = ktime_to_us(ktime_sub(time_end, time_start));
	if (diff > INT_MAX)
		diff = INT_MAX;

	dev->last_residency = (int) diff;

	return index;
}

251
#ifdef CONFIG_ARCH_HAS_CPU_RELAX
252
253
static int poll_idle(struct cpuidle_device *dev,
		struct cpuidle_driver *drv, int index)
254
255
256
257
258
259
260
261
262
263
264
265
266
267
{
	ktime_t	t1, t2;
	s64 diff;

	t1 = ktime_get();
	local_irq_enable();
	while (!need_resched())
		cpu_relax();

	t2 = ktime_get();
	diff = ktime_to_us(ktime_sub(t2, t1));
	if (diff > INT_MAX)
		diff = INT_MAX;

268
269
270
	dev->last_residency = (int) diff;

	return index;
271
272
}

273
static void poll_idle_init(struct cpuidle_driver *drv)
274
{
275
	struct cpuidle_state *state = &drv->states[0];
276

277
	snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
278
279
280
281
	snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
	state->exit_latency = 0;
	state->target_residency = 0;
	state->power_usage = -1;
282
	state->flags = 0;
283
284
285
	state->enter = poll_idle;
}
#else
286
static void poll_idle_init(struct cpuidle_driver *drv) {}
287
288
#endif /* CONFIG_ARCH_HAS_CPU_RELAX */

289
290
291
292
293
294
295
296
297
298
/**
 * cpuidle_enable_device - enables idle PM for a CPU
 * @dev: the CPU
 *
 * This function must be called between cpuidle_pause_and_lock and
 * cpuidle_resume_and_unlock when used externally.
 */
int cpuidle_enable_device(struct cpuidle_device *dev)
{
	int ret, i;
299
	struct cpuidle_driver *drv = cpuidle_get_driver();
300
301
302

	if (dev->enabled)
		return 0;
303
	if (!drv || !cpuidle_curr_governor)
304
305
		return -EIO;
	if (!dev->state_count)
306
		dev->state_count = drv->state_count;
307

308
309
310
311
312
313
	if (dev->registered == 0) {
		ret = __cpuidle_register_device(dev);
		if (ret)
			return ret;
	}

314
315
316
317
	cpuidle_enter_ops = drv->en_core_tk_irqen ?
		cpuidle_enter_tk : cpuidle_enter;

	poll_idle_init(drv);
318

319
320
321
322
	if ((ret = cpuidle_add_state_sysfs(dev)))
		return ret;

	if (cpuidle_curr_governor->enable &&
323
	    (ret = cpuidle_curr_governor->enable(drv, dev)))
324
325
326
		goto fail_sysfs;

	for (i = 0; i < dev->state_count; i++) {
327
328
		dev->states_usage[i].usage = 0;
		dev->states_usage[i].time = 0;
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
	}
	dev->last_residency = 0;

	smp_wmb();

	dev->enabled = 1;

	enabled_devices++;
	return 0;

fail_sysfs:
	cpuidle_remove_state_sysfs(dev);

	return ret;
}

EXPORT_SYMBOL_GPL(cpuidle_enable_device);

/**
 * cpuidle_disable_device - disables idle PM for a CPU
 * @dev: the CPU
 *
 * This function must be called between cpuidle_pause_and_lock and
 * cpuidle_resume_and_unlock when used externally.
 */
void cpuidle_disable_device(struct cpuidle_device *dev)
{
	if (!dev->enabled)
		return;
358
	if (!cpuidle_get_driver() || !cpuidle_curr_governor)
359
360
361
362
363
		return;

	dev->enabled = 0;

	if (cpuidle_curr_governor->disable)
364
		cpuidle_curr_governor->disable(cpuidle_get_driver(), dev);
365
366
367
368
369
370
371
372

	cpuidle_remove_state_sysfs(dev);
	enabled_devices--;
}

EXPORT_SYMBOL_GPL(cpuidle_disable_device);

/**
373
374
 * __cpuidle_register_device - internal register function called before register
 * and enable routines
375
 * @dev: the cpu
376
377
 *
 * cpuidle_lock mutex must be held before this is called
378
 */
379
static int __cpuidle_register_device(struct cpuidle_device *dev)
380
381
{
	int ret;
382
	struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
383
	struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
384

385
	if (!dev)
386
		return -EINVAL;
387
	if (!try_module_get(cpuidle_driver->owner))
388
389
390
391
392
393
		return -EINVAL;

	init_completion(&dev->kobj_unregister);

	per_cpu(cpuidle_devices, dev->cpu) = dev;
	list_add(&dev->device_list, &cpuidle_detected_devices);
394
	if ((ret = cpuidle_add_sysfs(cpu_dev))) {
395
		module_put(cpuidle_driver->owner);
396
397
398
		return ret;
	}

399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
	dev->registered = 1;
	return 0;
}

/**
 * cpuidle_register_device - registers a CPU's idle PM feature
 * @dev: the cpu
 */
int cpuidle_register_device(struct cpuidle_device *dev)
{
	int ret;

	mutex_lock(&cpuidle_lock);

	if ((ret = __cpuidle_register_device(dev))) {
		mutex_unlock(&cpuidle_lock);
		return ret;
	}

418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
	cpuidle_enable_device(dev);
	cpuidle_install_idle_handler();

	mutex_unlock(&cpuidle_lock);

	return 0;

}

EXPORT_SYMBOL_GPL(cpuidle_register_device);

/**
 * cpuidle_unregister_device - unregisters a CPU's idle PM feature
 * @dev: the cpu
 */
void cpuidle_unregister_device(struct cpuidle_device *dev)
{
435
	struct device *cpu_dev = get_cpu_device((unsigned long)dev->cpu);
436
	struct cpuidle_driver *cpuidle_driver = cpuidle_get_driver();
437

438
439
440
	if (dev->registered == 0)
		return;

441
442
443
444
	cpuidle_pause_and_lock();

	cpuidle_disable_device(dev);

445
	cpuidle_remove_sysfs(cpu_dev);
446
447
448
449
450
451
	list_del(&dev->device_list);
	wait_for_completion(&dev->kobj_unregister);
	per_cpu(cpuidle_devices, dev->cpu) = NULL;

	cpuidle_resume_and_unlock();

452
	module_put(cpuidle_driver->owner);
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
}

EXPORT_SYMBOL_GPL(cpuidle_unregister_device);

#ifdef CONFIG_SMP

static void smp_callback(void *v)
{
	/* we already woke the CPU up, nothing more to do */
}

/*
 * This function gets called when a part of the kernel has a new latency
 * requirement.  This means we need to get all processors out of their C-state,
 * and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
 * wakes them all right up.
 */
static int cpuidle_latency_notify(struct notifier_block *b,
		unsigned long l, void *v)
{
473
	smp_call_function(smp_callback, NULL, 1);
474
475
476
477
478
479
480
	return NOTIFY_OK;
}

static struct notifier_block cpuidle_latency_notifier = {
	.notifier_call = cpuidle_latency_notify,
};

481
482
483
484
static inline void latency_notifier_init(struct notifier_block *n)
{
	pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
}
485
486
487
488
489
490
491
492
493
494
495
496
497
498

#else /* CONFIG_SMP */

#define latency_notifier_init(x) do { } while (0)

#endif /* CONFIG_SMP */

/**
 * cpuidle_init - core initializer
 */
static int __init cpuidle_init(void)
{
	int ret;

499
500
501
	if (cpuidle_disabled())
		return -ENODEV;

502
	ret = cpuidle_add_interface(cpu_subsys.dev_root);
503
504
505
506
507
508
509
510
	if (ret)
		return ret;

	latency_notifier_init(&cpuidle_latency_notifier);

	return 0;
}

511
module_param(off, int, 0444);
512
core_initcall(cpuidle_init);