Commit b32fc0a0 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'upstream/jump-label-noearly' of...

Merge branch 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen

* 'upstream/jump-label-noearly' of git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen:
  jump-label: initialize jump-label subsystem much earlier
  x86/jump_label: add arch_jump_label_transform_static()
  s390/jump-label: add arch_jump_label_transform_static()
  jump_label: add arch_jump_label_transform_static() to optimise non-live code updates
  sparc/jump_label: drop arch_jump_label_text_poke_early()
  x86/jump_label: drop arch_jump_label_text_poke_early()
  jump_label: if a key has already been initialized, don't nop it out
  stop_machine: make stop_machine safe and efficient to call early
  jump_label: use proper atomic_t initializer

Conflicts:
 - arch/x86/kernel/jump_label.c
	Added __init_or_module to arch_jump_label_text_poke_early vs
	removal of that function entirely
 - kernel/stop_machine.c
	same patch ("stop_machine: make stop_machine safe and efficient
	to call early") merged twice, with whitespace fix in one version
parents 403299a8 97ce2c88
......@@ -18,26 +18,15 @@ struct insn {
} __packed;
struct insn_args {
unsigned long *target;
struct insn *insn;
ssize_t size;
struct jump_entry *entry;
enum jump_label_type type;
};
static int __arch_jump_label_transform(void *data)
static void __jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
struct insn_args *args = data;
int rc;
rc = probe_kernel_write(args->target, args->insn, args->size);
WARN_ON_ONCE(rc < 0);
return 0;
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
struct insn_args args;
struct insn insn;
int rc;
if (type == JUMP_LABEL_ENABLE) {
/* brcl 15,offset */
......@@ -49,11 +38,33 @@ void arch_jump_label_transform(struct jump_entry *entry,
insn.offset = 0;
}
args.target = (void *) entry->code;
args.insn = &insn;
args.size = JUMP_LABEL_NOP_SIZE;
rc = probe_kernel_write((void *)entry->code, &insn, JUMP_LABEL_NOP_SIZE);
WARN_ON_ONCE(rc < 0);
}
stop_machine(__arch_jump_label_transform, &args, NULL);
static int __sm_arch_jump_label_transform(void *data)
{
struct insn_args *args = data;
__jump_label_transform(args->entry, args->type);
return 0;
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
struct insn_args args;
args.entry = entry;
args.type = type;
stop_machine(__sm_arch_jump_label_transform, &args, NULL);
}
void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
__jump_label_transform(entry, type);
}
#endif
......@@ -36,12 +36,4 @@ void arch_jump_label_transform(struct jump_entry *entry,
put_online_cpus();
}
void arch_jump_label_text_poke_early(jump_label_t addr)
{
u32 *insn_p = (u32 *) (unsigned long) addr;
*insn_p = 0x01000000;
flushi(insn_p);
}
#endif
......@@ -24,8 +24,9 @@ union jump_code_union {
} __attribute__((packed));
};
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
static void __jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
void *(*poker)(void *, const void *, size_t))
{
union jump_code_union code;
......@@ -35,17 +36,24 @@ void arch_jump_label_transform(struct jump_entry *entry,
(entry->code + JUMP_LABEL_NOP_SIZE);
} else
memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
(*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
get_online_cpus();
mutex_lock(&text_mutex);
text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
__jump_label_transform(entry, type, text_poke_smp);
mutex_unlock(&text_mutex);
put_online_cpus();
}
void __init_or_module arch_jump_label_text_poke_early(jump_label_t addr)
void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
JUMP_LABEL_NOP_SIZE);
__jump_label_transform(entry, type, text_poke_early);
}
#endif
......@@ -16,7 +16,7 @@ struct jump_label_key {
# include <asm/jump_label.h>
# define HAVE_JUMP_LABEL
#endif
#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
enum jump_label_type {
JUMP_LABEL_DISABLE = 0,
......@@ -28,9 +28,9 @@ struct module;
#ifdef HAVE_JUMP_LABEL
#ifdef CONFIG_MODULES
#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
#else
#define JUMP_LABEL_INIT {{ 0 }, NULL}
#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
#endif
static __always_inline bool static_branch(struct jump_label_key *key)
......@@ -41,18 +41,20 @@ static __always_inline bool static_branch(struct jump_label_key *key)
extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];
extern void jump_label_init(void);
extern void jump_label_lock(void);
extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
extern void arch_jump_label_text_poke_early(jump_label_t addr);
enum jump_label_type type);
extern void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
extern void jump_label_inc(struct jump_label_key *key);
extern void jump_label_dec(struct jump_label_key *key);
extern bool jump_label_enabled(struct jump_label_key *key);
extern void jump_label_apply_nops(struct module *mod);
#else
#else /* !HAVE_JUMP_LABEL */
#include <linux/atomic.h>
......@@ -62,6 +64,10 @@ struct jump_label_key {
atomic_t enabled;
};
static __always_inline void jump_label_init(void)
{
}
static __always_inline bool static_branch(struct jump_label_key *key)
{
if (unlikely(atomic_read(&key->enabled)))
......@@ -96,7 +102,6 @@ static inline int jump_label_apply_nops(struct module *mod)
{
return 0;
}
#endif /* HAVE_JUMP_LABEL */
#endif
#endif
#endif /* _LINUX_JUMP_LABEL_H */
......@@ -512,6 +512,9 @@ asmlinkage void __init start_kernel(void)
parse_args("Booting kernel", static_command_line, __start___param,
__stop___param - __start___param,
&unknown_bootoption);
jump_label_init();
/*
* These use large bootmem allocations and must precede
* kmem_cache_init()
......
......@@ -104,6 +104,18 @@ static int __jump_label_text_reserved(struct jump_entry *iter_start,
return 0;
}
/*
* Update code which is definitely not currently executing.
* Architectures which need heavyweight synchronization to modify
* running code can override this to make the non-live update case
* cheaper.
*/
void __weak arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
arch_jump_label_transform(entry, type);
}
static void __jump_label_update(struct jump_label_key *key,
struct jump_entry *entry,
struct jump_entry *stop, int enable)
......@@ -121,14 +133,7 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
/*
* Not all archs need this.
*/
void __weak arch_jump_label_text_poke_early(jump_label_t addr)
{
}
static __init int jump_label_init(void)
void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
......@@ -139,22 +144,22 @@ static __init int jump_label_init(void)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
arch_jump_label_text_poke_early(iter->code);
if (iter->key == (jump_label_t)(unsigned long)key)
struct jump_label_key *iterk;
iterk = (struct jump_label_key *)(unsigned long)iter->key;
arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ?
JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
if (iterk == key)
continue;
key = (struct jump_label_key *)(unsigned long)iter->key;
atomic_set(&key->enabled, 0);
key = iterk;
key->entries = iter;
#ifdef CONFIG_MODULES
key->next = NULL;
#endif
}
jump_label_unlock();
return 0;
}
early_initcall(jump_label_init);
#ifdef CONFIG_MODULES
......@@ -212,7 +217,7 @@ void jump_label_apply_nops(struct module *mod)
return;
for (iter = iter_start; iter < iter_stop; iter++)
arch_jump_label_text_poke_early(iter->code);
arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE);
}
static int jump_label_add_module(struct module *mod)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment