diff --git a/Makefile b/Makefile index f70ef17a3d2f..709c6c0b3d1f 100644 --- a/Makefile +++ b/Makefile @@ -821,6 +821,32 @@ DISABLE_LTO := $(DISABLE_LTO_CLANG) export LTO_CFLAGS DISABLE_LTO endif +ifdef CONFIG_CFI_CLANG +cfi-clang-flags += -fsanitize=cfi $(call cc-option, -fsplit-lto-unit) +DISABLE_CFI_CLANG := -fno-sanitize=cfi +ifdef CONFIG_MODULES +cfi-clang-flags += -fsanitize-cfi-cross-dso +DISABLE_CFI_CLANG += -fno-sanitize-cfi-cross-dso +endif +ifdef CONFIG_CFI_PERMISSIVE +cfi-clang-flags += -fsanitize-recover=cfi -fno-sanitize-trap=cfi +endif + +# also disable CFI when LTO is disabled +DISABLE_LTO_CLANG += $(DISABLE_CFI_CLANG) +# allow disabling only clang CFI where needed +export DISABLE_CFI_CLANG +endif + +ifdef CONFIG_CFI +CFI_CFLAGS := $(cfi-clang-flags) +KBUILD_CFLAGS += $(CFI_CFLAGS) + +DISABLE_CFI := $(DISABLE_CFI_CLANG) +DISABLE_LTO += $(DISABLE_CFI) +export CFI_CFLAGS DISABLE_CFI +endif + # arch Makefile may override CC so keep this after arch Makefile is included NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include) diff --git a/arch/Kconfig b/arch/Kconfig index cb8ef5e2b322..30346f3d86be 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -513,6 +513,34 @@ config LTO_CLANG endchoice +config CFI + bool + +config CFI_PERMISSIVE + bool "Use CFI in permissive mode" + depends on CFI + help + When selected, Control Flow Integrity (CFI) violations result in a + warning instead of a kernel panic. This option is useful for finding + CFI violations in drivers during development. + +config CFI_CLANG + bool "Use clang Control Flow Integrity (CFI) (EXPERIMENTAL)" + depends on LTO_CLANG + depends on KALLSYMS + select CFI + help + This option enables clang Control Flow Integrity (CFI), which adds + runtime checking for indirect function calls. + +config CFI_CLANG_SHADOW + bool "Use CFI shadow to speed up cross-module checks" + default y + depends on CFI_CLANG + help + If you select this option, the kernel builds a fast look-up table of + CFI check functions in loaded modules to reduce overhead. + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index f12e43993d66..8667538d0b4b 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -68,6 +68,7 @@ */ #if defined(CONFIG_LD_DEAD_CODE_DATA_ELIMINATION) || defined(CONFIG_LTO_CLANG) #define TEXT_MAIN .text .text.[0-9a-zA-Z_]* +#define TEXT_CFI_MAIN .text.[0-9a-zA-Z_]*.cfi #define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data..LPBX* #define SDATA_MAIN .sdata .sdata.[0-9a-zA-Z_]* #define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* @@ -75,6 +76,7 @@ #define SBSS_MAIN .sbss .sbss.[0-9a-zA-Z_]* #else #define TEXT_MAIN .text +#define TEXT_CFI_MAIN .text.cfi #define DATA_MAIN .data #define SDATA_MAIN .sdata #define RODATA_MAIN .rodata @@ -491,6 +493,7 @@ #define TEXT_TEXT \ ALIGN_FUNCTION(); \ *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \ + *(TEXT_CFI_MAIN) \ *(.text..refcount) \ *(.text..ftrace) \ *(.ref.text) \ diff --git a/include/linux/cfi.h b/include/linux/cfi.h new file mode 100644 index 000000000000..e27033d5dd53 --- /dev/null +++ b/include/linux/cfi.h @@ -0,0 +1,38 @@ +#ifndef _LINUX_CFI_H +#define _LINUX_CFI_H + +#include + +#ifdef CONFIG_CFI_CLANG +#ifdef CONFIG_MODULES + +typedef void (*cfi_check_fn)(uint64_t, void *, void *); + +/* Compiler-generated function in each module, and the kernel */ +#define CFI_CHECK_FN __cfi_check +#define CFI_CHECK_FN_NAME __stringify(CFI_CHECK_FN) + +extern void CFI_CHECK_FN(uint64_t, void *, void *); + +#ifdef CONFIG_CFI_CLANG_SHADOW +extern void cfi_module_add(struct module *mod, unsigned long min_addr, + unsigned long max_addr); + +extern void cfi_module_remove(struct module *mod, unsigned long min_addr, + unsigned long max_addr); +#else +static inline void cfi_module_add(struct module *mod, unsigned long min_addr, + unsigned long max_addr) +{ +} + +static inline void cfi_module_remove(struct module *mod, unsigned long min_addr, + unsigned long max_addr) +{ +} +#endif /* CONFIG_CFI_CLANG_SHADOW */ + +#endif /* CONFIG_MODULES */ +#endif /* CONFIG_CFI_CLANG */ + +#endif /* _LINUX_CFI_H */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 64eb811a11e4..ed0f2ad51ee2 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -49,4 +49,6 @@ #define __norecordmcount \ __attribute__((__section__(".text..ftrace"))) #endif + +#define __nocfi __attribute__((no_sanitize("cfi"))) #endif diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 4f498e3cc760..fcfd54fe235d 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -252,6 +252,10 @@ struct ftrace_likely_data { #define __norecordmcount #endif +#ifndef __nocfi +#define __nocfi +#endif + /* * Force always-inline if the user requests it so via the .config. * GCC does not warn about unused static inline functions for diff --git a/include/linux/init.h b/include/linux/init.h index 892fe8e04c82..847e8536228f 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,7 +47,7 @@ /* These are for everybody (although not all archs will actually discard it in modules) */ -#define __init __section(.init.text) __cold __latent_entropy __noinitretpoline +#define __init __section(.init.text) __cold __latent_entropy __noinitretpoline __nocfi #define __initdata __section(.init.data) #define __initconst __section(.init.rodata) #define __exitdata __section(.exit.data) diff --git a/include/linux/module.h b/include/linux/module.h index 904f94628132..ffc1fbc11273 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -348,6 +349,10 @@ struct module { const s32 *crcs; unsigned int num_syms; +#ifdef CONFIG_CFI_CLANG + cfi_check_fn cfi_check; +#endif + /* Kernel parameters. */ #ifdef CONFIG_SYSFS struct mutex param_lock; diff --git a/init/Kconfig b/init/Kconfig index 0332548d7fa7..8a03d8b466a8 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2043,7 +2043,7 @@ endif # MODULES config MODULES_TREE_LOOKUP def_bool y - depends on PERF_EVENTS || TRACING + depends on PERF_EVENTS || TRACING || CFI_CLANG config INIT_ALL_POSSIBLE bool diff --git a/kernel/Makefile b/kernel/Makefile index 7a63d567fdb5..7d146967402a 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -34,6 +34,9 @@ KASAN_SANITIZE_kcov.o := n # cond_syscall is currently not LTO compatible CFLAGS_sys_ni.o = $(DISABLE_LTO) +# Don't instrument error handlers +CFLAGS_cfi.o = $(DISABLE_CFI_CLANG) + obj-y += sched/ obj-y += locking/ obj-y += power/ @@ -103,6 +106,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_CFI_CLANG) += cfi.o obj-$(CONFIG_PERF_EVENTS) += events/ diff --git a/kernel/cfi.c b/kernel/cfi.c new file mode 100644 index 000000000000..9e9992258b48 --- /dev/null +++ b/kernel/cfi.c @@ -0,0 +1,306 @@ +/* + * CFI (Control Flow Integrity) error and slowpath handling + * + * Copyright (C) 2017 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Compiler-defined handler names */ +#ifdef CONFIG_CFI_PERMISSIVE +#define cfi_failure_handler __ubsan_handle_cfi_check_fail +#define cfi_slowpath_handler __cfi_slowpath_diag +#else /* enforcing */ +#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort +#define cfi_slowpath_handler __cfi_slowpath +#endif /* CONFIG_CFI_PERMISSIVE */ + +static inline void handle_cfi_failure(void *ptr) +{ +#ifdef CONFIG_CFI_PERMISSIVE + WARN_RATELIMIT(1, "CFI failure (target: %pF):\n", ptr); +#else + pr_err("CFI failure (target: %pF):\n", ptr); + BUG(); +#endif +} + +#ifdef CONFIG_MODULES +#ifdef CONFIG_CFI_CLANG_SHADOW +struct shadow_range { + /* Module address range */ + unsigned long mod_min_addr; + unsigned long mod_max_addr; + /* Module page range */ + unsigned long min_page; + unsigned long max_page; +}; + +#define SHADOW_ORDER 1 +#define SHADOW_PAGES (1 << SHADOW_ORDER) +#define SHADOW_SIZE \ + ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16)) +#define SHADOW_INVALID 0xFFFF + +struct cfi_shadow { + /* Page range covered by the shadow */ + struct shadow_range r; + /* Page offsets to __cfi_check functions in modules */ + u16 shadow[SHADOW_SIZE]; +}; + +static DEFINE_SPINLOCK(shadow_update_lock); +static struct cfi_shadow __rcu *cfi_shadow __read_mostly = NULL; + +static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr) +{ + unsigned long index; + unsigned long page = ptr >> PAGE_SHIFT; + + if (unlikely(page < s->r.min_page)) + return -1; /* Outside of module area */ + + index = page - s->r.min_page; + + if (index >= SHADOW_SIZE) + return -1; /* Cannot be addressed with shadow */ + + return (int)index; +} + +static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s, + int index) +{ + BUG_ON(index < 0 || index >= SHADOW_SIZE); + + if (unlikely(s->shadow[index] == SHADOW_INVALID)) + return 0; + + return (s->r.min_page + s->shadow[index]) << PAGE_SHIFT; +} + +static inline unsigned long shadow_to_page(const struct cfi_shadow *s, + int index) +{ + BUG_ON(index < 0 || index >= SHADOW_SIZE); + + return (s->r.min_page + index) << PAGE_SHIFT; +} + +static void prepare_next_shadow(const struct cfi_shadow __rcu *prev, + struct cfi_shadow *next) +{ + int i, index, check; + + /* Mark everything invalid */ + memset(next->shadow, 0xFF, sizeof(next->shadow)); + + if (!prev) + return; /* No previous shadow */ + + /* If the base address didn't change, update is not needed */ + if (prev->r.min_page == next->r.min_page) { + memcpy(next->shadow, prev->shadow, sizeof(next->shadow)); + return; + } + + /* Convert the previous shadow to the new address range */ + for (i = 0; i < SHADOW_SIZE; ++i) { + if (prev->shadow[i] == SHADOW_INVALID) + continue; + + index = ptr_to_shadow(next, shadow_to_page(prev, i)); + if (index < 0) + continue; + + check = ptr_to_shadow(next, + shadow_to_ptr(prev, prev->shadow[i])); + if (check < 0) + continue; + + next->shadow[index] = (u16)check; + } +} + +static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod) +{ + unsigned long ptr; + unsigned long min_page_addr; + unsigned long max_page_addr; + unsigned long check = (unsigned long)mod->cfi_check; + int check_index = ptr_to_shadow(s, check); + + BUG_ON((check & PAGE_MASK) != check); /* Must be page aligned */ + + if (check_index < 0) + return; /* Module not addressable with shadow */ + + min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK; + max_page_addr = (unsigned long)mod->core_layout.base + + mod->core_layout.text_size; + max_page_addr &= PAGE_MASK; + + /* For each page, store the check function index in the shadow */ + for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) { + int index = ptr_to_shadow(s, ptr); + if (index >= 0) { + /* Assume a page only contains code for one module */ + BUG_ON(s->shadow[index] != SHADOW_INVALID); + s->shadow[index] = (u16)check_index; + } + } +} + +static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod) +{ + unsigned long ptr; + unsigned long min_page_addr; + unsigned long max_page_addr; + + min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK; + max_page_addr = (unsigned long)mod->core_layout.base + + mod->core_layout.text_size; + max_page_addr &= PAGE_MASK; + + for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) { + int index = ptr_to_shadow(s, ptr); + if (index >= 0) + s->shadow[index] = SHADOW_INVALID; + } +} + +typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *); + +static void update_shadow(struct module *mod, unsigned long min_addr, + unsigned long max_addr, update_shadow_fn fn) +{ + struct cfi_shadow *prev; + struct cfi_shadow *next = (struct cfi_shadow *) + __get_free_pages(GFP_KERNEL, SHADOW_ORDER); + + BUG_ON(!next); + + next->r.mod_min_addr = min_addr; + next->r.mod_max_addr = max_addr; + next->r.min_page = min_addr >> PAGE_SHIFT; + next->r.max_page = max_addr >> PAGE_SHIFT; + + spin_lock(&shadow_update_lock); + prev = rcu_dereference_protected(cfi_shadow, 1); + prepare_next_shadow(prev, next); + + fn(next, mod); + set_memory_ro((unsigned long)next, SHADOW_PAGES); + rcu_assign_pointer(cfi_shadow, next); + + spin_unlock(&shadow_update_lock); + synchronize_rcu(); + + if (prev) { + set_memory_rw((unsigned long)prev, SHADOW_PAGES); + free_pages((unsigned long)prev, SHADOW_ORDER); + } +} + +void cfi_module_add(struct module *mod, unsigned long min_addr, + unsigned long max_addr) +{ + update_shadow(mod, min_addr, max_addr, add_module_to_shadow); +} +EXPORT_SYMBOL(cfi_module_add); + +void cfi_module_remove(struct module *mod, unsigned long min_addr, + unsigned long max_addr) +{ + update_shadow(mod, min_addr, max_addr, remove_module_from_shadow); +} +EXPORT_SYMBOL(cfi_module_remove); + +static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s, + unsigned long ptr) +{ + int index; + + if (unlikely(!s)) + return NULL; /* No shadow available */ + + if (ptr < s->r.mod_min_addr || ptr > s->r.mod_max_addr) + return NULL; /* Not in a mapped module */ + + index = ptr_to_shadow(s, ptr); + if (index < 0) + return NULL; /* Cannot be addressed with shadow */ + + return (cfi_check_fn)shadow_to_ptr(s, index); +} +#endif /* CONFIG_CFI_CLANG_SHADOW */ + +static inline cfi_check_fn find_module_cfi_check(void *ptr) +{ + struct module *mod; + + preempt_disable(); + mod = __module_address((unsigned long)ptr); + preempt_enable(); + + if (mod) + return mod->cfi_check; + + return CFI_CHECK_FN; +} + +static inline cfi_check_fn find_cfi_check(void *ptr) +{ +#ifdef CONFIG_CFI_CLANG_SHADOW + cfi_check_fn f; + + if (!rcu_access_pointer(cfi_shadow)) + return CFI_CHECK_FN; /* No loaded modules */ + + /* Look up the __cfi_check function to use */ + rcu_read_lock(); + f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr); + rcu_read_unlock(); + + if (f) + return f; + + /* + * Fall back to find_module_cfi_check, which works also for a larger + * module address space, but is slower. + */ +#endif /* CONFIG_CFI_CLANG_SHADOW */ + + return find_module_cfi_check(ptr); +} + +void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag) +{ + cfi_check_fn check = find_cfi_check(ptr); + + if (likely(check)) + check(id, ptr, diag); + else /* Don't allow unchecked modules */ + handle_cfi_failure(ptr); +} +EXPORT_SYMBOL(cfi_slowpath_handler); +#endif /* CONFIG_MODULES */ + +void cfi_failure_handler(void *data, void *ptr, void *vtable) +{ + handle_cfi_failure(ptr); +} +EXPORT_SYMBOL(cfi_failure_handler); + +void __cfi_check_fail(void *data, void *ptr) +{ + handle_cfi_failure(ptr); +} diff --git a/kernel/module.c b/kernel/module.c index 38bf28b5cc20..e1793038823b 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2125,6 +2125,8 @@ void __weak module_arch_freeing_init(struct module *mod) { } +static void cfi_cleanup(struct module *mod); + /* Free a module, remove from lists, etc. */ static void free_module(struct module *mod) { @@ -2166,6 +2168,10 @@ static void free_module(struct module *mod) /* This may be empty, but that's OK */ disable_ro_nx(&mod->init_layout); + + /* Clean up CFI for the module. */ + cfi_cleanup(mod); + module_arch_freeing_init(mod); module_memfree(mod->init_layout.base); kfree(mod->args); @@ -3353,6 +3359,8 @@ int __weak module_finalize(const Elf_Ehdr *hdr, return 0; } +static void cfi_init(struct module *mod); + static int post_relocation(struct module *mod, const struct load_info *info) { /* Sort exception table now relocations are done. */ @@ -3365,6 +3373,9 @@ static int post_relocation(struct module *mod, const struct load_info *info) /* Setup kallsyms-specific fields. */ add_kallsyms(mod, info); + /* Setup CFI for the module. */ + cfi_init(mod); + /* Arch-specific module finalizing. */ return module_finalize(info->hdr, info->sechdrs, mod); } @@ -4134,6 +4145,22 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, } #endif /* CONFIG_KALLSYMS */ +static void cfi_init(struct module *mod) +{ +#ifdef CONFIG_CFI_CLANG + mod->cfi_check = + (cfi_check_fn)mod_find_symname(mod, CFI_CHECK_FN_NAME); + cfi_module_add(mod, module_addr_min, module_addr_max); +#endif +} + +static void cfi_cleanup(struct module *mod) +{ +#ifdef CONFIG_CFI_CLANG + cfi_module_remove(mod, module_addr_min, module_addr_max); +#endif +} + /* Maximum number of characters written by module_flags() */ #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) diff --git a/net/Kconfig b/net/Kconfig index 228dfa382eec..8cc61e392256 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -287,6 +287,7 @@ config BPF_JIT bool "enable BPF Just In Time compiler" depends on HAVE_CBPF_JIT || HAVE_EBPF_JIT depends on MODULES + depends on !CFI ---help--- Berkeley Packet Filter filtering capabilities are normally handled by an interpreter. This option allows kernel to generate a native