2 * This file contains shadow memory manipulation code.
4 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
5 * Author: Andrey Ryabinin <a.ryabinin@samsung.com>
7 * Some of code borrowed from https://github.com/xairy/linux by
8 * Andrey Konovalov <adech.fo@gmail.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
17 #define DISABLE_BRANCH_PROFILING
19 #include <linux/export.h>
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/memblock.h>
24 #include <linux/printk.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
27 #include <linux/stacktrace.h>
28 #include <linux/string.h>
29 #include <linux/types.h>
30 #include <linux/kasan.h>
35 * Poisons the shadow memory for 'size' bytes starting from 'addr'.
36 * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
38 static void kasan_poison_shadow(const void *address, size_t size, u8 value)
40 void *shadow_start, *shadow_end;
42 shadow_start = kasan_mem_to_shadow(address);
43 shadow_end = kasan_mem_to_shadow(address + size);
45 memset(shadow_start, value, shadow_end - shadow_start);
48 void kasan_unpoison_shadow(const void *address, size_t size)
50 kasan_poison_shadow(address, size, 0);
52 if (size & KASAN_SHADOW_MASK) {
53 u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size);
54 *shadow = size & KASAN_SHADOW_MASK;
60 * All functions below always inlined so compiler could
61 * perform better optimizations in each of __asan_loadX/__assn_storeX
62 * depending on memory access size X.
65 static __always_inline bool memory_is_poisoned_1(unsigned long addr)
67 s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr);
69 if (unlikely(shadow_value)) {
70 s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
71 return unlikely(last_accessible_byte >= shadow_value);
77 static __always_inline bool memory_is_poisoned_2(unsigned long addr)
79 u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);
81 if (unlikely(*shadow_addr)) {
82 if (memory_is_poisoned_1(addr + 1))
85 if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
88 return unlikely(*(u8 *)shadow_addr);
94 static __always_inline bool memory_is_poisoned_4(unsigned long addr)
96 u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);
98 if (unlikely(*shadow_addr)) {
99 if (memory_is_poisoned_1(addr + 3))
102 if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
105 return unlikely(*(u8 *)shadow_addr);
111 static __always_inline bool memory_is_poisoned_8(unsigned long addr)
113 u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);
115 if (unlikely(*shadow_addr)) {
116 if (memory_is_poisoned_1(addr + 7))
119 if (likely(((addr + 7) & KASAN_SHADOW_MASK) >= 7))
122 return unlikely(*(u8 *)shadow_addr);
128 static __always_inline bool memory_is_poisoned_16(unsigned long addr)
130 u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr);
132 if (unlikely(*shadow_addr)) {
133 u16 shadow_first_bytes = *(u16 *)shadow_addr;
134 s8 last_byte = (addr + 15) & KASAN_SHADOW_MASK;
136 if (unlikely(shadow_first_bytes))
139 if (likely(!last_byte))
142 return memory_is_poisoned_1(addr + 15);
148 static __always_inline unsigned long bytes_is_zero(const u8 *start,
152 if (unlikely(*start))
153 return (unsigned long)start;
161 static __always_inline unsigned long memory_is_zero(const void *start,
166 unsigned int prefix = (unsigned long)start % 8;
168 if (end - start <= 16)
169 return bytes_is_zero(start, end - start);
173 ret = bytes_is_zero(start, prefix);
179 words = (end - start) / 8;
181 if (unlikely(*(u64 *)start))
182 return bytes_is_zero(start, 8);
187 return bytes_is_zero(start, (end - start) % 8);
190 static __always_inline bool memory_is_poisoned_n(unsigned long addr,
195 ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
196 kasan_mem_to_shadow((void *)addr + size - 1) + 1);
199 unsigned long last_byte = addr + size - 1;
200 s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte);
202 if (unlikely(ret != (unsigned long)last_shadow ||
203 ((last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
209 static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
211 if (__builtin_constant_p(size)) {
214 return memory_is_poisoned_1(addr);
216 return memory_is_poisoned_2(addr);
218 return memory_is_poisoned_4(addr);
220 return memory_is_poisoned_8(addr);
222 return memory_is_poisoned_16(addr);
228 return memory_is_poisoned_n(addr, size);
232 static __always_inline void check_memory_region(unsigned long addr,
233 size_t size, bool write)
235 struct kasan_access_info info;
237 if (unlikely(size == 0))
240 if (unlikely((void *)addr <
241 kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
242 info.access_addr = (void *)addr;
243 info.access_size = size;
244 info.is_write = write;
246 kasan_report_user_access(&info);
250 if (likely(!memory_is_poisoned(addr, size)))
253 kasan_report(addr, size, write, _RET_IP_);
256 #define DEFINE_ASAN_LOAD_STORE(size) \
257 void __asan_load##size(unsigned long addr) \
259 check_memory_region(addr, size, false); \
261 EXPORT_SYMBOL(__asan_load##size); \
262 __alias(__asan_load##size) \
263 void __asan_load##size##_noabort(unsigned long); \
264 EXPORT_SYMBOL(__asan_load##size##_noabort); \
265 void __asan_store##size(unsigned long addr) \
267 check_memory_region(addr, size, true); \
269 EXPORT_SYMBOL(__asan_store##size); \
270 __alias(__asan_store##size) \
271 void __asan_store##size##_noabort(unsigned long); \
272 EXPORT_SYMBOL(__asan_store##size##_noabort)
274 DEFINE_ASAN_LOAD_STORE(1);
275 DEFINE_ASAN_LOAD_STORE(2);
276 DEFINE_ASAN_LOAD_STORE(4);
277 DEFINE_ASAN_LOAD_STORE(8);
278 DEFINE_ASAN_LOAD_STORE(16);
280 void __asan_loadN(unsigned long addr, size_t size)
282 check_memory_region(addr, size, false);
284 EXPORT_SYMBOL(__asan_loadN);
286 __alias(__asan_loadN)
287 void __asan_loadN_noabort(unsigned long, size_t);
288 EXPORT_SYMBOL(__asan_loadN_noabort);
290 void __asan_storeN(unsigned long addr, size_t size)
292 check_memory_region(addr, size, true);
294 EXPORT_SYMBOL(__asan_storeN);
296 __alias(__asan_storeN)
297 void __asan_storeN_noabort(unsigned long, size_t);
298 EXPORT_SYMBOL(__asan_storeN_noabort);
300 /* to shut up compiler complaints */
301 void __asan_handle_no_return(void) {}
302 EXPORT_SYMBOL(__asan_handle_no_return);