GitHub-Mirror/Star64_linux
2
0
Fork 0
mirror of https://github.com/Fishwaldo/Star64_linux.git synced 2025-03-16 20:24:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

kasan: speed up mte_set_mem_tag_range

Browse source 
Use DC GVA / DC GZVA to speed up KASan memory tagging in HW tags mode.

The first cacheline is always tagged using STG/STZG even if the address is
cacheline-aligned, as benchmarks show it is faster than a conditional
branch.

Signed-off-by: Evgenii Stepanov <eugenis@google.com>
Co-developed-by: Peter Collingbourne <pcc@google.com>
Signed-off-by: Peter Collingbourne <pcc@google.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20210521010023.3244784-1-eugenis@google.com
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Evgenii Stepanov 2021-05-20 18:00:23 -07:00 • committed by Will Deacon
parent c4681547bc
commit 3d0cca0b02
1 changed files with 67 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
arch/arm64/include/asm/mte-kasan.h
View file

@ -48,43 +48,84 @@ static inline u8 mte_get_random_tag(void)
return mte_get_ptr_tag(addr); return mte_get_ptr_tag(addr);
} }
static inline u64 __stg_post(u64 p)
{
asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16"
: "+r"(p)
:
: "memory");
return p;
}
static inline u64 __stzg_post(u64 p)
{
asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16"
: "+r"(p)
:
: "memory");
return p;
}
static inline void __dc_gva(u64 p)
{
asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory");
}
static inline void __dc_gzva(u64 p)
{
asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory");
}
/* /*
* Assign allocation tags for a region of memory based on the pointer tag. * Assign allocation tags for a region of memory based on the pointer tag.
* Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
* size must be non-zero and MTE_GRANULE_SIZE aligned. * size must be MTE_GRANULE_SIZE aligned.
*/ */
static inline void mte_set_mem_tag_range(void *addr, size_t size, static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
u8 tag, bool init) bool init)
{ {
u64 curr, end; u64 curr, mask, dczid_bs, end1, end2, end3;
if (!size) /* Read DC G(Z)VA block size from the system register. */
return; dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf);
curr = (u64)__tag_set(addr, tag); curr = (u64)__tag_set(addr, tag);
end = curr + size; mask = dczid_bs - 1;
/* STG/STZG up to the end of the first block. */
end1 = curr | mask;
end3 = curr + size;
/* DC GVA / GZVA in [end1, end2) */
end2 = end3 & ~mask;
/* /*
* 'asm volatile' is required to prevent the compiler to move * The following code uses STG on the first DC GVA block even if the
* the statement outside of the loop. * start address is aligned - it appears to be faster than an alignment
* check + conditional branch. Also, if the range size is at least 2 DC
* GVA blocks, the first two loops can use post-condition to save one
* branch each.
*/ */
if (init) { #define SET_MEMTAG_RANGE(stg_post, dc_gva) \
do { do { \
asm volatile(__MTE_PREAMBLE "stzg %0, [%0]" if (size >= 2 * dczid_bs) { \
: do { \
: "r" (curr) curr = stg_post(curr); \
: "memory"); } while (curr < end1); \
curr += MTE_GRANULE_SIZE; \
} while (curr != end); do { \
} else { dc_gva(curr); \
do { curr += dczid_bs; \
asm volatile(__MTE_PREAMBLE "stg %0, [%0]" } while (curr < end2); \
: } \
: "r" (curr) \
: "memory"); while (curr < end3) \
curr += MTE_GRANULE_SIZE; curr = stg_post(curr); \
} while (curr != end); } while (0)
}
if (init)
SET_MEMTAG_RANGE(__stzg_post, __dc_gzva);
else
SET_MEMTAG_RANGE(__stg_post, __dc_gva);
#undef SET_MEMTAG_RANGE
} }
void mte_enable_kernel_sync(void); void mte_enable_kernel_sync(void);