Skip to content

Commit

Permalink
Detect more VMA types
Browse files Browse the repository at this point in the history
Golang heaps can be determined by a pattern in the address, dictated by address hints supplied to mmap while allocating memory for them.
Thread stacks can be identified by tracking the stack VMA for all newly created threads.
  • Loading branch information
oshaked1 committed Jan 15, 2025
1 parent efe246f commit 0955d4d
Show file tree
Hide file tree
Showing 8 changed files with 223 additions and 31 deletions.
88 changes: 76 additions & 12 deletions pkg/ebpf/c/common/memory.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@

enum vma_type
{
VMA_FILE_BACKED,
VMA_STACK,
VMA_HEAP,
VMA_GOLANG_HEAP,
VMA_THREAD_STACK,
VMA_VDSO,
VMA_ANON,
VMA_OTHER
VMA_UNKNOWN,
};

// PROTOTYPES
Expand All @@ -22,11 +26,14 @@ statfunc unsigned long get_env_start_from_mm(struct mm_struct *);
statfunc unsigned long get_env_end_from_mm(struct mm_struct *);
statfunc unsigned long get_vma_flags(struct vm_area_struct *);
statfunc struct vm_area_struct *find_vma(void *ctx, struct task_struct *task, u64 addr);
statfunc bool vma_is_stack(struct vm_area_struct *vma);
statfunc bool vma_is_heap(struct vm_area_struct *vma);
statfunc bool vma_is_file_backed(struct vm_area_struct *vma);
statfunc bool vma_is_initial_stack(struct vm_area_struct *vma);
statfunc bool vma_is_initial_heap(struct vm_area_struct *vma);
statfunc bool vma_is_anon(struct vm_area_struct *vma);
statfunc bool vma_is_golang_heap(struct vm_area_struct *vma);
statfunc bool vma_is_thread_stack(task_info_t *task_info, struct vm_area_struct *vma);
statfunc bool vma_is_vdso(struct vm_area_struct *vma);
statfunc enum vma_type get_vma_type(struct vm_area_struct *vma);
statfunc enum vma_type get_vma_type(task_info_t *task_info, struct vm_area_struct *vma);

// FUNCTIONS

Expand Down Expand Up @@ -121,7 +128,12 @@ statfunc struct vm_area_struct *find_vma(void *ctx, struct task_struct *task, u6
return vma;
}

statfunc bool vma_is_stack(struct vm_area_struct *vma)
statfunc bool vma_is_file_backed(struct vm_area_struct *vma)
{
return BPF_CORE_READ(vma, vm_file) != NULL;
}

statfunc bool vma_is_initial_stack(struct vm_area_struct *vma)
{
struct mm_struct *vm_mm = BPF_CORE_READ(vma, vm_mm);
if (vm_mm == NULL)
Expand All @@ -138,7 +150,7 @@ statfunc bool vma_is_stack(struct vm_area_struct *vma)
return false;
}

statfunc bool vma_is_heap(struct vm_area_struct *vma)
statfunc bool vma_is_initial_heap(struct vm_area_struct *vma)
{
struct mm_struct *vm_mm = BPF_CORE_READ(vma, vm_mm);
if (vm_mm == NULL)
Expand All @@ -158,7 +170,45 @@ statfunc bool vma_is_heap(struct vm_area_struct *vma)

statfunc bool vma_is_anon(struct vm_area_struct *vma)
{
return BPF_CORE_READ(vma, vm_file) == NULL;
return !vma_is_file_backed(vma);
}

// The golang heap consists of arenas which are memory regions mapped using mmap.
// When allocating areans, golang supplies mmap with an address hint, which is an
// address that the kernel should place the mapping at.
// Hints are constant and vary between architectures, see `mallocinit()` in
// https://github.com/golang/go/blob/master/src/runtime/malloc.go
// From observation, when allocating arenas the MAP_FIXED flag is used which forces
// the kernel to use the specified address or fail the mapping, so it is safe to
// rely on the address pattern to determine if it belongs to a heap arena.
#define GOLANG_ARENA_HINT_MASK 0x80ff00000000UL
#if defined(bpf_target_x86)
#define GOLANG_ARENA_HINT (0xc0UL << 32)
#elif defined(bpf_target_arm64)
#define GOLANG_ARENA_HINT (0x40UL << 32)
#else
#error Unsupported architecture
#endif

statfunc bool vma_is_golang_heap(struct vm_area_struct *vma)
{
u64 vm_start = BPF_CORE_READ(vma, vm_start);

return (vm_start & GOLANG_ARENA_HINT_MASK) == GOLANG_ARENA_HINT;
}

statfunc bool vma_is_thread_stack(task_info_t *task_info, struct vm_area_struct *vma)
{
// Get the stack area for this task
address_range_t *stack = &task_info->stack;
if (stack->start == 0 && stack->end == 0)
// This thread's stack isn't tracked
return false;

// Check if the VMA is **contained** in the thread stack range.
// We don't check exact address range match because a change to the permissions
// of part of the stack VMA will split it into multiple VMAs.
return BPF_CORE_READ(vma, vm_start) >= stack->start && BPF_CORE_READ(vma, vm_end) <= stack->end;
}

statfunc bool vma_is_vdso(struct vm_area_struct *vma)
Expand All @@ -174,19 +224,33 @@ statfunc bool vma_is_vdso(struct vm_area_struct *vma)
return strncmp("[vdso]", mapping_name, 7) == 0;
}

statfunc enum vma_type get_vma_type(struct vm_area_struct *vma)
statfunc enum vma_type get_vma_type(task_info_t *task_info, struct vm_area_struct *vma)
{
if (vma_is_stack(vma))
// The check order is a balance between how expensive the check is and how likely it is to pass

if (vma_is_file_backed(vma))
return VMA_FILE_BACKED;

if (vma_is_initial_stack(vma))
return VMA_STACK;

if (vma_is_heap(vma))
if (vma_is_initial_heap(vma))
return VMA_HEAP;

if (vma_is_anon(vma) && !vma_is_vdso(vma)) {
if (vma_is_anon(vma)) {
if (vma_is_golang_heap(vma))
return VMA_GOLANG_HEAP;

if (vma_is_thread_stack(task_info, vma))
return VMA_THREAD_STACK;

if (vma_is_vdso(vma))
return VMA_VDSO;

return VMA_ANON;
}

return VMA_OTHER;
return VMA_UNKNOWN;
}

#endif
62 changes: 55 additions & 7 deletions pkg/ebpf/c/tracee.bpf.c
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,35 @@ int sys_dup_exit_tail(void *ctx)
return 0;
}

statfunc void update_thread_stack(void *ctx, task_info_t *task_info, struct task_struct *task)
{
// Kernel threads and group leaders are not relevant, reset their stack area
if (get_task_flags(task) & PF_KTHREAD || BPF_CORE_READ(task, pid) == BPF_CORE_READ(task, tgid))
task_info->stack = (address_range_t){0};

// Get user SP of new thread
#if defined(bpf_target_x86)
struct fork_frame *fork_frame = (struct fork_frame *) BPF_CORE_READ(task, thread.sp);
u64 thread_sp = BPF_CORE_READ(fork_frame, regs.sp);
#elif defined(bpf_target_arm64)
struct pt_regs *thread_regs = (struct pt_regs *) BPF_CORE_READ(task, thread.cpu_context.sp);
u64 thread_sp = BPF_CORE_READ(thread_regs, sp);
#else
#error Unsupported architecture
#endif

// Find VMA which contains the SP
struct vm_area_struct *vma = find_vma(ctx, task, thread_sp);
if (unlikely(vma == NULL))
return;

// Add the VMA address range to the task info
task_info->stack = (address_range_t){
.start = BPF_CORE_READ(vma, vm_start),
.end = BPF_CORE_READ(vma, vm_end)
};
}

// trace/events/sched.h: TP_PROTO(struct task_struct *parent, struct task_struct *child)
SEC("raw_tracepoint/sched_process_fork")
int tracepoint__sched__sched_process_fork(struct bpf_raw_tracepoint_args *ctx)
Expand Down Expand Up @@ -608,6 +637,10 @@ int tracepoint__sched__sched_process_fork(struct bpf_raw_tracepoint_args *ctx)
task->context.host_tid = child_tid;
task->context.start_time = child_start_time;

// Track thread stack if needed
if (event_is_selected(SUSPICIOUS_SYSCALL_SOURCE, p.event->context.policies_version))
update_thread_stack(ctx, task, child);

// Update the proc_info_map with the new process's info (from parent)

proc_info_t *c_proc_info = bpf_map_lookup_elem(&proc_info_map, &child_pid);
Expand Down Expand Up @@ -1344,6 +1377,9 @@ int tracepoint__sched__sched_process_exec(struct bpf_raw_tracepoint_args *ctx)
program_data_t p = {};
if (!init_program_data(&p, ctx, SCHED_PROCESS_EXEC))
return 0;

// Reset thread stack area
p.task_info->stack = (address_range_t){0};

// Perform checks below before evaluate_scope_filters(), so tracee can filter by newly created containers
// or processes. Assume that a new container, or pod, has started when a process of a newly
Expand Down Expand Up @@ -5271,11 +5307,13 @@ statfunc void check_suspicious_syscall_source(void *ctx, struct pt_regs *regs, u
if (unlikely(vma == NULL))
return;

// Get VMA type and make sure it's abnormal (stack/heap/anonymous VMA)
enum vma_type vma_type = get_vma_type(vma);
if (vma_type == VMA_OTHER)
// If the VMA is file-backed, the syscall is determined to be legitimate
if (vma_is_file_backed(vma))
return;

// Get VMA type
enum vma_type vma_type = get_vma_type(p.task_info, vma);

// Build a key that identifies the combination of syscall,
// source VMA and process so we don't submit it multiple times
syscall_source_key_t key = {.syscall = syscall,
Expand All @@ -5293,17 +5331,27 @@ statfunc void check_suspicious_syscall_source(void *ctx, struct pt_regs *regs, u

switch (vma_type) {
case VMA_STACK:
vma_type_str = "stack";
vma_type_str = "main stack";
break;
case VMA_THREAD_STACK:
vma_type_str = "thread stack";
break;
case VMA_HEAP:
vma_type_str = "heap";
break;
case VMA_GOLANG_HEAP:
// Goroutine stacks are allocated on the golang heap
vma_type_str = "golang heap/stack";
break;
case VMA_ANON:
vma_type_str = "anonymous";
break;
// shouldn't happen
case VMA_VDSO:
vma_type_str = "vdso";
break;
default:
return;
vma_type_str = "unknown";
break;
}

unsigned long vma_start = BPF_CORE_READ(vma, vm_start);
Expand All @@ -5327,7 +5375,7 @@ int BPF_KPROBE(syscall_checker)
struct pt_regs *regs = ctx;
if (get_kconfig(ARCH_HAS_SYSCALL_WRAPPER))
regs = (struct pt_regs *) PT_REGS_PARM1(ctx);

// Get syscall ID
u32 syscall = get_syscall_id_from_regs(regs);

Expand Down
10 changes: 8 additions & 2 deletions pkg/ebpf/c/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -223,11 +223,17 @@ enum container_state_e
CONTAINER_STARTED // a process in the cgroup executed a new binary
};

typedef struct {
u64 start;
u64 end;
} address_range_t;

typedef struct task_info {
task_context_t context;
syscall_data_t syscall_data;
bool syscall_traced; // indicates that syscall_data is valid
u8 container_state; // the state of the container the task resides in
bool syscall_traced; // indicates that syscall_data is valid
u8 container_state; // the state of the container the task resides in
address_range_t stack; // stack area, only relevant for tasks that aren't group leaders (threads)
} task_info_t;

typedef struct file_id {
Expand Down
23 changes: 23 additions & 0 deletions pkg/ebpf/c/vmlinux.h
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,28 @@ typedef struct {
uid_t val;
} kuid_t;

#if defined(__TARGET_ARCH_x86)

struct thread_struct {
unsigned long sp;
};

struct fork_frame {
struct pt_regs regs;
};

#elif defined(__TARGET_ARCH_arm64)

struct cpu_context {
unsigned long sp;
};

struct thread_struct {
struct cpu_context cpu_context;
};

#endif

struct task_struct {
struct thread_info thread_info;
unsigned int flags;
Expand All @@ -278,6 +300,7 @@ struct task_struct {
struct signal_struct *signal;
void *stack;
struct sighand_struct *sighand;
struct thread_struct thread;
};

typedef struct {
Expand Down
8 changes: 7 additions & 1 deletion pkg/events/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -13066,7 +13066,13 @@ var CoreEvents = map[ID]Definition{
id: SuspiciousSyscallSource,
id32Bit: Sys32Undefined,
name: "suspicious_syscall_source",
sets: []string{},
dependencies: Dependencies{
probes: []Probe{
{handle: probes.SchedProcessFork, required: false}, // for thread stack tracking
{handle: probes.SchedProcessExec, required: false}, // for thread stack tracking
},
},
sets: []string{},
fields: []trace.ArgMeta{
{Type: "int", Name: "syscall"},
{Type: "void*", Name: "ip"},
Expand Down
17 changes: 10 additions & 7 deletions tests/e2e-inst-signatures/e2e-suspicious_syscall_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@ import (
)

type e2eSuspiciousSyscallSource struct {
cb detect.SignatureHandler
foundStack bool
foundHeap bool
foundAnonVma bool
cb detect.SignatureHandler
foundMainStack bool
foundHeap bool
foundAnonVma bool
foundThreadStack bool
}

func (sig *e2eSuspiciousSyscallSource) Init(ctx detect.SignatureContext) error {
Expand Down Expand Up @@ -62,17 +63,19 @@ func (sig *e2eSuspiciousSyscallSource) OnEvent(event protocol.Event) error {
return nil
}

if vmaType == "stack" {
sig.foundStack = true
if vmaType == "main stack" {
sig.foundMainStack = true
} else if vmaType == "heap" {
sig.foundHeap = true
} else if vmaType == "anonymous" {
sig.foundAnonVma = true
} else if vmaType == "thread stack" {
sig.foundThreadStack = true
} else {
return nil
}

if !sig.foundStack || !sig.foundHeap || !sig.foundAnonVma {
if !sig.foundMainStack || !sig.foundHeap || !sig.foundAnonVma || !sig.foundThreadStack {
return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ exit_err() {

prog=sys_src_tester
dir=tests/e2e-inst-signatures/scripts
gcc $dir/$prog.c -o $dir/$prog -z execstack || exit_err "could not compile $prog.c"
gcc $dir/$prog.c -pthread -o $dir/$prog -z execstack || exit_err "could not compile $prog.c"
./$dir/$prog stack 2>&1 > /tmp/$prog.log || exit_err "could not run $prog"
./$dir/$prog heap 2>&1 > /tmp/$prog.log || exit_err "could not run $prog"
./$dir/$prog mmap 2>&1 > /tmp/$prog.log || exit_err "could not run $prog"
./$dir/$prog mmap 2>&1 > /tmp/$prog.log || exit_err "could not run $prog"
./$dir/$prog thread-stack 2>&1 > /tmp/$prog.log || exit_err "could not run $prog"
Loading

0 comments on commit 0955d4d

Please sign in to comment.