From 6064d76e0bbade24acc80193dda7981bfa362083 Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Fri, 11 Jun 2021 23:41:34 -0700 Subject: [PATCH 001/165] ghOSt release for v5.11 (up through and including kcl/387592) Change-Id: I6665a8dc6cb3ac0b11df2eafb9459f67e74315d1 --- Gconfig.ghost | 2 + arch/x86/entry/syscalls/syscall_64.tbl | 2 + arch/x86/include/asm/preempt.h | 7 + fs/eventpoll.c | 2 +- fs/kernfs/file.c | 31 + fs/timerfd.c | 52 +- include/asm-generic/vmlinux.lds.h | 14 + include/linux/bpf_types.h | 2 + include/linux/kernfs.h | 4 + include/linux/sched.h | 74 +- include/linux/sched/sysctl.h | 7 + include/linux/sched/task.h | 1 + include/linux/syscalls.h | 6 + include/linux/vmalloc.h | 1 + include/trace/events/sched.h | 20 + include/uapi/linux/bpf.h | 5 + include/uapi/linux/ghost.h | 452 +++ include/uapi/linux/sched.h | 1 + init/Kconfig | 15 + kernel/bpf/syscall.c | 17 + kernel/bpf/verifier.c | 2 + kernel/entry/common.c | 10 + kernel/fork.c | 10 +- kernel/futex.c | 2 +- kernel/sched/Makefile | 3 + kernel/sched/bpf.c | 263 ++ kernel/sched/core.c | 426 +- kernel/sched/fair.c | 44 +- kernel/sched/ghost.c | 5157 ++++++++++++++++++++++++ kernel/sched/ghost.h | 18 + kernel/sched/ghostfs.c | 903 +++++ kernel/sched/sched.h | 342 +- kernel/sysctl.c | 31 + kernel/time/hrtimer.c | 2 +- mm/vmalloc.c | 20 + 35 files changed, 7900 insertions(+), 48 deletions(-) create mode 100644 Gconfig.ghost create mode 100644 include/uapi/linux/ghost.h create mode 100644 kernel/sched/bpf.c create mode 100644 kernel/sched/ghost.c create mode 100644 kernel/sched/ghost.h create mode 100644 kernel/sched/ghostfs.c diff --git a/Gconfig.ghost b/Gconfig.ghost new file mode 100644 index 000000000000..2c15bc75cabc --- /dev/null +++ b/Gconfig.ghost @@ -0,0 +1,2 @@ +x86_64:CONFIG_SCHED_CLASS_GHOST=y +x86_64 and debug:CONFIG_DEBUG_GHOST=y diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 78672124d28b..b7b24da49776 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -363,6 +363,8 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +450 64 ghost_run sys_ghost_run +451 64 ghost sys_ghost # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 69485ca13665..133eb3a1697d 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -94,11 +94,18 @@ static __always_inline bool __preempt_count_dec_and_test(void) return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var])); } +#ifdef CONFIG_SCHED_CLASS_GHOST +extern void ghost_commit_greedy_txn(void); +#endif + /* * Returns true when we need to resched and can (barring IRQ state). */ static __always_inline bool should_resched(int preempt_offset) { +#ifdef CONFIG_SCHED_CLASS_GHOST + ghost_commit_greedy_txn(); +#endif return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset); } diff --git a/fs/eventpoll.c b/fs/eventpoll.c index a829af074eb5..9e4272dc35a1 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -1823,7 +1823,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, * the same lock on wakeup ep_poll_callback() side, so it * is safe to avoid an explicit barrier. */ - __set_current_state(TASK_INTERRUPTIBLE); + __set_current_state(TASK_INTERRUPTIBLE|__TASK_DEFERRABLE_WAKEUP); /* * Do the final check under the lock. ep_scan_ready_list() diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index c75719312147..335dfbf44408 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -843,6 +843,29 @@ static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait) return ret; } +static long kernfs_fop_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct kernfs_open_file *of = kernfs_of(file); + const struct kernfs_ops *ops; + long ret; + + /* + * Not locking of->mutex: you can have concurrent ioctl calls on the + * same file. + */ + if (!kernfs_get_active(of->kn)) + return -ENODEV; + ops = kernfs_ops(of->kn); + if (ops->ioctl) + ret = ops->ioctl(of, cmd, arg); + else + ret = -ENOIOCTLCMD; + kernfs_put_active(of->kn); + + return ret; +} + static void kernfs_notify_workfn(struct work_struct *work) { struct kernfs_node *kn; @@ -948,11 +971,19 @@ const struct file_operations kernfs_file_fops = { .open = kernfs_fop_open, .release = kernfs_fop_release, .poll = kernfs_fop_poll, + .unlocked_ioctl = kernfs_fop_ioctl, .fsync = noop_fsync, .splice_read = generic_file_splice_read, .splice_write = iter_file_splice_write, }; +struct kernfs_node *kernfs_node_from_file(struct file *file) +{ + if (file->f_op != &kernfs_file_fops) + return NULL; + return file->f_inode->i_private; +} + /** * __kernfs_create_file - kernfs internal function to create a file * @parent: directory to create the file in diff --git a/fs/timerfd.c b/fs/timerfd.c index c5509d2448e3..d3e0f0733dc1 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -28,6 +28,11 @@ #include #include +#ifdef CONFIG_SCHED_CLASS_GHOST +#include +#include +#endif + struct timerfd_ctx { union { struct hrtimer tmr; @@ -44,6 +49,9 @@ struct timerfd_ctx { struct list_head clist; spinlock_t cancel_lock; bool might_cancel; +#ifdef CONFIG_SCHED_CLASS_GHOST + struct timerfd_ghost timerfd_ghost; +#endif }; static LIST_HEAD(cancel_list); @@ -55,6 +63,13 @@ static inline bool isalarm(struct timerfd_ctx *ctx) ctx->clockid == CLOCK_BOOTTIME_ALARM; } +#ifdef CONFIG_SCHED_CLASS_GHOST +static inline bool timerfd_ghost_enabled(struct timerfd_ghost *timerfd_ghost) +{ + return timerfd_ghost->flags & TIMERFD_GHOST_ENABLED; +} +#endif + /* * This gets called when the timer event triggers. We set the "expired" * flag, but we do not re-arm the timer (in case it's necessary, @@ -64,11 +79,26 @@ static void timerfd_triggered(struct timerfd_ctx *ctx) { unsigned long flags; +#ifdef CONFIG_SCHED_CLASS_GHOST + struct timerfd_ghost timerfd_ghost = { + .flags = 0, /* disabled */ + }; +#endif + spin_lock_irqsave(&ctx->wqh.lock, flags); ctx->expired = 1; ctx->ticks++; wake_up_locked_poll(&ctx->wqh, EPOLLIN); +#ifdef CONFIG_SCHED_CLASS_GHOST + if (unlikely(timerfd_ghost_enabled(&ctx->timerfd_ghost))) + timerfd_ghost = ctx->timerfd_ghost; +#endif spin_unlock_irqrestore(&ctx->wqh.lock, flags); + +#ifdef CONFIG_SCHED_CLASS_GHOST + if (unlikely(timerfd_ghost_enabled(&timerfd_ghost))) + ghost_timerfd_triggered(&timerfd_ghost); +#endif } static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr) @@ -435,9 +465,14 @@ SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags) return ufd; } -static int do_timerfd_settime(int ufd, int flags, +#ifdef CONFIG_SCHED_CLASS_GHOST +int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, + struct itimerspec64 *old, struct timerfd_ghost *tfdl) +#else +static int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, struct itimerspec64 *old) +#endif { struct fd f; struct timerfd_ctx *ctx; @@ -501,7 +536,12 @@ static int do_timerfd_settime(int ufd, int flags, * Re-program the timer to the new value ... */ ret = timerfd_setup(ctx, flags, new); - +#ifdef CONFIG_SCHED_CLASS_GHOST + if (tfdl) + memcpy(&ctx->timerfd_ghost, tfdl, sizeof(struct timerfd_ghost)); + else + ctx->timerfd_ghost.flags = 0; /* disabled */ +#endif spin_unlock_irq(&ctx->wqh.lock); fdput(f); return ret; @@ -548,7 +588,11 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags, if (get_itimerspec64(&new, utmr)) return -EFAULT; +#ifdef CONFIG_SCHED_CLASS_GHOST + ret = do_timerfd_settime(ufd, flags, &new, &old, NULL); +#else ret = do_timerfd_settime(ufd, flags, &new, &old); +#endif if (ret) return ret; if (otmr && put_itimerspec64(&old, otmr)) @@ -576,7 +620,11 @@ SYSCALL_DEFINE4(timerfd_settime32, int, ufd, int, flags, if (get_old_itimerspec32(&new, utmr)) return -EFAULT; +#ifdef CONFIG_SCHED_CLASS_GHOST + ret = do_timerfd_settime(ufd, flags, &new, &old, NULL); +#else ret = do_timerfd_settime(ufd, flags, &new, &old); +#endif if (ret) return ret; if (otmr && put_old_itimerspec32(&old, otmr)) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b97c628ad91f..6bb060dc2f19 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -125,15 +125,29 @@ * used to determine the order of the priority of each sched class in * relation to each other. */ +#ifdef CONFIG_SCHED_CLASS_GHOST #define SCHED_DATA \ STRUCT_ALIGN(); \ __begin_sched_classes = .; \ *(__idle_sched_class) \ + *(__ghost_sched_class) \ *(__fair_sched_class) \ *(__rt_sched_class) \ *(__dl_sched_class) \ + *(__ghost_agent_sched_class) \ *(__stop_sched_class) \ __end_sched_classes = .; +#else +#define SCHED_DATA \ + STRUCT_ALIGN(); \ + __begin_sched_classes = .; \ + *(__idle_sched_class) \ + *(__fair_sched_class) \ + *(__rt_sched_class) \ + *(__dl_sched_class) \ + *(__stop_sched_class) \ + __end_sched_classes = .; +#endif /* The actual configuration determine if the init/exit sections * are handled as text/data or they can be discarded (which diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 99f7fd657d87..1a24c4d0686c 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -77,6 +77,8 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm, void *, void *) #endif /* CONFIG_BPF_LSM */ #endif +BPF_PROG_TYPE(BPF_PROG_TYPE_SCHEDULER, scheduler, struct bpf_scheduler, + struct bpf_scheduler_kern) BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 9e8ca8743c26..777878ddd05c 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -263,6 +263,9 @@ struct kernfs_ops { __poll_t (*poll)(struct kernfs_open_file *of, struct poll_table_struct *pt); + long (*ioctl)(struct kernfs_open_file *of, unsigned int cmd, + unsigned long arg); + int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma); #ifdef CONFIG_DEBUG_LOCK_ALLOC @@ -357,6 +360,7 @@ void kernfs_get(struct kernfs_node *kn); void kernfs_put(struct kernfs_node *kn); struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry); +struct kernfs_node *kernfs_node_from_file(struct file *file); struct kernfs_root *kernfs_root_from_sb(struct super_block *sb); struct inode *kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn); diff --git a/include/linux/sched.h b/include/linux/sched.h index 6e3a5eeec509..eb87eefdf9d7 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -95,7 +95,8 @@ struct io_uring_task; #define TASK_WAKING 0x0200 #define TASK_NOLOAD 0x0400 #define TASK_NEW 0x0800 -#define TASK_STATE_MAX 0x1000 +#define __TASK_DEFERRABLE_WAKEUP 0x1000 +#define TASK_STATE_MAX 0x2000 /* Convenience macros for the sake of set_current_state: */ #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) @@ -583,6 +584,38 @@ struct sched_dl_entity { #endif }; +#ifdef CONFIG_SCHED_CLASS_GHOST +struct ghost_queue; +struct ghost_status_word; +struct ghost_enclave; +struct timerfd_ghost; + +struct sched_ghost_entity { + struct list_head run_list; + ktime_t last_runnable_at; + + /* The following fields are protected by 'task_rq(p)->lock' */ + struct ghost_queue *dst_q; + struct ghost_status_word *status_word; + struct ghost_enclave *enclave; + + /* + * See also ghost_prepare_task_switch() and ghost_deferred_msgs() + * for flags that are used to defer messages. + */ + uint blocked_task : 1; + uint yield_task : 1; + uint new_task : 1; + uint agent : 1; + + struct list_head task_list; +}; + +extern void ghost_commit_greedy_txn(void); +extern void ghost_timerfd_triggered(struct timerfd_ghost *timer); + +#endif + #ifdef CONFIG_UCLAMP_TASK /* Number of utilization clamp buckets (shorter alias) */ #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT @@ -700,6 +733,10 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; +#ifdef CONFIG_SCHED_CLASS_GHOST + int64_t gtid; /* ghost tid */ + struct sched_ghost_entity ghost; +#endif #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif @@ -815,6 +852,7 @@ struct task_struct { * ->sched_remote_wakeup gets used, so it can be in this word. */ unsigned sched_remote_wakeup:1; + unsigned sched_deferrable_wakeup:1; /* Bit to tell LSMs we're in execve(): */ unsigned in_execve:1; @@ -1385,6 +1423,19 @@ struct task_struct { */ }; +struct bpf_scheduler_kern { +}; + +struct bpf_prog; +union bpf_attr; +enum bpf_prog_type; +int scheduler_bpf_prog_attach(const union bpf_attr *attr, + struct bpf_prog *prog); +int scheduler_bpf_prog_detach(const union bpf_attr *attr, + enum bpf_prog_type ptype); +int scheduler_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog); + static inline struct pid *task_pid(struct task_struct *task) { return task->thread_pid; @@ -1797,14 +1848,32 @@ extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk); }) #ifdef CONFIG_SMP +extern void do_resched_ipi_work(void); static __always_inline void scheduler_ipi(void) { +#ifdef CONFIG_SCHED_CLASS_GHOST + /* + * ghost_commit_pending_txn() needs RCU for correct operation so + * make sure RCU is watching in case we interrupted an idle CPU. + * + * Rather than the full irq_enter/irq_exit we do the bare minimum + * required for RCU to avoid pessimizing the common case (all work + * done in the irq return path). Also see the comment below about + * irq_enter/irq_exit. + */ + if (is_idle_task(current)) + rcu_irq_enter(); + ghost_commit_greedy_txn(); + if (is_idle_task(current)) + rcu_irq_exit(); +#endif /* * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting * TIF_NEED_RESCHED remotely (for the first time) will also send * this IPI. */ preempt_fold_need_resched(); + do_resched_ipi_work(); } extern unsigned long wait_task_inactive(struct task_struct *, long match_state); #else @@ -1961,6 +2030,9 @@ static inline bool vcpu_is_preempted(int cpu) } #endif +extern int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, + struct cpumask *new_mask); + extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); extern long sched_getaffinity(pid_t pid, struct cpumask *mask); diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 3c31ba88aca5..05bd0481985f 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -31,6 +31,13 @@ extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; +#ifdef CONFIG_SCHED_CLASS_GHOST +extern unsigned long sysctl_ghost_cfs_load_added; +extern int sysctl_ghost_wake_on_waker_cpu; +extern int sysctl_ghost_switchto_disable; +extern int sysctl_ghost_commit_at_tick; +#endif + enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, SCHED_TUNABLESCALING_LOG, diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index c0f71f2e7160..c21f17f2b2c2 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -53,6 +53,7 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); extern void init_idle(struct task_struct *idle, int cpu); extern int sched_fork(unsigned long clone_flags, struct task_struct *p); +extern void sched_cleanup_fork(struct task_struct *p); extern void sched_post_fork(struct task_struct *p); extern void sched_dead(struct task_struct *p); diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 7688bc983de5..fad3ee1a703a 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1364,4 +1364,10 @@ int __sys_getsockopt(int fd, int level, int optname, char __user *optval, int __user *optlen); int __sys_setsockopt(int fd, int level, int optname, char __user *optval, int optlen); + +asmlinkage long gsys_ghost_run(s64 gtid, u32 agent_barrier, u32 task_barrier, + int run_cpu, int run_flags); +asmlinkage long gsys_ghost(u64 op, u64 arg1, u64 arg2, u64 arg3, u64 arg4, + u64 arg5); + #endif diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index cedcda6593f6..b46e2ca10e35 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -100,6 +100,7 @@ extern void *vzalloc(unsigned long size); extern void *vmalloc_user(unsigned long size); extern void *vmalloc_node(unsigned long size, int node); extern void *vzalloc_node(unsigned long size, int node); +extern void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags); extern void *vmalloc_32(unsigned long size); extern void *vmalloc_32_user(unsigned long size); extern void *__vmalloc(unsigned long size, gfp_t gfp_mask); diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index cbe3e152d24c..683d8fdfa78d 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -684,6 +684,26 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TP_printk("cpu=%d", __entry->cpu) ); +TRACE_EVENT(sched_ghost_latched, + + TP_PROTO(struct task_struct *old, + struct task_struct *new), + + TP_ARGS(old, new), + + TP_STRUCT__entry( + __field(pid_t, old_pid) + __field(pid_t, new_pid) + ), + + TP_fast_assign( + __entry->old_pid = old ? old->pid : 0; + __entry->new_pid = new ? new->pid : 0; + ), + + TP_printk("old_pid=%d, new_pid=%d", __entry->old_pid, __entry->new_pid) +); + /* * Following tracepoints are not exported in tracefs and provide hooking * mechanisms only for testing and debugging purposes. diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 77d7c1bb2923..0f0448ca3af6 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -200,6 +200,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_EXT, BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, + BPF_PROG_TYPE_SCHEDULER = 35, }; enum bpf_attach_type { @@ -241,6 +242,7 @@ enum bpf_attach_type { BPF_XDP_CPUMAP, BPF_SK_LOOKUP, BPF_XDP, + BPF_SCHEDULER_TICK = 50, __MAX_BPF_ATTACH_TYPE }; @@ -5151,4 +5153,7 @@ enum { BTF_F_ZERO = (1ULL << 3), }; +struct bpf_scheduler { +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h new file mode 100644 index 000000000000..4cc099211501 --- /dev/null +++ b/include/uapi/linux/ghost.h @@ -0,0 +1,452 @@ +/* + * Copyright 2021 Google LLC + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _SCHED_GHOST_H_ +#define _SCHED_GHOST_H_ + +#include + +#ifndef __KERNEL__ +#include +#include +#endif + +// NOLINTBEGIN + +/* + * The version of ghOSt. It is important that the kernel and the userspace + * process are the same version as each other. Each successive version changes + * values in this header file, assumptions about operations in the kernel, etc. + */ +#define GHOST_VERSION 30 + +/* + * Define SCHED_GHOST via the ghost uapi unless it has already been defined + * via the proper channels (i.e. the official uapi header file). + */ +#ifndef SCHED_GHOST +#define SCHED_GHOST 18 +#endif + +/* + * This is userspace API so the usual suspects like __cacheline_aligned + * will not work. + * + * TODO: these are architecture-specific. + */ +#define __ghost_cacheline_aligned __attribute__((__aligned__(64))) +#define __ghost_page_aligned __attribute__((__aligned__(4096))) + +enum ghost_type { + GHOST_AGENT, + GHOST_TASK, +}; + +enum { + GHOST_SCHED_TASK_PRIO, + GHOST_SCHED_AGENT_PRIO, + GHOST_SCHED_MAX_PRIO, +}; + +/* + * Collateral for GHOST_CONFIG_QUEUE_WAKEUP op: + * - kernel supports up to 'GHOST_MAX_WINFO' wakeup targets for a queue. + * - 'prio' must be zero (intended to be a tiebreaker but not defined yet). + */ +#define GHOST_MAX_WINFO 2 +struct ghost_agent_wakeup { + int cpu; + int prio; +}; + +/* + * tuple decoding: + * 'type' is GHOST_TASK -> 'arg' is + * 'type' is GHOST_AGENT -> 'arg' is + */ +struct ghost_msg_src { + enum ghost_type type; + uint64_t arg; +}; +#define GHOST_THIS_CPU -1 + +struct ghost_sw_info { + uint32_t id; /* status_word region id */ + uint32_t index; /* index into the status_word array */ +}; + +struct ghost_ioc_sw_get_info { + struct ghost_msg_src request; + struct ghost_sw_info response; +}; + +#define GHOST_IOC_NULL _IO('g', 0) +#define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) +#define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) + +/* + * Status word region APIs. + */ +struct ghost_sw_region_header { + uint32_t version; /* ABI version */ + uint32_t id; /* region id */ + uint32_t numa_node; + uint32_t start; /* offset of the first status word */ + uint32_t capacity; /* total status words in this region */ + uint32_t available; /* free status words in this region */ +} __ghost_cacheline_aligned; +#define GHOST_SW_REGION_VERSION 0 + +/* + * Align status words up to the next power of two so we do not cross cache lines + * unnecessarily. At our current size, that is one cache line per status word, + * and two status words per cache line. + */ +struct ghost_status_word { + uint32_t barrier; + uint32_t flags; + uint64_t gtid; + uint64_t runtime; /* total time spent on the CPU in nsecs */ +} __attribute__((__aligned__(32))); + +#define GHOST_SW_F_INUSE (1U << 0) /* status_word in use */ +#define GHOST_SW_F_CANFREE (1U << 1) /* status_word can be freed */ + +/* agent-specific status_word.flags */ +#define GHOST_SW_CPU_AVAIL (1U << 8) /* CPU available hint */ +#define GHOST_SW_BOOST_PRIO (1U << 9) /* agent with boosted prio */ + +/* task-specific status_word.flags */ +#define GHOST_SW_TASK_ONCPU (1U << 16) /* task is oncpu */ +#define GHOST_SW_TASK_RUNNABLE (1U << 17) /* task is runnable */ +#define GHOST_SW_TASK_IS_AGENT (1U << 18) + +/* + * Queue APIs. + */ +struct ghost_queue_header { + uint32_t version; /* ABI version */ + uint32_t start; /* offset from the header to start of ring */ + uint32_t nelems; /* power-of-2 size of ghost_ring.msgs[] */ +} __ghost_cacheline_aligned; +#define GHOST_QUEUE_VERSION 0 + +struct ghost_msg { + uint16_t type; /* message type */ + uint16_t length; /* length of this message including payload */ + uint32_t seqnum; /* sequence number for this msg source */ + uint32_t payload[0]; /* variable length payload */ +}; + +/* + * Messages are grouped by type and each type can have up to 64 messages. + * (the limit of 64 is arbitrary). + */ +#define _MSG_TASK_FIRST 64 +#define _MSG_TASK_LAST (_MSG_TASK_FIRST + 64 - 1) + +#define _MSG_CPU_FIRST 128 +#define _MSG_CPU_LAST (_MSG_CPU_FIRST + 64 - 1) + +/* ghost message types */ +enum { + /* misc msgs */ + MSG_NOP = 0, + + /* task messages */ + MSG_TASK_DEAD = _MSG_TASK_FIRST, + MSG_TASK_BLOCKED, + MSG_TASK_WAKEUP, + MSG_TASK_NEW, + MSG_TASK_PREEMPT, + MSG_TASK_YIELD, + MSG_TASK_DEPARTED, + MSG_TASK_SWITCHTO, + + /* cpu messages */ + MSG_CPU_TICK = _MSG_CPU_FIRST, + MSG_CPU_TIMER_EXPIRED, + MSG_CPU_NOT_IDLE, /* requested via run_flags: NEED_CPU_NOT_IDLE */ +}; + +/* TODO: Move payload to header once all clients updated. */ +struct ghost_msg_payload_task_new { + uint64_t gtid; + uint64_t runtime; /* cumulative runtime in ns */ + uint16_t runnable; + struct ghost_sw_info sw_info; +}; + +struct ghost_msg_payload_task_preempt { + uint64_t gtid; + uint64_t runtime; /* cumulative runtime in ns */ + int cpu; + char from_switchto; +}; + +struct ghost_msg_payload_task_yield { + uint64_t gtid; + uint64_t runtime; /* cumulative runtime in ns */ + int cpu; + char from_switchto; +}; + +struct ghost_msg_payload_task_blocked { + uint64_t gtid; + uint64_t runtime; /* cumulative runtime in ns */ + int cpu; + char from_switchto; +}; + +struct ghost_msg_payload_task_dead { + uint64_t gtid; +}; + +struct ghost_msg_payload_task_departed { + uint64_t gtid; + int cpu; + char from_switchto; +}; + +struct ghost_msg_payload_task_wakeup { + uint64_t gtid; + char deferrable; /* bool: 0 or 1 */ +}; + +struct ghost_msg_payload_task_switchto { + uint64_t gtid; + uint64_t runtime; /* cumulative runtime in ns */ + int cpu; +}; + +struct ghost_msg_payload_cpu_not_idle { + int cpu; + uint64_t next_gtid; +}; + +struct ghost_msg_payload_cpu_tick { + int cpu; +}; + +struct ghost_msg_payload_timer { + int cpu; + uint64_t cookie; +}; + +#ifdef __cplusplus +#include +typedef std::atomic _ghost_ring_index_t; +#else +typedef volatile uint32_t _ghost_ring_index_t; +#endif + +struct ghost_ring { + /* + * kernel produces at 'head & (nelems-1)' and + * agent consumes from 'tail & (nelems-1)'. + * + * kernel increments 'overflow' any time there aren't enough + * free slots to produce a message. + */ + _ghost_ring_index_t head; + _ghost_ring_index_t tail; + _ghost_ring_index_t overflow; + struct ghost_msg msgs[0]; /* array of size 'header->nelems' */ +}; + +#define GHOST_MAX_QUEUE_ELEMS 65536 /* arbitrary */ + +/* + * Define ghOSt syscall numbers here until they can be discovered via + * . + */ +#ifndef __NR_ghost_run +#define __NR_ghost_run 450 +#endif +#ifndef __NR_ghost +#define __NR_ghost 451 +#endif + +/* + * 'ops' supported by gsys_ghost(). + */ +enum ghost_ops { + GHOST_NULL, + GHOST_CREATE_QUEUE, + GHOST_ASSOCIATE_QUEUE, + GHOST_CONFIG_QUEUE_WAKEUP, + GHOST_SET_OPTION, + GHOST_GET_CPU_TIME, + GHOST_COMMIT_TXN, + GHOST_SYNC_GROUP_TXN, + GHOST_TIMERFD_SETTIME, + GHOST_GTID_LOOKUP, + GHOST_GET_GTID_10, /* TODO: deprecate */ + GHOST_GET_GTID_11, /* TODO: deprecate */ + GHOST_SET_DEFAULT_QUEUE, +}; + +/* + * 'ops' supported by gsys_ghost() that are used by the 'base' library in + * userspace. Arbitrary applications may use the 'base' library requiring + * these 'ops' to be immutable (both in terms of the entry-point and the + * functionality). + * + * The 'base' library can detect support for a particular op as follows: + * - return value 0 indicates that the 'op' was successful. + * - return value of -1 indicates that 'op' was not successful: + * - ENOSYS: kernel does not support ghost. + * - EOPNOTSUPP: kernel supports ghost but doesn't recognize this op. + * (for e.g. newer binary running on an older kernel). + * - any other errno indicates an op-specific error. + */ +enum ghost_base_ops { + _GHOST_BASE_OP_FIRST = 65536, /* avoid overlap with 'ghost_ops' */ + GHOST_BASE_GET_GTID = _GHOST_BASE_OP_FIRST, + + /* + * New ops must be added to the end of this enumeration. + */ +}; + +/* status flags for GHOST_ASSOCIATE_QUEUE */ +#define GHOST_ASSOC_SF_ALREADY (1 << 0) /* Queue already set */ +#define GHOST_ASSOC_SF_BRAND_NEW (1 << 1) /* TASK_NEW not sent yet */ + +/* flags accepted by ghost_run() */ +#define RTLA_ON_PREEMPT (1 << 0) /* Return To Local Agent on preemption */ +#define RTLA_ON_BLOCKED (1 << 1) /* Return To Local Agent on block */ +#define RTLA_ON_YIELD (1 << 2) /* Return To Local Agent on yield */ +#define RTLA_ON_IDLE (1 << 5) /* Return To Local Agent on idle */ +#define NEED_L1D_FLUSH (1 << 6) /* Flush L1 dcache before entering guest */ +#define NEED_CPU_NOT_IDLE (1 << 7) /* Notify agent when a non-idle task is + * scheduled on the cpu. + */ +#define ALLOW_TASK_ONCPU (1 << 8) /* If task is already running on remote + * cpu then let it keep running there. + */ + +/* txn->commit_flags */ +enum txn_commit_at { + /* + * commit_flags = 0 indicates a greedy commit (i.e. agent doesn't + * care where the commit happens). The kernel tries to apply the + * commit at the earliest opportunity (e.g. return-to-user). + */ + COMMIT_AT_SCHEDULE = 1, /* commit when oncpu task schedules */ + COMMIT_AT_TXN_COMMIT, /* commit in GHOST_COMMIT_TXN op */ +}; + +/* special 'gtid' encodings that can be passed to ghost_run() */ +#define GHOST_NULL_GTID (0) +#define GHOST_AGENT_GTID (-1) +#define GHOST_IDLE_GTID (-2) + +/* ghost transaction */ + +/* + * _ghost_txn_state_t is not expected to be used anywhere except as the type + * of ghost_txn::state below. See cl/350179823 as an example of unintentional + * consequences. + */ +#ifdef __cplusplus +#include +typedef std::atomic _ghost_txn_state_t; +typedef std::atomic _ghost_txn_owner_t; +#else +typedef int32_t _ghost_txn_state_t; +typedef int32_t _ghost_txn_owner_t; +#endif + +struct ghost_txn { + int32_t version; + int32_t cpu; /* readonly-after-init */ + _ghost_txn_state_t state; + uint32_t agent_barrier; + uint32_t task_barrier; + uint16_t run_flags; + uint8_t commit_flags; + uint8_t unused; + int64_t gtid; + /* + * Context-dependent fields. + */ + union { + /* only used during a sync-group commit */ + _ghost_txn_owner_t sync_group_owner; + } u; +} __ghost_cacheline_aligned; + +struct ghost_cpu_data { + struct ghost_txn txn; +} __ghost_page_aligned; + +#define GHOST_TXN_VERSION 0 + +/* + * (txn->state == GHOST_TXN_READY) transaction is ready + * (txn->state >= 0 && txn->state < nr_cpu_ids) transaction is claimed + * (txn->state < 0) transaction is committed + */ +enum ghost_txn_state { + GHOST_TXN_COMPLETE = INT_MIN, + GHOST_TXN_ABORTED, + GHOST_TXN_TARGET_ONCPU, + GHOST_TXN_TARGET_STALE, + GHOST_TXN_TARGET_NOT_FOUND, + GHOST_TXN_TARGET_NOT_RUNNABLE, + GHOST_TXN_AGENT_STALE, + GHOST_TXN_CPU_OFFLINE, + GHOST_TXN_CPU_UNAVAIL, + GHOST_TXN_INVALID_FLAGS, + GHOST_TXN_INVALID_TARGET, + GHOST_TXN_NOT_PERMITTED, + GHOST_TXN_INVALID_CPU, + GHOST_TXN_NO_AGENT, + GHOST_TXN_UNSUPPORTED_VERSION, + GHOST_TXN_POISONED, + + /* + * Values [0-nr_cpu_ids) indicates that the transaction is claimed + * by the specific CPU. + */ + + GHOST_TXN_READY = INT_MAX, +}; + +/* GHOST_TIMERFD_SETTIME */ +struct timerfd_ghost { + int cpu; + int flags; + uint64_t cookie; +}; +#define TIMERFD_GHOST_ENABLED (1 << 0) + +/* GHOST_GTID_LOOKUP */ +enum { + GHOST_GTID_LOOKUP_TGID, /* return group_leader pid */ +}; + +/* + * ghost tids referring to normal tasks always have a positive value: + * (0 | 22 bits of actual pid_t | 41 bit non-zero seqnum) + * + * The embedded 'pid' following linux terminology is actually referring + * to the thread id (i.e. what would be returned by syscall(__NR_gettid)). + */ +#define GHOST_TID_SEQNUM_BITS 41 +#define GHOST_TID_PID_BITS 22 + +// NOLINTEND + +#endif /* _SCHED_GHOST_H_ */ diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h index 3bac0a8ceab2..fe5590b89eb5 100644 --- a/include/uapi/linux/sched.h +++ b/include/uapi/linux/sched.h @@ -118,6 +118,7 @@ struct clone_args { /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 #define SCHED_DEADLINE 6 +#define SCHED_GHOST 18 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 diff --git a/init/Kconfig b/init/Kconfig index 29ad68325028..cec0f33a7f0d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2355,3 +2355,18 @@ config ARCH_HAS_SYNC_CORE_BEFORE_USERMODE # . config ARCH_HAS_SYSCALL_WRAPPER def_bool n + +config SCHED_CLASS_GHOST + bool "ghOSt scheduling class" + depends on X86_64 + default n + help + ghOSt is a scheduling class with policy decisions delegated + to a user-space agent. + +config DEBUG_GHOST + bool "Extra debug checks for ghOSt." + depends on SCHED_CLASS_GHOST + default n + help + Enable extra runtime debug checks for ghOSt. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e5999d86c76e..7980c5a5e1db 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2002,6 +2002,13 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, if (expected_attach_type == BPF_SK_LOOKUP) return 0; return -EINVAL; + case BPF_PROG_TYPE_SCHEDULER: + switch (expected_attach_type) { + case BPF_SCHEDULER_TICK: + return 0; + default: + return -EINVAL; + } case BPF_PROG_TYPE_EXT: if (expected_attach_type) return -EINVAL; @@ -2952,6 +2959,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) return BPF_PROG_TYPE_SK_LOOKUP; case BPF_XDP: return BPF_PROG_TYPE_XDP; + case BPF_SCHEDULER_TICK: + return BPF_PROG_TYPE_SCHEDULER; default: return BPF_PROG_TYPE_UNSPEC; } @@ -3007,6 +3016,9 @@ static int bpf_prog_attach(const union bpf_attr *attr) case BPF_PROG_TYPE_SOCK_OPS: ret = cgroup_bpf_prog_attach(attr, ptype, prog); break; + case BPF_PROG_TYPE_SCHEDULER: + ret = scheduler_bpf_prog_attach(attr, prog); + break; default: ret = -EINVAL; } @@ -3043,6 +3055,8 @@ static int bpf_prog_detach(const union bpf_attr *attr) case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_SOCK_OPS: return cgroup_bpf_prog_detach(attr, ptype); + case BPF_PROG_TYPE_SCHEDULER: + return scheduler_bpf_prog_detach(attr, ptype); default: return -EINVAL; } @@ -4060,6 +4074,9 @@ static int link_create(union bpf_attr *attr) ret = bpf_xdp_link_attach(attr, prog); break; #endif + case BPF_PROG_TYPE_SCHEDULER: + ret = scheduler_bpf_link_attach(attr, prog); + break; default: ret = -EINVAL; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 37581919e050..033f3a32a73b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7981,6 +7981,8 @@ static int check_return_code(struct bpf_verifier_env *env) case BPF_PROG_TYPE_SK_LOOKUP: range = tnum_range(SK_DROP, SK_PASS); break; + case BPF_PROG_TYPE_SCHEDULER: + break; case BPF_PROG_TYPE_EXT: /* freplace program can return anything as its return value * depends on the to-be-replaced kernel func or bpf program. diff --git a/kernel/entry/common.c b/kernel/entry/common.c index f9d491b17b78..a438a6a885ed 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -8,6 +8,10 @@ #include "common.h" +#ifdef CONFIG_SCHED_CLASS_GHOST +#include +#endif + #define CREATE_TRACE_POINTS #include @@ -184,6 +188,9 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, * enabled above. */ local_irq_disable_exit_to_user(); +#ifdef CONFIG_SCHED_CLASS_GHOST + ghost_commit_greedy_txn(); +#endif ti_work = READ_ONCE(current_thread_info()->flags); } @@ -196,6 +203,9 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) unsigned long ti_work = READ_ONCE(current_thread_info()->flags); lockdep_assert_irqs_disabled(); +#ifdef CONFIG_SCHED_CLASS_GHOST + ghost_commit_greedy_txn(); +#endif if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) ti_work = exit_to_user_mode_loop(regs, ti_work); diff --git a/kernel/fork.c b/kernel/fork.c index d66cd1014211..a5842287d897 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -103,6 +103,8 @@ #include #include +#include "sched/sched.h" + #include #define CREATE_TRACE_POINTS @@ -2070,7 +2072,7 @@ static __latent_entropy struct task_struct *copy_process( retval = perf_event_init_task(p); if (retval) - goto bad_fork_cleanup_policy; + goto bad_fork_cleanup_sched; retval = audit_alloc(p); if (retval) goto bad_fork_cleanup_perf; @@ -2315,6 +2317,10 @@ static __latent_entropy struct task_struct *copy_process( copy_oom_score_adj(clone_flags, p); +#ifdef CONFIG_SCHED_CLASS_GHOST + p->gtid = ghost_alloc_gtid(p); +#endif + return p; bad_fork_cancel_cgroup: @@ -2358,6 +2364,8 @@ static __latent_entropy struct task_struct *copy_process( audit_free(p); bad_fork_cleanup_perf: perf_event_free_task(p); +bad_fork_cleanup_sched: + sched_cleanup_fork(p); bad_fork_cleanup_policy: lockdep_free_task(p); #ifdef CONFIG_NUMA diff --git a/kernel/futex.c b/kernel/futex.c index 45a13eb8894e..c1acd1db71cb 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -2585,7 +2585,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, * queue_me() calls spin_unlock() upon completion, both serializing * access to the hash list and forcing another memory barrier. */ - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE | __TASK_DEFERRABLE_WAKEUP); queue_me(q, hb); /* Arm the timer */ diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 5fc9c9b70862..f97342908659 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -36,3 +36,6 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o +obj-$(CONFIG_SCHED_CLASS_GHOST) += ghostfs.o +obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost.o +obj-$(CONFIG_BPF) += bpf.o diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c new file mode 100644 index 000000000000..588ec7e292c8 --- /dev/null +++ b/kernel/sched/bpf.c @@ -0,0 +1,263 @@ +/* + * Copyright 2021 Google LLC + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "sched.h" +#include + +static DEFINE_MUTEX(sched_tick); +static struct bpf_prog *tick_prog; + +bool bpf_sched_ghost_skip_tick(void) +{ + struct bpf_scheduler_kern ctx = {}; + struct bpf_prog *prog; + int ret; + + rcu_read_lock(); + + prog = rcu_dereference(tick_prog); + if (!prog) { + rcu_read_unlock(); + return false; + } + + /* prog returns 1 if we want a tick on this cpu. */ + ret = BPF_PROG_RUN(prog, &ctx); + + rcu_read_unlock(); + + return ret != 1; +} + +static int sched_tick_attach(struct bpf_prog *prog) +{ + mutex_lock(&sched_tick); + if (tick_prog) { + mutex_unlock(&sched_tick); + return -EBUSY; + } + rcu_assign_pointer(tick_prog, prog); + mutex_unlock(&sched_tick); + return 0; +} + +static void sched_tick_detach(struct bpf_prog *prog) +{ + mutex_lock(&sched_tick); + rcu_replace_pointer(tick_prog, NULL, lockdep_is_held(&sched_tick)); + mutex_unlock(&sched_tick); +} + +struct bpf_sched_link { + struct bpf_link link; + enum bpf_attach_type ea_type; +}; + +static void bpf_sched_link_release(struct bpf_link *link) +{ + struct bpf_sched_link *sc_link = + container_of(link, struct bpf_sched_link, link); + + switch (sc_link->ea_type) { + case BPF_SCHEDULER_TICK: + sched_tick_detach(link->prog); + break; + default: + break; + }; + +} + +static void bpf_sched_link_dealloc(struct bpf_link *link) +{ + struct bpf_sched_link *sc_link = + container_of(link, struct bpf_sched_link, link); + + kfree(sc_link); +} + +static const struct bpf_link_ops bpf_sched_link_ops = { + .release = bpf_sched_link_release, + .dealloc = bpf_sched_link_dealloc, +}; + +int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + struct bpf_link_primer link_primer; + struct bpf_sched_link *sc_link; + enum bpf_attach_type ea_type; + int err; + + if (attr->link_create.flags) + return -EINVAL; + if (prog->expected_attach_type != attr->link_create.attach_type) + return -EINVAL; + ea_type = prog->expected_attach_type; + + switch (ea_type) { + case BPF_SCHEDULER_TICK: + break; + default: + return -EINVAL; + }; + + sc_link = kzalloc(sizeof(*sc_link), GFP_USER); + if (!sc_link) + return -ENOMEM; + bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, + &bpf_sched_link_ops, prog); + sc_link->ea_type = ea_type; + + err = bpf_link_prime(&sc_link->link, &link_primer); + if (err) { + kfree(sc_link); + return -EINVAL; + } + + switch (ea_type) { + case BPF_SCHEDULER_TICK: + err = sched_tick_attach(prog); + break; + default: + pr_warn("bad sched bpf ea_type %d, should be unreachable", + ea_type); + err = -EINVAL; + break; + }; + if (err) { + bpf_link_cleanup(&link_primer); + return err; + } + + return bpf_link_settle(&link_primer); +} + +/* netns does this to have a packed array of progs[type]. might do this for the + * task type only, or maybe for all sched types. + */ +enum sched_bpf_attach_type { + SCHED_BPF_INVALID = -1, + SCHED_BPF_TICK = 0, + MAX_SCHED_BPF_ATTACH_TYPE +}; + +static inline enum sched_bpf_attach_type +to_sched_bpf_attach_type(enum bpf_attach_type attach_type) +{ + switch (attach_type) { + case BPF_SCHEDULER_TICK: + return SCHED_BPF_TICK; + default: + return SCHED_BPF_INVALID; + } +} + +int scheduler_bpf_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) +{ + enum sched_bpf_attach_type type; + + if (attr->target_fd || attr->attach_flags || attr->replace_bpf_fd) + return -EINVAL; + type = to_sched_bpf_attach_type(attr->attach_type); + if (type < 0) + return -EINVAL; + + /* no task attachable types yet */ + + return -1; +} + +int scheduler_bpf_prog_detach(const union bpf_attr *attr, + enum bpf_prog_type ptype) +{ + struct bpf_prog *prog; + enum sched_bpf_attach_type type; + + if (attr->attach_flags) + return -EINVAL; + + type = to_sched_bpf_attach_type(attr->attach_type); + if (type < 0) + return -EINVAL; + + prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + if (prog->expected_attach_type != attr->attach_type) { + bpf_prog_put(prog); + return -EINVAL; + } + + /* no task attachable types yet */ + + bpf_prog_put(prog); + + return -1; +} + +static const struct bpf_func_proto * +scheduler_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + default: + return bpf_base_func_proto(func_id); + } +} + +static bool scheduler_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + /* The verifier guarantees that size > 0. */ + if (off < 0 || off + size > sizeof(struct bpf_scheduler) || off % size) + return false; + + switch (off) { + default: + return false; + } +} + +#define SCHEDULER_ACCESS_FIELD(T, F) \ + T(BPF_FIELD_SIZEOF(struct bpf_scheduler_kern, F), \ + si->dst_reg, si->src_reg, \ + offsetof(struct bpf_scheduler_kern, F)) + +static u32 scheduler_convert_ctx_access(enum bpf_access_type type, + const struct bpf_insn *si, + struct bpf_insn *insn_buf, + struct bpf_prog *prog, u32 *target_size) +{ + struct bpf_insn *insn = insn_buf; + + switch (si->off) { + default: + *target_size = 0; + break; + } + + *target_size = sizeof(u32); + + return insn - insn_buf; +} + +const struct bpf_verifier_ops scheduler_verifier_ops = { + .get_func_proto = scheduler_func_proto, + .is_valid_access = scheduler_is_valid_access, + .convert_ctx_access = scheduler_convert_ctx_access, +}; + +const struct bpf_prog_ops scheduler_prog_ops = { +}; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ff74fca39ed2..52ef3198d183 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -10,6 +10,7 @@ #include #undef CREATE_TRACE_POINTS +#include "ghost.h" #include "sched.h" #include @@ -347,6 +348,10 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) rq->curr->sched_class->task_tick(rq, rq->curr, 1); rq_unlock(rq, &rf); +#ifdef CONFIG_SCHED_CLASS_GHOST + ghost_tick(rq); +#endif + return HRTIMER_NORESTART; } @@ -459,7 +464,7 @@ static inline void hrtick_rq_init(struct rq *rq) * this avoids any races wrt polling state changes and thereby avoids * spurious IPIs. */ -static bool set_nr_and_not_polling(struct task_struct *p) +bool set_nr_and_not_polling(struct task_struct *p) { struct thread_info *ti = task_thread_info(p); return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); @@ -490,7 +495,7 @@ static bool set_nr_if_polling(struct task_struct *p) } #else -static bool set_nr_and_not_polling(struct task_struct *p) +bool set_nr_and_not_polling(struct task_struct *p) { set_tsk_need_resched(p); return true; @@ -633,6 +638,53 @@ void resched_cpu(int cpu) raw_spin_unlock_irqrestore(&rq->lock, flags); } +#ifdef CONFIG_SMP +/* + * Guarantee that the given cpu will reschedule. Useful for situations where + * we cannot hold dst_rq->lock. + */ +void resched_cpu_unlocked(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (cmpxchg(&rq->resched_ipi_work, 0, 1) == 0) { +#if 0 + /* + * XXX unlike 4.15 the idle loop in 5.11 does not call + * scheduler_ipi() after it detects that cpu needs to + * reschedule (therefore we always send an IPI). + */ + if (READ_ONCE(rq->curr) == rq->idle && + set_nr_if_polling(rq->idle)) + trace_sched_wake_idle_without_ipi(cpu); + else +#endif + smp_send_reschedule(cpu); + } +} + +/* + * Called from scheduler_ipi() in hardirq context with interrupts disabled. + */ +void do_resched_ipi_work(void) +{ + struct rq *rq = this_rq(); + struct rq_flags rf; + + if (!rq_has_resched_ipi_work(rq)) + return; + + irq_enter(); + rq_lock_irqsave(rq, &rf); + if (rq_has_resched_ipi_work(rq)) { + rq->resched_ipi_work = 0; + resched_curr(rq); + } + rq_unlock_irqrestore(rq, &rf); + irq_exit(); +} +#endif + #ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ_COMMON /* @@ -1705,10 +1757,25 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) { - if (p->sched_class == rq->curr->sched_class) + if (p->sched_class == rq->curr->sched_class) { rq->curr->sched_class->check_preempt_curr(rq, p, flags); - else if (p->sched_class > rq->curr->sched_class) + } else if (p->sched_class > rq->curr->sched_class) { resched_curr(rq); + } +#ifdef CONFIG_SCHED_CLASS_GHOST + else if (&ghost_agent_sched_class > rq->curr->sched_class && + is_agent(rq, p)) { + /* + * Normally, ghost threads have the lowest + * priority. The ghost agent thread, however, is + * allowed to run in the higher priority ghost + * agent class when it would otherwise be + * preempted by another sched_class. See + * GHOST_SW_BOOST_PRIO for more details. + */ + resched_curr(rq); + } +#endif /* * A queue event has occurred, and we're going to schedule. In @@ -1839,8 +1906,8 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) * * Returns (locked) new rq. Old rq's lock is released. */ -static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, - struct task_struct *p, int new_cpu) +struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, int new_cpu) { lockdep_assert_held(&rq->lock); @@ -2358,6 +2425,13 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, * Must re-check here, to close a race against __kthread_bind(), * sched_setaffinity() is not guaranteed to observe the flag. */ +#ifdef CONFIG_SCHED_CLASS_GHOST + /* ghost agents do not allow affinity manipulations. */ + if (p->sched_class == &ghost_sched_class && p->ghost.agent) { + ret = -EINVAL; + goto out; + } +#endif if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { ret = -EINVAL; goto out; @@ -2702,6 +2776,26 @@ void kick_process(struct task_struct *p) cpu = task_cpu(p); if ((cpu != smp_processor_id()) && task_curr(p)) smp_send_reschedule(cpu); +#ifdef CONFIG_SCHED_CLASS_GHOST + /* + * When an agent is 'blocked_in_run' its 'task->state' is TASK_RUNNING + * so it won't be "woken up" when a signal is delivered to it (see + * signal_wake_up_state() for details). This in turn implies that + * the signal handling is delayed until there is a scheduling edge + * on the agent's CPU (see pick_agent() for details). + * + * Ensure timely signal handling by forcing a scheduling edge on + * the agent's CPU. + */ + else if (unlikely(p->ghost.agent && signal_pending(p))) { + if (cpu == smp_processor_id()) { + set_tsk_need_resched(current); + set_preempt_need_resched(); + } else { + resched_cpu_unlocked(cpu); + } + } +#endif preempt_enable(); } EXPORT_SYMBOL_GPL(kick_process); @@ -3092,6 +3186,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags struct rq *rq = cpu_rq(cpu); p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); + p->sched_deferrable_wakeup = !!(wake_flags & WF_DEFERRABLE_WAKEUP); WRITE_ONCE(rq->ttwu_pending, 1); __smp_call_single_queue(cpu, &p->wake_entry.llist); @@ -3316,6 +3411,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { unsigned long flags; int cpu, success = 0; + bool deferrable_wakeup; preempt_disable(); if (p == current) { @@ -3408,6 +3504,13 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_acquire__after_ctrl_dep(); + /* + * Latch 'sched_deferrable_wakeup' before we change 'p->state' below. + * Everything before this was a light wakeup (wakee was 'current' + * or 'on_rq') and thus don't actually produce TASK_WAKEUP msgs. + */ + deferrable_wakeup = !!(p->state & __TASK_DEFERRABLE_WAKEUP); + /* * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq * == 0), which means we need to do an enqueue, change p->state to @@ -3436,7 +3539,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) * scheduling. */ if (smp_load_acquire(&p->on_cpu) && - ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU)) + ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU | + (deferrable_wakeup ? WF_DEFERRABLE_WAKEUP : 0))) goto unlock; /* @@ -3450,6 +3554,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + p->sched_deferrable_wakeup = deferrable_wakeup; + cpu = select_task_rq(p, p->wake_cpu, wake_flags | WF_TTWU); if (task_cpu(p) != cpu) { if (p->in_iowait) { @@ -3582,6 +3688,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->rt.on_rq = 0; p->rt.on_list = 0; +#ifdef CONFIG_SCHED_CLASS_GHOST + sched_ghost_entity_init(p); +#endif + #ifdef CONFIG_PREEMPT_NOTIFIERS INIT_HLIST_HEAD(&p->preempt_notifiers); #endif @@ -3734,7 +3844,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) * Revert to default priority/policy on fork if requested. */ if (unlikely(p->sched_reset_on_fork)) { - if (task_has_dl_policy(p) || task_has_rt_policy(p)) { + if (task_has_dl_policy(p) || task_has_rt_policy(p) || task_has_ghost_policy(p)) { p->policy = SCHED_NORMAL; p->static_prio = NICE_TO_PRIO(0); p->rt_priority = 0; @@ -3751,6 +3861,18 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) p->sched_reset_on_fork = 0; } +#ifdef CONFIG_SCHED_CLASS_GHOST + if (task_has_ghost_policy(p)) { + int error; + + p->sched_class = &ghost_sched_class; + error = ghost_sched_fork(p); + if (error) { + put_cpu(); + return error; + } + } else +#endif if (dl_prio(p->prio)) return -EAGAIN; else if (rt_prio(p->prio)) @@ -3798,6 +3920,14 @@ void sched_post_fork(struct task_struct *p) uclamp_post_fork(p); } +void sched_cleanup_fork(struct task_struct *p) +{ +#ifdef CONFIG_SCHED_CLASS_GHOST + if (task_has_ghost_policy(p)) + ghost_sched_cleanup_fork(p); +#endif +} + unsigned long to_ratio(u64 period, u64 runtime) { if (runtime == RUNTIME_INF) @@ -3991,7 +4121,8 @@ static void do_balance_callbacks(struct rq *rq, struct callback_head *head) while (head) { func = (void (*)(struct rq *))head->func; - next = head->next; + /* The last element pointed to itself */ + next = head->next == head ? NULL : head->next; head->next = NULL; head = next; @@ -4107,6 +4238,88 @@ static inline void kmap_local_sched_in(void) #endif } +#ifdef CONFIG_SCHED_CLASS_GHOST +#include +static inline void ghost_prepare_task_switch(struct rq *rq, + struct task_struct *prev, + struct task_struct *next) +{ + struct ghost_status_word *agent_sw; + + if (rq->ghost.agent) { + /* + * XXX pick_next_task_fair() can return 'rq->idle' via + * core_tag_pick_next_matching_rendezvous(). + */ + agent_sw = rq->ghost.agent->ghost.status_word; + if (!task_has_ghost_policy(next) && next != rq->idle) { + ghost_sw_clear_flag(agent_sw, GHOST_SW_CPU_AVAIL); + if (rq->ghost.run_flags & NEED_CPU_NOT_IDLE) { + rq->ghost.run_flags &= ~NEED_CPU_NOT_IDLE; + ghost_need_cpu_not_idle(rq, next); + } + } else { + ghost_sw_set_flag(agent_sw, GHOST_SW_CPU_AVAIL); + } + } + + if (!task_has_ghost_policy(prev)) + goto done; + + /* Who watches the watchmen? */ + if (prev == rq->ghost.agent) + goto done; + + /* If we're on the ghost.tasks list, then we're runnable. */ + if (!list_empty(&prev->ghost.run_list)) { + /* + * Keep ghost.tasks sorted by last_runnable_at. Whenever we set + * the time, we append to the tail. + */ + list_del_init(&prev->ghost.run_list); + list_add_tail(&prev->ghost.run_list, &rq->ghost.tasks); + prev->ghost.last_runnable_at = ktime_get(); + } + + if (rq->ghost.check_prev_preemption) { + rq->ghost.check_prev_preemption = false; + WARN_ON_ONCE(task_has_ghost_policy(next)); + ghost_task_preempted(rq, prev); + ghost_wake_agent_of(prev); + } + +done: + /* + * Clear the ONCPU bit after producing the task state change msg + * (e.g. preempted). This guarantees that when a task is offcpu + * its 'task_barrier' is stable. + */ + if (task_has_ghost_policy(prev)) { + struct ghost_status_word *prev_sw = prev->ghost.status_word; + WARN_ON_ONCE(!(prev_sw->flags & GHOST_SW_TASK_ONCPU)); + ghost_sw_clear_flag(prev_sw, GHOST_SW_TASK_ONCPU); + } + + /* + * CPU in a ghost switchto chain or switching to a non-ghost task + * while a task is latched. + * + * Treat this like latched_task preemption because we don't know when + * the CPU will be available again so no point in keeping it latched. + */ + if (rq->ghost.latched_task && + (!task_has_ghost_policy(next) || rq->ghost.switchto_count > 0)) { + ghost_latched_task_preempted(rq); + } + + /* + * The last task in the chain scheduled (blocked/yielded/preempted). + */ + if (rq->ghost.switchto_count < 0) + rq->ghost.switchto_count = 0; +} +#endif /* CONFIG_SCHED_CLASS_GHOST */ + /** * prepare_task_switch - prepare to switch tasks * @rq: the runqueue preparing to switch @@ -4130,6 +4343,9 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, rseq_preempt(prev); fire_sched_out_preempt_notifiers(prev, next); kmap_local_sched_out(); +#ifdef CONFIG_SCHED_CLASS_GHOST + ghost_prepare_task_switch(rq, prev, next); +#endif prepare_task(next); prepare_arch_switch(next); } @@ -4242,6 +4458,14 @@ static struct rq *finish_task_switch(struct task_struct *prev) return rq; } +void schedule_callback(struct rq *rq) +{ +#ifdef CONFIG_SCHED_CLASS_GHOST + if (unlikely(ghost_need_rendezvous(rq))) + ghost_wait_for_rendezvous(rq); +#endif +} + /** * schedule_tail - first thing a freshly forked thread must call. * @prev: the thread we just switched away from. @@ -4261,6 +4485,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) */ rq = finish_task_switch(prev); + schedule_callback(rq); preempt_enable(); if (current->set_child_tid) @@ -4272,7 +4497,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) /* * context_switch - switch to the new MM and the new thread's register state. */ -static __always_inline struct rq * +noinline struct rq * context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf) { @@ -4553,6 +4778,10 @@ void scheduler_tick(void) perf_event_task_tick(); +#ifdef CONFIG_SCHED_CLASS_GHOST + ghost_tick(rq); +#endif + #ifdef CONFIG_SMP rq->idle_balance = idle_cpu(cpu); trigger_load_balance(rq); @@ -4882,6 +5111,20 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) const struct sched_class *class; struct task_struct *p; +#ifdef CONFIG_SCHED_CLASS_GHOST + rq->ghost.check_prev_preemption = ghost_produce_prev_msgs(rq, prev); + + /* a negative 'switchto_count' indicates end of the chain */ + rq->ghost.switchto_count = -rq->ghost.switchto_count; + WARN_ON_ONCE(rq->ghost.switchto_count > 0); +#endif + + /* + * FIXME(ghost): in the common case a ghost agent is always runnable + * and contributes to 'rq->nr_running'. This in turn means that the + * 'rq->nr_running == rq->cfs.h_nr_running' check below will always + * be false and disable the optimization for CFS. + */ /* * Optimization: we know that if all tasks are in the fair class we can * call that function directly, but only if the @prev task wasn't of a @@ -5083,6 +5326,7 @@ static void __sched notrace __schedule(bool preempt) __balance_callbacks(rq); raw_spin_unlock_irq(&rq->lock); } + schedule_callback(rq); } void __noreturn do_task_dead(void) @@ -5161,6 +5405,32 @@ asmlinkage __visible void __sched schedule(void) } EXPORT_SYMBOL(schedule); +#ifdef CONFIG_SCHED_CLASS_GHOST +void ghost_agent_schedule(void) +{ + const int cpu = raw_smp_processor_id(); + + /* Verify that agent is voluntarily giving up CPU. */ + VM_BUG_ON(this_rq()->ghost.agent != current); + VM_BUG_ON(current->state != TASK_RUNNING); + + VM_BUG_ON(preempt_count() != PREEMPT_DISABLE_OFFSET); + + __schedule(false); + + VM_BUG_ON(preempt_count() != PREEMPT_DISABLE_OFFSET); + VM_BUG_ON(this_rq()->ghost.blocked_in_run); + + /* + * The agent is per-cpu and must always schedule on that CPU. + * + * In other words it cannot __schedule() on one CPU and wake up + * on a different one. + */ + VM_BUG_ON(this_rq()->cpu != cpu); +} +#endif + /* * synchronize_rcu_tasks() makes sure that no task is stuck in preempted * state (have scheduled out non-voluntarily) by making sure that all @@ -5637,7 +5907,7 @@ int idle_cpu(int cpu) if (rq->curr != rq->idle) return 0; - if (rq->nr_running) + if (rq->nr_running > extra_nr_running(rq)) return 0; #ifdef CONFIG_SMP @@ -5703,6 +5973,22 @@ static void __setscheduler_params(struct task_struct *p, p->policy = policy; +#ifdef CONFIG_SCHED_CLASS_GHOST + if (ghost_policy(policy)) { + if (ghost_agent(attr)) { + /* + * This might catch an unlock in between + * ghost_setscheduler() and here. + */ + WARN_ON_ONCE(task_rq(p)->ghost.agent != p); + } + p->rt_priority = 0; + p->normal_prio = normal_prio(p); + set_load_weight(p, true); + return; + } +#endif + if (dl_policy(policy)) __setparam_dl(p, attr); else if (fair_policy(policy)) @@ -5739,6 +6025,13 @@ static void __setscheduler(struct rq *rq, struct task_struct *p, if (keep_boost) p->prio = rt_effective_prio(p, p->prio); +#ifdef CONFIG_SCHED_CLASS_GHOST + if (ghost_policy(attr->sched_policy)) { + p->sched_class = &ghost_sched_class; + return; + } +#endif + if (dl_prio(p->prio)) p->sched_class = &dl_sched_class; else if (rt_prio(p->prio)) @@ -5777,6 +6070,10 @@ static int __sched_setscheduler(struct task_struct *p, int reset_on_fork; int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; struct rq *rq; +#ifdef CONFIG_SCHED_CLASS_GHOST + struct ghost_enclave *new_e; + struct fd f_enc; +#endif /* The pi code expects interrupts enabled */ BUG_ON(pi && in_interrupt()); @@ -5795,17 +6092,26 @@ static int __sched_setscheduler(struct task_struct *p, if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV)) return -EINVAL; - /* - * Valid priorities for SCHED_FIFO and SCHED_RR are - * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, - * SCHED_BATCH and SCHED_IDLE is 0. - */ - if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || - (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) - return -EINVAL; - if ((dl_policy(policy) && !__checkparam_dl(attr)) || - (rt_policy(policy) != (attr->sched_priority != 0))) - return -EINVAL; +#ifdef CONFIG_SCHED_CLASS_GHOST + if (ghost_policy(policy)) { + retval = ghost_validate_sched_attr(attr); + if (retval) + return retval; + } else +#endif + { + /* + * Valid priorities for SCHED_FIFO and SCHED_RR are + * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, + * SCHED_BATCH and SCHED_IDLE is 0. + */ + if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || + (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) + return -EINVAL; + if ((dl_policy(policy) && !__checkparam_dl(attr)) || + (rt_policy(policy) != (attr->sched_priority != 0))) + return -EINVAL; + } /* * Allow unprivileged RT tasks to decrease priority: @@ -5909,6 +6215,15 @@ static int __sched_setscheduler(struct task_struct *p, if (attr->sched_flags & SCHED_FLAG_UTIL_CLAMP) goto change; +#ifdef CONFIG_SCHED_CLASS_GHOST + /* + * ghost_setscheduler() is the one-stop-shop for policy + * and sched_attr changes. + */ + if (ghost_policy(policy)) + goto change; +#endif + p->sched_reset_on_fork = reset_on_fork; retval = 0; goto unlock; @@ -5966,6 +6281,28 @@ static int __sched_setscheduler(struct task_struct *p, goto unlock; } +#ifdef CONFIG_SCHED_CLASS_GHOST + if (ghost_policy(policy)) { + new_e = ghost_fdget_enclave(ghost_schedattr_to_enclave_fd(attr), + &f_enc); + } else { + new_e = NULL; + } + if (ghost_policy(policy) || ghost_policy(p->policy)) { + int error = ghost_setscheduler(p, rq, attr, new_e, + &reset_on_fork); + + if (error) { + task_rq_unlock(rq, p, &rf); + if (pi) + cpuset_read_unlock(); + if (ghost_policy(policy)) + ghost_fdput_enclave(new_e, &f_enc); + return error; + } + } +#endif + p->sched_reset_on_fork = reset_on_fork; oldprio = p->prio; @@ -5978,7 +6315,12 @@ static int __sched_setscheduler(struct task_struct *p, * itself. */ new_effective_prio = rt_effective_prio(p, newprio); +#ifdef CONFIG_SCHED_CLASS_GHOST + if (!ghost_policy(policy) && !ghost_policy(oldpolicy) && + new_effective_prio == oldprio) +#else if (new_effective_prio == oldprio) +#endif queue_flags &= ~DEQUEUE_MOVE; } @@ -6023,6 +6365,11 @@ static int __sched_setscheduler(struct task_struct *p, balance_callbacks(rq, head); preempt_enable(); +#ifdef CONFIG_SCHED_CLASS_GHOST + if (ghost_policy(policy)) + ghost_fdput_enclave(new_e, &f_enc); +#endif + return 0; unlock: @@ -6538,7 +6885,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) return retval; } -static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, +int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, struct cpumask *new_mask) { if (len < cpumask_size()) @@ -7679,6 +8026,9 @@ void __init sched_init_smp(void) init_sched_rt_class(); init_sched_dl_class(); +#ifdef CONFIG_SCHED_CLASS_GHOST + init_sched_ghost_class(); +#endif sched_smp_initialized = true; } @@ -7725,11 +8075,18 @@ void __init sched_init(void) int i; /* Make sure the linker didn't screw up */ - BUG_ON(&idle_sched_class + 1 != &fair_sched_class || - &fair_sched_class + 1 != &rt_sched_class || + BUG_ON(&fair_sched_class + 1 != &rt_sched_class || &rt_sched_class + 1 != &dl_sched_class); +#ifdef CONFIG_SCHED_CLASS_GHOST + BUG_ON(&idle_sched_class + 1 != &ghost_sched_class || + &ghost_sched_class + 1 != &fair_sched_class || + &dl_sched_class + 1 != &ghost_agent_sched_class || + &ghost_agent_sched_class + 1 != &stop_sched_class); +#else + BUG_ON(&idle_sched_class + 1 != &fair_sched_class); #ifdef CONFIG_SMP BUG_ON(&dl_sched_class + 1 != &stop_sched_class); +#endif #endif wait_bit_init(); @@ -7803,6 +8160,9 @@ void __init sched_init(void) init_cfs_rq(&rq->cfs); init_rt_rq(&rq->rt); init_dl_rq(&rq->dl); +#ifdef CONFIG_SCHED_CLASS_GHOST + init_ghost_rq(&rq->ghost); +#endif #ifdef CONFIG_FAIR_GROUP_SCHED INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; @@ -7849,6 +8209,8 @@ void __init sched_init(void) INIT_LIST_HEAD(&rq->cfs_tasks); rq_attach_root(rq, &def_root_domain); + + rq->resched_ipi_work = 0; #ifdef CONFIG_NO_HZ_COMMON rq->last_blocked_load_update_tick = jiffies; atomic_set(&rq->nohz_flags, 0); @@ -9131,6 +9493,20 @@ struct cgroup_subsys cpu_cgrp_subsys = { #endif /* CONFIG_CGROUP_SCHED */ +#ifndef CONFIG_SCHED_CLASS_GHOST +SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, u32, task_barrier, + int, run_cpu, int, run_flags) +{ + return -ENOSYS; +} + +SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, u64, arg3, u64, arg4, + u64, arg5) +{ + return -ENOSYS; +} +#endif + void dump_cpu_task(int cpu) { pr_info("Task dump for CPU %d:\n", cpu); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 04a3ce20da67..ba187a863fb7 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1629,7 +1629,7 @@ static void update_numa_stats(struct task_numa_env *env, ns->nr_running += rq->cfs.h_nr_running; ns->compute_capacity += capacity_of(cpu); - if (find_idle && !rq->nr_running && idle_cpu(cpu)) { + if (find_idle && !rq_adj_nr_running(rq) && idle_cpu(cpu)) { if (READ_ONCE(rq->numa_migrate_on) || !cpumask_test_cpu(cpu, env->p->cpus_ptr)) continue; @@ -5683,7 +5683,8 @@ static struct { static unsigned long cpu_load(struct rq *rq) { - return cfs_rq_load_avg(&rq->cfs); + return cfs_rq_load_avg(&rq->cfs) + + ghost_cfs_added_load(rq); } /* @@ -5714,12 +5715,13 @@ static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) /* Discount task's util from CPU's util */ lsub_positive(&load, task_h_load(p)); - return load; + return load + ghost_cfs_added_load(rq); } static unsigned long cpu_runnable(struct rq *rq) { - return cfs_rq_runnable_avg(&rq->cfs); + return cfs_rq_runnable_avg(&rq->cfs) + + ghost_cfs_added_load(rq); } static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) @@ -5737,7 +5739,7 @@ static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) /* Discount task's runnable from CPU's runnable */ lsub_positive(&runnable, p->se.avg.runnable_avg); - return runnable; + return runnable + ghost_cfs_added_load(rq); } static unsigned long capacity_of(int cpu) @@ -6271,7 +6273,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ if (is_per_cpu_kthread(current) && prev == smp_processor_id() && - this_rq()->nr_running <= 1) { + rq_adj_nr_running(this_rq()) <= 1) { return prev; } @@ -6849,8 +6851,16 @@ static void task_dead_fair(struct task_struct *p) static int balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) { +#ifdef CONFIG_SCHED_CLASS_GHOST + if (rq_adj_nr_running(rq)) + return 1; + + if (skip_fair_idle_balance(&rq->cfs, prev)) + return 0; +#else if (rq->nr_running) return 1; +#endif return newidle_balance(rq, rf) != 0; } @@ -7140,6 +7150,10 @@ done: __maybe_unused; return p; idle: +#ifdef CONFIG_SCHED_CLASS_GHOST + if (skip_fair_idle_balance(cfs_rq, prev)) + return NULL; +#endif if (!rf) return NULL; @@ -7683,7 +7697,8 @@ static int detach_tasks(struct lb_env *env) * We don't want to steal all, otherwise we may be treated likewise, * which could at worst lead to a livelock crash. */ - if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) + if (env->idle != CPU_NOT_IDLE && + rq_adj_nr_running(env->src_rq) <= 1) break; p = list_last_entry(tasks, struct task_struct, se.group_node); @@ -8421,7 +8436,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_runnable += cpu_runnable(rq); sgs->sum_h_nr_running += rq->cfs.h_nr_running; - nr_running = rq->nr_running; + nr_running = rq_adj_nr_running(rq); sgs->sum_nr_running += nr_running; if (nr_running > 1) @@ -9623,7 +9638,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, env.src_rq = busiest; ld_moved = 0; - if (busiest->nr_running > 1) { + if (rq_adj_nr_running(busiest) > 1) { /* * Attempt to move tasks. If find_busiest_group has found * an imbalance but busiest->nr_running <= 1, the group is @@ -9631,7 +9646,8 @@ static int load_balance(int this_cpu, struct rq *this_rq, * correctly treated as an imbalance. */ env.flags |= LBF_ALL_PINNED; - env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); + env.loop_max = min(sysctl_sched_nr_migrate, + rq_adj_nr_running(busiest)); more_balance: rq_lock_irqsave(busiest, &rf); @@ -9912,7 +9928,7 @@ static int active_load_balance_cpu_stop(void *data) goto out_unlock; /* Is there any task to move? */ - if (busiest_rq->nr_running <= 1) + if (rq_adj_nr_running(busiest_rq) <= 1) goto out_unlock; /* @@ -10191,7 +10207,7 @@ static void nohz_balancer_kick(struct rq *rq) if (time_before(now, nohz.next_balance)) goto out; - if (rq->nr_running >= 2) { + if (rq_adj_nr_running(rq) >= 2) { flags = NOHZ_KICK_MASK; goto out; } @@ -10634,7 +10650,7 @@ static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) * Stop searching for tasks to pull if there are * now runnable tasks on this rq. */ - if (pulled_task || this_rq->nr_running > 0) + if (pulled_task || rq_adj_nr_running(this_rq) > 0) break; } rcu_read_unlock(); @@ -10658,7 +10674,7 @@ static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) this_rq->next_balance = next_balance; /* Is there a task of a high priority class? */ - if (this_rq->nr_running != this_rq->cfs.h_nr_running) + if (rq_adj_nr_running(this_rq) != this_rq->cfs.h_nr_running) pulled_task = -1; if (pulled_task) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c new file mode 100644 index 000000000000..68d3f94f947a --- /dev/null +++ b/kernel/sched/ghost.c @@ -0,0 +1,5157 @@ +/* + * Copyright 2021 Google LLC + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_X86_64 +#include +#endif + +#include + +#include + +#include "ghost.h" +#include "sched.h" + +/* ghost tid */ +typedef int64_t gtid_t; + +/* Lock ordering is enclave -> cpu_rsvp */ +static DEFINE_SPINLOCK(cpu_rsvp); +static struct cpumask cpus_in_enclave; /* protected by cpu_rsvp lock */ + +/* The ghost_txn pointer equals NULL or &enclave->cpu_data[this_cpu].txn */ +static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_txn *, ghost_txn); +static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); + +static DEFINE_PER_CPU(int64_t, sync_group_cookie); + +/* + * We do not want to make 'SG_COOKIE_CPU_BITS' larger than necessary so that + * we can maximize the number of times a sequence counter can increment before + * overflowing. + */ +#if (CONFIG_NR_CPUS < 2048) +#define SG_COOKIE_CPU_BITS 11 +#else +#define SG_COOKIE_CPU_BITS 14 +#endif +#define SG_COOKIE_CPU_SHIFT (63 - SG_COOKIE_CPU_BITS) /* MSB=0 */ + +/* The load contribution that CFS sees for a running ghOSt task */ +unsigned long sysctl_ghost_cfs_load_added = 1024; +int __read_mostly sysctl_ghost_wake_on_waker_cpu; +int sysctl_ghost_switchto_disable; +int __read_mostly sysctl_ghost_commit_at_tick; + +static void ghost_task_new(struct rq *rq, struct task_struct *p); +static void ghost_task_yield(struct rq *rq, struct task_struct *p); +static void ghost_task_blocked(struct rq *rq, struct task_struct *p); +static void task_dead_ghost(struct task_struct *p); +static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p); +static bool task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p); +static bool cpu_deliver_msg_tick(struct rq *rq); +static int task_target_cpu(struct task_struct *p); +static int agent_target_cpu(struct rq *rq); +static inline bool ghost_txn_ready(int cpu); +static void ghost_commit_all_greedy_txns(void); +static void ghost_commit_pending_txn(enum txn_commit_at where); +static inline int queue_decref(struct ghost_queue *q); +static void release_from_ghost(struct rq *rq, struct task_struct *p); + +struct rq *context_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next, struct rq_flags *rf); + +void schedule_callback(struct rq *rq); +struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, + struct task_struct *p, int new_cpu); + +static void __enclave_remove_task(struct ghost_enclave *e, + struct task_struct *p); +static int __sw_region_free(struct ghost_sw_region *region); +static const struct file_operations queue_fops; + +/* + * There are certain things we can't do in the scheduler while holding rq locks + * (or enclave locks, which are ordered after the rq lock), such as + * schedule_work() (which grabs a scheduler lock internally), or even kref_put + * an enclave, since enclave_release calls vfree, which grabs a lot of locks and + * can't sleep. It is safe to kref_put while holding locks if we know it's not + * the last kref. + * + * We still need to schedule_work and drop refcounts, so we need another layer + * of indirection: defer the schedule_work, which is itself a deferred work + * mechanism. The typical way to defer work from within the scheduler is to use + * a balance_callback. These can unlock and relock the RQ lock (and e->lock), + * so we can schedule_work. + * + * Why not just do the work in balance_callback? We still want to + * schedule_work, since our worker may want to block or could take a long time. + * balance_callbacks still run from within the scheduler, after all, so we + * shouldn't block or take a long time. task_reaper probably doesn't block, but + * it may take a while, so it's better to run it from a workqueue. vfree + * (called from enclave_release) may sleep, so it needs to be called from a + * workqueue. + * + * Notes: + * - Typically you must hold both RQ and e locks to muck with enclave_work: + * - Hold both locks to enqueue/dequeue enclave_work + * - Hold the RQ lock to queue_balance_callback. + * - Hold the E lock to access any field in enclave_work + * - You can schedule_work for e.g. task_reaper without using enclave_work. + * enclave_work is just a helper to schedule_work(). + * - The enclave_work can be submitted to at most one cpu's rq at a time. But + * any cpu can come along and add work to it. + * - What keeps the enclave, and thus the embedded enclave_work struct alive? + * krefs. Whoever submits enclave_work passes along a kref. In the case of + * running the task_reaper, this is done in submit_enclave_work. In the case + * of decrefs, this is done implicitly: we are passing a bunch of references + * to be put later. + * - It's highly unlikely that we'll ever have more than one enclave_work queued + * up on a cpu, since a cpu has at most one enclave at a time. + * - enclave_release itself calls schedule_work. It is possible for + * enclave_release to be called from some path other than enclave_work: from + * many other paths actually. The balance_callback allows us to safely + * schedule_work, but not vfree. So we can decref, and enclave_release + * schedules the actual freeing of the enclave. + */ + +/* + * The balance_callback. rq is locked, but we can temporarily unlock it. We + * are not preemptible. It's safe to do a raw_spin_unlock, just like + * double_lock_balance(). + */ +static void __do_enclave_work(struct rq *rq) +{ + struct enclave_work *ew; + struct ghost_enclave *e; + unsigned long irq_fl; + bool run_task_reaper; + unsigned int nr_decrefs; + + while ((ew = list_first_entry_or_null(&rq->ghost.enclave_work, + struct enclave_work, link))) { + e = container_of(ew, struct ghost_enclave, ew); + spin_lock_irqsave(&e->lock, irq_fl); + + list_del_init(&ew->link); + + run_task_reaper = ew->run_task_reaper; + ew->run_task_reaper = false; + nr_decrefs = ew->nr_decrefs; + ew->nr_decrefs = 0; + + spin_unlock_irqrestore(&e->lock, irq_fl); + /* + * At this point, all that keeps e alive is the krefs passed + * with ew. + */ + + raw_spin_unlock(&rq->lock); + + if (run_task_reaper) { + /* + * Whoever set run_task_reaper increffed. This is + * passed to task_reaper. In theory, rq and task_reaper + * could be on another cpu, and it could run and decref. + * That could be the last ref on e. This is fine - we + * only touch e if we have more krefs still. + */ + if (!schedule_work(&e->task_reaper)) + kref_put(&e->kref, enclave_release); + } + for (; nr_decrefs > 0; nr_decrefs--) + kref_put(&e->kref, enclave_release); + + raw_spin_lock(&rq->lock); + } +} + +static void submit_enclave_work(struct ghost_enclave *e, struct rq *rq, + bool run_task_reaper, int nr_decrefs) +{ + struct enclave_work *ew = &e->ew; + bool need_work = false; + + VM_BUG_ON(nr_decrefs < 0); + + lockdep_assert_held(&rq->lock); + lockdep_assert_held(&e->lock); + + if (run_task_reaper && !ew->run_task_reaper) { + kref_get(&e->kref); + ew->run_task_reaper = true; + need_work = true; + } + if (nr_decrefs) { + ew->nr_decrefs += nr_decrefs; + need_work = true; + } + if (!need_work) { + /* + * Note that if we have added no work, then we've added no + * krefs, and cannot submit ew to the rq's enclave_work list. + */ + return; + } + if (!list_empty(&ew->link)) { + /* Already queued on some rq, but maybe not this one! */ + return; + } + list_add(&ew->link, &rq->ghost.enclave_work); + /* OK to resubmit ew_head, so long as the func stays the same */ + queue_balance_callback(rq, &rq->ghost.ew_head, __do_enclave_work); +} + +void init_ghost_rq(struct ghost_rq *ghost_rq) +{ + INIT_LIST_HEAD(&ghost_rq->tasks); + INIT_LIST_HEAD(&ghost_rq->enclave_work); +} + +static inline gtid_t gtid(struct task_struct *p) +{ + return p->gtid; +} + +static inline bool on_ghost_rq(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(rq != task_rq(p)); + return !list_empty(&p->ghost.run_list); +} + +bool is_agent(struct rq *rq, struct task_struct *p) +{ + if (rq->ghost.agent == p) { + VM_BUG_ON(!p->ghost.agent); + return true; + } + + VM_BUG_ON(p->ghost.agent); + return false; +} + +static inline bool ghost_can_schedule(struct rq *rq, gtid_t gtid) +{ + const struct sched_class *class = rq->curr->sched_class; + + lockdep_assert_held(&rq->lock); + if (ghost_class(class) || class == &idle_sched_class) + return true; + + /* agent is always allowed to preempt */ + if (gtid == GHOST_AGENT_GTID) + return true; + + /* A higher priority task is running, must wait to schedule ghost. */ + return false; +} + +static inline void task_barrier_inc(struct rq *rq, struct task_struct *p) +{ + struct ghost_status_word *sw; + + lockdep_assert_held(&rq->lock); + VM_BUG_ON(rq != task_rq(p)); + + sw = p->ghost.status_word; + /* + * Order any SW flag changes before the barrier, even though the barrier + * is primarily used for making sure we don't miss messages. + */ + smp_store_release(&sw->barrier, sw->barrier + 1); +} + +static inline void agent_barrier_inc(struct rq *rq) +{ + struct ghost_status_word *sw = rq->ghost.agent->ghost.status_word; + uint32_t b; + + lockdep_assert_held(&rq->lock); + + /* We may clobber a lockless increment. See pick_agent() for details. */ + b = READ_ONCE(rq->ghost.agent_barrier) + 1; + WRITE_ONCE(rq->ghost.agent_barrier, b); + sw->barrier = b; +} + +static inline uint32_t task_barrier_get(struct task_struct *p) +{ + struct rq *rq = task_rq(p); + + lockdep_assert_held(&rq->lock); + VM_BUG_ON(is_agent(rq, p)); + VM_BUG_ON(!p->ghost.status_word); + + return p->ghost.status_word->barrier; +} + +static inline uint32_t agent_barrier_get(struct task_struct *agent) +{ + struct ghost_status_word *sw = agent->ghost.status_word; + struct rq *rq = task_rq(agent); + + lockdep_assert_held(&rq->lock); + VM_BUG_ON(!is_agent(rq, agent)); + VM_BUG_ON(!agent->ghost.status_word); + + sw->barrier = READ_ONCE(rq->ghost.agent_barrier); + return sw->barrier; +} + +static inline uint32_t barrier_get(struct task_struct *p) +{ + if (p->ghost.agent) + return agent_barrier_get(p); + return task_barrier_get(p); +} + +static inline void invalidate_cached_tasks(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + + if (unlikely(rq->ghost.latched_task == p)) + rq->ghost.latched_task = NULL; +} + +static inline bool is_cached_task(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + + return rq->ghost.latched_task == p; +} + +static inline void schedule_next(struct rq *rq, gtid_t gtid, bool resched) +{ + lockdep_assert_held(&rq->lock); + + VM_BUG_ON(gtid != GHOST_NULL_GTID && gtid != GHOST_AGENT_GTID); + + /* + * If a task running on a non-ghost CPU switches into ghost then + * this function is called via set_next_task_ghost() but without + * an agent associated with 'rq'. + * + * Regardless we still need to do resched_curr() so resist the + * temptation to do an early return. + */ + if (unlikely(!rq->ghost.agent)) + goto done; + + if (gtid == GHOST_AGENT_GTID) { + /* + * Increment barrier to guarantee that a concurrent ghost_run() + * doesn't "lose" this schedule-to-agent edge. In other words + * we want that ghost_run() to return -ESTALE. + */ + agent_barrier_inc(rq); + rq->ghost.blocked_in_run = false; + } else if (gtid == GHOST_NULL_GTID) { + rq->ghost.must_resched = true; + } + +done: + if (resched && ghost_can_schedule(rq, gtid)) + resched_curr(rq); +} + +static inline void schedule_agent(struct rq *rq, bool resched) +{ + schedule_next(rq, GHOST_AGENT_GTID, resched); +} + +static inline void force_offcpu(struct rq *rq, bool resched) +{ + VM_BUG_ON(!ghost_class(rq->curr->sched_class)); + + schedule_next(rq, GHOST_NULL_GTID, resched); +} + +static void update_curr_ghost(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + struct ghost_status_word *sw = curr->ghost.status_word; + u64 delta, now; + + /* + * Bail if this due to a "spurious" dequeue. + * + * For e.g. dequeue_task_ghost() is called when scheduling properties + * of a runnable ghost task change (e.g. nice or cpu affinity), but + * if that task is not oncpu then nothing needs to be done here. + */ + if (!ghost_class(curr->sched_class)) + return; + + VM_BUG_ON(!curr->se.exec_start); + + now = rq_clock_task(rq); + delta = now - curr->se.exec_start; + if (unlikely((s64)delta <= 0)) + return; + + curr->se.sum_exec_runtime += delta; + account_group_exec_runtime(curr, delta); + cgroup_account_cputime(curr, delta); + curr->se.exec_start = now; + + WRITE_ONCE(sw->runtime, curr->se.sum_exec_runtime); +} + +static void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) +{ + /* + * XXX produce MSG_TASK_PRIO_CHANGE into p->ghost.dst_q. + */ + ghost_wake_agent_of(p); + + /* + * Note that if a running task changes priority then + * __sched_setscheduler -> set_next_task_ghost guarantees + * a switch to the local agent. + */ +} + +static void switched_to_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_status_word *status_word = p->ghost.status_word; + + if (task_running(rq, p)) + ghost_sw_set_flag(status_word, GHOST_SW_TASK_ONCPU); + + if (task_on_rq_queued(p)) + ghost_sw_set_flag(status_word, GHOST_SW_TASK_RUNNABLE); + + WRITE_ONCE(status_word->runtime, p->se.sum_exec_runtime); + + if (is_agent(rq, p) || !task_running(rq, p)) { + ghost_task_new(rq, p); + ghost_wake_agent_of(p); /* no-op if 'p' is an agent */ + } else { + /* + * Wait for an oncpu task to schedule before advertising + * it to the agent. There isn't much the agent can do as + * long as the task is oncpu anyways. + * + * Note that if a running task switches into ghost then + * __sched_setscheduler -> set_next_task_ghost guarantees + * a context switch to the local agent at the earliest + * possible opportunity. + */ + VM_BUG_ON(p->ghost.new_task); + p->ghost.new_task = true; /* see ghost_prepare_task_switch() */ + } +} + +static void switched_from_ghost(struct rq *rq, struct task_struct *p) +{ + release_from_ghost(rq, p); + + /* + * Mark end of the switchto chain (if any) since the oncpu task + * is no longer being scheduled by ghost. + */ + if (task_current(rq, p)) { + WARN_ON_ONCE(rq->ghost.switchto_count < 0); + rq->ghost.switchto_count = 0; + } +} + +static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +{ + const bool spurious = flags & DEQUEUE_SAVE; + const bool sleeping = flags & DEQUEUE_SLEEP; + struct ghost_status_word *sw = p->ghost.status_word; + + update_curr_ghost(rq); + + BUG_ON(rq->ghost.ghost_nr_running <= 0); + rq->ghost.ghost_nr_running--; + sub_nr_running(rq, 1); + + WARN_ON_ONCE(!on_ghost_rq(rq, p)); + list_del_init(&p->ghost.run_list); + + /* + * Clear the remotely latched task when it is dequeued (but only if + * it isn't spurious). Although unlikely this can happen via the + * ghost_run()->move_queued_task() code path. + */ + if (!spurious) + invalidate_cached_tasks(rq, p); + + if (sleeping) { + WARN_ON_ONCE(!(sw->flags & GHOST_SW_TASK_RUNNABLE)); + ghost_sw_clear_flag(sw, GHOST_SW_TASK_RUNNABLE); + } + + if (is_agent(rq, p)) { + WARN_ON_ONCE(rq->ghost.blocked_in_run && !spurious); + return; + } + + if (sleeping) { + WARN_ON_ONCE(p->ghost.blocked_task); + p->ghost.blocked_task = true; + + /* + * Return to local agent if it has expressed interest in + * this edge. + * + * We don't need the full resched_curr() functionality here + * because this must be followed by pick_next_task(). + */ + if (rq->ghost.run_flags & RTLA_ON_BLOCKED) + schedule_agent(rq, false); + } +} + +static void put_prev_task_ghost(struct rq *rq, struct task_struct *p) +{ + update_curr_ghost(rq); +} + +static void +enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +{ + add_nr_running(rq, 1); + rq->ghost.ghost_nr_running++; + + WARN_ON_ONCE(on_ghost_rq(rq, p)); + list_add_tail(&p->ghost.run_list, &rq->ghost.tasks); + p->ghost.last_runnable_at = ktime_get(); + + if (flags & ENQUEUE_WAKEUP) { + WARN_ON_ONCE(is_agent(rq, p) && rq->ghost.blocked_in_run); + } +} + +static void set_next_task_ghost(struct rq *rq, struct task_struct *p, + bool first) +{ + WARN_ON_ONCE(first); + + p->se.exec_start = rq_clock_task(rq); + + if (is_agent(rq, p)) + return; + + /* + * This can happen when a running task switches into ghost on a cpu + * without an agent (not common). + */ + if (unlikely(!rq->ghost.agent)) { + force_offcpu(rq, true); + return; + } + + /* + * set_curr_task() is called when scheduling properties of a running + * task change (e.g. affinity, priority, sched_class etc). Get this + * task offcpu so the agent can incorporate these changes into its + * scheduling policy. Note that schedule_agent() below achieves two + * things: + * 1. get 'rq->curr' offcpu. + * 2. produce TASK_PREEMPTED so the agent knows 'rq->curr' got offcpu. + * + * The assumption behind return-to-local-agent is that changes to these + * properties are advertised via messages (e.g. TASK_AFFINITY etc). + * + * (another approach might be to produce a TASK_CHANGED msg here and + * let the agent figure out exactly what changed). + */ + schedule_agent(rq, true); +} + +/* + * Called from the scheduler tick handler. Returns true if everything is OK, + * false if someone has been runnable but not scheduled for more than + * e->max_unscheduled. + * + * Enclave and RQ are *unlocked*. + */ +static bool check_runnable_timeout(struct ghost_enclave *e, struct rq *rq) +{ + struct task_struct *p; + struct rq_flags rf; + ktime_t timeout; + bool ok = true; + + timeout = READ_ONCE(e->max_unscheduled); + + if (!timeout) + return ok; + + rq_lock_irqsave(rq, &rf); + + /* + * The ghost.tasks list is sorted; earliest non-zero runnable time + * first. + */ + list_for_each_entry(p, &rq->ghost.tasks, ghost.run_list) { + if (is_agent(rq, p)) + continue; + if (!p->ghost.last_runnable_at) + continue; + if (ktime_before(ktime_add_safe(p->ghost.last_runnable_at, + timeout), + ktime_get())) { + ok = false; + } + + break; + } + rq_unlock_irqrestore(rq, &rf); + + return ok; +} + +/* + * Called from the timer tick handler after dropping rq->lock. Called + * regardless of whether a ghost task is current or not. + */ +void ghost_tick(struct rq *rq) +{ + struct ghost_enclave *e; + + if (sysctl_ghost_commit_at_tick) + ghost_commit_all_greedy_txns(); + + ghost_commit_all_greedy_txns(); + + rcu_read_lock(); + e = rcu_dereference(*this_cpu_ptr(&enclave)); + if (e) { + if (!check_runnable_timeout(e, rq)) { + kref_get(&e->kref); + if (!schedule_work(&e->enclave_destroyer)) { + /* + * Safe since it is not the last reference, due + * to RCU protecting per_cpu(enclave). + */ + kref_put(&e->kref, enclave_release); + } + } + } + rcu_read_unlock(); +} + +/* + * Called from the timer tick handler while holding the rq->lock. Called only + * if a ghost task is current. + */ +void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) +{ + struct task_struct *agent = rq->ghost.agent; + + /* + * This can happen if a running task enters ghost on a CPU that + * is not associated with an agent but a timer interrupt sneaks + * in before we get the task offcpu. + */ + if (unlikely(!agent)) + return; + + update_curr_ghost(rq); + + if (cpu_deliver_msg_tick(rq)) + ghost_wake_agent_on(agent_target_cpu(rq)); +} + +static int balance_ghost(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) +{ + struct task_struct *agent = rq->ghost.agent; + + if (!agent || !agent->on_rq) + return 0; + + /* + * Try to commit a ready txn iff: + * - there is no higher priority runnable task. + * - there is no 'latched_task' + */ + if (!rq->ghost.latched_task && !rq_adj_nr_running(rq) && + ghost_txn_ready(cpu_of(rq))) { + rq_unpin_lock(rq, rf); + raw_spin_unlock(&rq->lock); + + ghost_commit_pending_txn(COMMIT_AT_SCHEDULE); + + raw_spin_lock(&rq->lock); + rq_repin_lock(rq, rf); + } + + /* + * We have something to run in 'latched_task' or a higher priority + * sched_class became runnable while the rq->lock was dropped. + */ + return rq->ghost.latched_task || rq_adj_nr_running(rq); +} + +int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) +{ + /* For anything but wake ups, just return the task_cpu */ + if (!(wake_flags & (WF_TTWU | WF_FORK))) + goto out; + + /* + * We have at least a couple of obvious choices here: + * + * 1. Keep the task on the same CPU it last ran on: + * This is good because it doesn't migrate the task unbeknownst + * to the agent but on the flip side we pay the cost of an IPI + * if the waking CPU is different than task_cpu(p). + * + * 2. Migrate the task to the waking CPU: + * This is good because we don't need an IPI to do the wakeup + * but on the flip side this may end up being a pessimistic + * choice when the agent actually schedules the task. + * + * For e.g. if the waking cpu is in a different NUMA domain + * than where the agent later schedules the task then we'll + * pay the cost of task_rq_lock() across sockets which is + * exactly what ttwu_queue_remote() aims to minimize. + * + * This can be configured using 'sysctl kernel.ghost_wake_on_waker_cpu' + * and the default is to keep the task on the same CPU it last ran on. + */ + if (sysctl_ghost_wake_on_waker_cpu) + return smp_processor_id(); +out: + return task_cpu(p); +} + +static inline bool task_is_dead(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + + VM_BUG_ON(!task_has_ghost_policy(p)); + + /* + * A ghost task always has a valid 'status_word' when it is alive + * so use that as a proxy to detect a dead task. + */ + return p->ghost.status_word == NULL; +} + +static void set_txn_state(int *ptr, enum ghost_txn_state val) +{ + if (ptr) + *ptr = val; +} + +static int validate_next_task(struct rq *rq, struct task_struct *next, + uint32_t task_barrier, int *state) +{ + lockdep_assert_held(&rq->lock); + + if (!task_has_ghost_policy(next) || next->ghost.agent) { + set_txn_state(state, GHOST_TXN_INVALID_TARGET); + return -EINVAL; + } + + /* + * ghost_run raced with task exit such that find_task_by_pid() was + * able to find the task but it was dead by the time task_rq_lock() + * returned. The most likely scenario is that agent hasn't seen the + * TASK_DEAD msg so treat this as a stale barrier. + */ + if (unlikely(task_is_dead(rq, next))) { + set_txn_state(state, GHOST_TXN_TARGET_STALE); + return -ESTALE; + } + + if (unlikely(task_barrier_get(next) != task_barrier)) { + set_txn_state(state, GHOST_TXN_TARGET_STALE); + return -ESTALE; + } + + if (!task_on_rq_queued(next)) { + set_txn_state(state, GHOST_TXN_TARGET_NOT_RUNNABLE); + return -EINVAL; + } + + return 0; +} + +int validate_next_offcpu(struct rq *rq, struct task_struct *next, int *state) +{ + lockdep_assert_held(&rq->lock); + + if (next && task_running(rq, next)) { + /* + * An agent must have attempted to preempt 'next' prior + * to scheduling it. But its possible that 'next' hasn't + * vacated its CPU yet. + */ + set_txn_state(state, GHOST_TXN_TARGET_ONCPU); + return -EAGAIN; + } + + return 0; +} + +static inline struct task_struct *pick_agent(struct rq *rq) +{ + struct task_struct *agent = rq->ghost.agent; + struct task_struct *next = NULL; + + BUG_ON(!agent || !agent->on_rq || !on_ghost_rq(rq, agent)); + + /* + * 'agent' is on_rq so 'ghost_nr_running' must be at least 1. + */ + BUG_ON(rq->ghost.ghost_nr_running < 1); + + if (READ_ONCE(rq->ghost.agent_should_wake)) { + WRITE_ONCE(rq->ghost.agent_should_wake, false); + /* + * Preemptively sync the agent_barrier, since we know our caller + * only incremented the RQ agent_barrier. + */ + (void) agent_barrier_get(agent); + rq->ghost.blocked_in_run = false; + } + + /* + * Unblock the agent if it has a signal to handle. + */ + if (unlikely(signal_pending(agent))) + rq->ghost.blocked_in_run = false; + + /* + * If the agent is not in a state where it is productively scheduling + * then try to get it there as soon as possible. This includes the + * case where the kernel has posted new messages for the agent. + */ + if (!rq->ghost.blocked_in_run) + next = agent; + + return next; +} + +static inline void ghost_prepare_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) +{ + + if (next) { + next->ghost.last_runnable_at = 0; + + WARN_ON_ONCE(!on_ghost_rq(rq, next)); + if (likely(next != prev)) { + struct ghost_status_word *sw; + + next->se.exec_start = rq_clock_task(rq); + sw = next->ghost.status_word; + WARN_ON_ONCE(sw->flags & GHOST_SW_TASK_ONCPU); + ghost_sw_set_flag(sw, GHOST_SW_TASK_ONCPU); + } else { + WARN_ON_ONCE(rq->ghost.must_resched && + next != rq->ghost.agent); + } + } +} + +/* + * Produce voluntary task state change msgs first (e.g. TASK_BLOCKED, + * TASK_YIELD in case they end up waking the local agent). + * + * Returns 'false' if 'prev' should not be considered for a preemption edge. + * + * The basic idea is to elide TASK_PREEMPTED if a voluntary task state change + * msg was already produced for 'prev' (for e.g. agents don't expect to see a + * TASK_PREEMPTED immediately after a TASK_BLOCKED). + */ +bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev) +{ + if (!task_has_ghost_policy(prev)) + return false; + + /* Who watches the watchmen? */ + if (prev == rq->ghost.agent) + return false; + + if (prev->ghost.new_task) { + prev->ghost.new_task = false; + ghost_task_new(rq, prev); + ghost_wake_agent_of(prev); + + /* + * An oncpu task can switch into ghost and yield or block + * before getting offcpu. We don't want this leaking into + * the next time this task gets oncpu (for e.g. imagine + * 'yield_task' leaking and the task blocks the next time + * it gets oncpu). + */ + prev->ghost.blocked_task = false; + prev->ghost.yield_task = false; + return false; + } + + /* + * 'prev' was running when it yielded but now that it's + * off the cpu we can safely let the agent know about it. + */ + if (prev->ghost.yield_task) { + WARN_ON_ONCE(prev->ghost.blocked_task); + prev->ghost.yield_task = false; + ghost_task_yield(rq, prev); + ghost_wake_agent_of(prev); + return false; + } + + if (prev->ghost.blocked_task) { + prev->ghost.blocked_task = false; + ghost_task_blocked(rq, prev); + ghost_wake_agent_of(prev); + return false; + } + + return true; +} + +void ghost_latched_task_preempted(struct rq *rq) +{ + struct task_struct *latched = rq->ghost.latched_task; + + WARN_ON_ONCE(!rq->ghost.agent); + + if (task_has_ghost_policy(latched)) { + ghost_task_preempted(rq, latched); + ghost_wake_agent_of(latched); + } else { + /* + * Idle task was latched and agent must have made + * other arrangements to be notified that CPU is + * no longer idle (for e.g. NEED_CPU_NOT_IDLE). + */ + WARN_ON_ONCE(latched != rq->idle); + } + rq->ghost.latched_task = NULL; +} + +static inline void ghost_update_boost_prio(struct task_struct *p, + bool preempted) +{ + if (preempted) + ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_BOOST_PRIO); + else + ghost_sw_clear_flag(p->ghost.status_word, GHOST_SW_BOOST_PRIO); +} + +static struct task_struct *pick_next_ghost_agent(struct rq *rq) +{ + struct task_struct *agent = rq->ghost.agent; + struct task_struct *next = NULL; + bool preempted; + struct task_struct *prev = rq->curr; + + if (!agent || !agent->on_rq) + goto done; + + /* + * Evaluate after produce_prev_msgs() in case it wakes up the local + * agent. + */ + next = pick_agent(rq); + + /* + * 'preempted' is true if a higher priority sched_class (e.g. CFS) + * has runnable tasks. We use it as follows: + * + * 1. indicate via GHOST_SW_BOOST_PRIO that the agent must yield + * the CPU asap. + * + * 2. produce a TASK_PREEMPTED msg on behalf of 'prev' (note that the + * TASK_PREEMPTED msg could trigger a local agent wakeup; producing + * the msg here as opposed to after pick_next_task() allows us to + * eliminate one redundant context switch from 'ghost->CFS->agent' + * to 'ghost->agent'. + */ + WARN_ON_ONCE(rq->nr_running < rq->ghost.ghost_nr_running); + preempted = (rq->nr_running > rq->ghost.ghost_nr_running); + + if (rq->ghost.check_prev_preemption) { + if (next || preempted) { + /* + * Preempted by local agent or another sched_class. + * + * Paranoia: force_offcpu guarantees that 'prev' does + * not stay oncpu after producing TASK_PREEMPTED(prev). + */ + ghost_task_preempted(rq, prev); + ghost_wake_agent_of(prev); + force_offcpu(rq, false); + rq->ghost.check_prev_preemption = false; + + /* did the preemption msg wake up the local agent? */ + if (!next) + next = pick_agent(rq); + } + } + + /* + * CPU is switching to a non-ghost task while a task is latched. + * + * Treat this like latched_task preemption because we don't know when + * the CPU will be available again so no point in keeping it latched. + */ + if (rq->ghost.latched_task && preempted) { + ghost_latched_task_preempted(rq); + + /* did the preemption msg wake up the local agent? */ + if (!next) + next = pick_agent(rq); + } + + if (!next) + goto done; + + ghost_update_boost_prio(next, preempted); + ghost_prepare_switch(rq, prev, next); + +done: + return next; +} + +static struct task_struct *pick_next_task_ghost(struct rq *rq) +{ + struct task_struct *agent = rq->ghost.agent; + struct task_struct *prev = rq->curr; + struct task_struct *next = NULL; + + /* + * We made it to ghost's pick_next_task so no need to check whether + * 'prev' was preempted by a higher priority sched_class. + * + * We prefer to use an explicit signal over checking the sched_class + * of 'next' in ghost_prepare_task_switch() because sometimes even + * higher priority sched classes can pick 'rq->idle' to run next. + * (e.g. pick_next_task_fair() does this with core tagging enabled). + */ + if (rq->ghost.switchto_count == 0) { + rq->ghost.check_prev_preemption = false; + } else { + WARN_ON_ONCE(rq->ghost.switchto_count > 0); + + /* + * We mark that the switchto chain has ended at the top of PNT + * (switchto_count < 0). Usually we will pick a different task + * (another sched_class or rq->ghost.latched_task) but this is + * not guaranteed (rq->lock can be dropped in PNT and runnable + * tasks can migrate to other cpus). + * + * We set 'must_resched' to guarantee a context switch on this + * CPU so the 'switchto_count' bookkeeping can be squared away + * via context_switch()->ghost_prepare_task_switch(). + * + * Note that if 'prev' is forced offcpu we will still produce + * a TASK_PREEMPTED(prev) courtesy of 'check_prev_preemption'. + */ + rq->ghost.must_resched = true; + + /* + * Don't reset 'check_prev_preemption' here: + * + * This trip through pick_next_task() could be due to an agent + * initiated preemption while this cpu was in a switchto chain. + * We must produce a TASK_PREEMPTED for the oncpu task since + * the agent couldn't possibly know which task was preempted. + */ + } + + /* + * 'prev' may have entered ghost on a CPU which doesn't have an agent + * or the agent is not runnable. In either case, we need to get 'prev' + * off this CPU and let an agent (remote or local) decide where and + * when to bring it back. + */ + if (!agent || !agent->on_rq) + return NULL; + + if (rq->ghost.latched_task) { + uint32_t barrier; + + next = rq->ghost.latched_task; + rq->ghost.latched_task = NULL; + + WARN_ON_ONCE(rq->curr == next); + VM_BUG_ON(task_rq(next) != rq); + + if (next == rq->idle) { + /* + * N.B. even though 'next' is the idle task this CPU + * is not "idle" (presumably the sibling is running + * a vcpu as part of a sync-group commit). Therefore + * we return 'rq->idle' directly as opposed to via + * pick_next_task_idle() and its side-effects. + */ + put_prev_task(rq, prev); + return rq->idle; + } + + /* Suppress barrier check in validate_next_task(). */ + barrier = task_barrier_get(next); + if (validate_next_task(rq, next, barrier, NULL) || + validate_next_offcpu(rq, next, NULL)) { + /* + * Welp! A previously latched and validated + * task is now failing checks. Let the local + * agent figure out what to do. + */ + pr_warn("ghost: likely leaking task %d", next->pid); + next = agent; + rq->ghost.blocked_in_run = false; + } else { + if (unlikely(rq->ghost.run_flags & NEED_L1D_FLUSH)) + kvm_register_core_conflict(cpu_of(rq)); + } + goto done; + } + + /* + * Handle a couple of unusual code paths: + * - 'prev' blocked but it was woken up before it got off the + * runqueue (see 'light' wakeup in ttwu_remote()). + * - 'prev' blocked voluntarily but __schedule() made it runnable + * to handle a pending signal. + * - cond_resched() called __schedule(preempt) but there isn't + * any higher priority task to switch to. + */ + if (task_has_ghost_policy(prev) && prev->state == TASK_RUNNING) { + /* + * When an agent blocks via ghost_run() we end up here with + * 'prev == agent' via schedule(). Without the check below + * we will simply return 'prev' (aka the agent) from this + * function and subvert the blocking in ghost_run(). + */ + if (unlikely(prev != agent && !rq->ghost.must_resched)) { + next = prev; + goto done; + } + } + +done: + if (unlikely(!next && rq->ghost.run_flags & RTLA_ON_IDLE)) { + rq->ghost.blocked_in_run = false; + return pick_next_ghost_agent(rq); + } + + ghost_prepare_switch(rq, prev, next); + rq->ghost.must_resched = false; + + return next; +} + +static void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, + int wake_flags) +{ + if (is_agent(rq, p)) { + /* + * Agents always preempt. + * + * Agents can run in the high priority ghost agent class, but + * they belong to the lower priority ghost class. For this + * reason, we have special preemption handling for ghost agents + * directly in check_preeempt_curr(). Thus, we will rarely + * hit this case in check_preempt_curr_ghost(), since + * - agents don't move across cpus + * - a ghost thread running on the cpu implies that the agent + * could not have been blocked + * One example where we _could_ hit this case is when we + * setsched a task into ghost on a cpu where the agent is + * blocked (and then wakes up before the task gets off cpu). + */ + resched_curr(rq); + } else { + /* + * The agent makes the preemption decision (if needed) + * when it notices that 'p' is runnable. + * + * The kernel can do nothing here because it does not + * implement scheduling policy. + */ + } +} + +static void yield_task_ghost(struct rq *rq) +{ + struct task_struct *curr = rq->curr; + + if (unlikely(is_agent(rq, curr))) { + /* Agent should never yield in its critical section */ + WARN_ON_ONCE(rq->ghost.blocked_in_run); + return; + } + + /* + * Task is yielding so get it offcpu. We don't need the full + * resched_curr() functionality here because sched_yield() + * calls schedule() immediately after. + */ + if (rq->ghost.run_flags & RTLA_ON_YIELD) + schedule_agent(rq, false); + else + force_offcpu(rq, false); + + /* + * Hold off on announcing that the task has yielded just yet. + * + * The agent is allowed to do a ghost_run() as soon as it sees + * the YIELD msg, but this task is oncpu without 'need_resched' + * so validate_next_task() will flag this as an error. + * + * Fix this by deferring the YIELD msg until the task is truly + * off the cpu. + * + * N.B. although 'rq->lock' is held here sched_yield() drops + * it before calling schedule() making the race with ghost_run() + * possible. + */ + WARN_ON_ONCE(curr->ghost.yield_task); + curr->ghost.yield_task = true; +} + +static void set_cpus_allowed_ghost(struct task_struct *p, + const struct cpumask *newmask, u32 flags) +{ + /* + * Agents: not allowed (rejected in __set_cpus_allowed_ptr); + * Normal ghost tasks: WARN until we figure out what to do here. + */ + WARN_ON_ONCE(1); +} + +static void task_woken_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_status_word *sw = p->ghost.status_word; + + WARN_ON_ONCE(!task_on_rq_queued(p)); + + /* + * If this is a "light" wakeup via ttwu_remote() then don't produce + * a TASK_WAKEUP msg because we haven't actually dequeued the task. + */ + if (sw->flags & GHOST_SW_TASK_RUNNABLE) + return; + + ghost_sw_set_flag(sw, GHOST_SW_TASK_RUNNABLE); + + if (unlikely(p->ghost.new_task)) { + ghost_task_new(rq, p); + p->ghost.new_task = false; + goto done; + } + task_deliver_msg_wakeup(rq, p); +done: + ghost_wake_agent_of(p); +} + +DEFINE_SCHED_CLASS(ghost) = { + .update_curr = update_curr_ghost, + .prio_changed = prio_changed_ghost, + .switched_to = switched_to_ghost, + .switched_from = switched_from_ghost, + .task_dead = task_dead_ghost, + .dequeue_task = dequeue_task_ghost, + .put_prev_task = put_prev_task_ghost, + .enqueue_task = enqueue_task_ghost, + .set_next_task = set_next_task_ghost, + .task_tick = task_tick_ghost, + .pick_next_task = pick_next_task_ghost, + .check_preempt_curr = check_preempt_curr_ghost, + .yield_task = yield_task_ghost, +#ifdef CONFIG_SMP + .balance = balance_ghost, + .select_task_rq = select_task_rq_ghost, + .task_woken = task_woken_ghost, + .set_cpus_allowed = set_cpus_allowed_ghost, +#endif +}; + +static int balance_agent(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) +{ + return 0; +} + +/* + * An interstitial sched class that operates below the stop_sched_class. It + * enables returning to an agent at the highest priority when the agent is about + * to lose its CPU to another sched class. This allows the agent to transition + * to another CPU before giving up its CPU. + */ +DEFINE_SCHED_CLASS(ghost_agent) = { + .pick_next_task = pick_next_ghost_agent, +#ifdef CONFIG_SMP + .balance = balance_agent, +#endif +}; + +/* + * Migrate 'next' (if necessary) in preparation to run it on 'cpu'. + * + * An important side-effect is that 'next' is guaranteed to not be + * cached on any cpu when this function returns (e.g. latched_task). + */ +static struct rq *ghost_move_task(struct rq *rq, struct task_struct *next, + int cpu, struct rq_flags *rf) +{ + lockdep_assert_held(&rq->lock); + lockdep_assert_held(&next->pi_lock); + + /* + * 'next' was enqueued on a different CPU than where the agent + * wants to run it now so migrate it to this runqueue before + * switching to it. + */ + if (unlikely(task_cpu(next) != cpu)) { + VM_BUG_ON(task_running(rq, next)); + VM_BUG_ON(!task_on_rq_queued(next)); + update_rq_clock(rq); + rq = move_queued_task(rq, rf, next, cpu); + } else { + invalidate_cached_tasks(rq, next); + } + + return rq; +} + +int _ghost_mmap_common(struct vm_area_struct *vma, ulong mapsize) +{ + static const struct vm_operations_struct ghost_vm_ops = {}; + + /* + * VM_MAYSHARE indicates that MAP_SHARED was set in 'mmap' flags. + * + * Checking VM_SHARED seems intuitive here but this bit is cleared + * by do_mmap() if the underlying file is readonly (as is the case + * for a sw_region file). + */ + if (!(vma->vm_flags & VM_MAYSHARE)) + return -EINVAL; + + /* + * Mappings are always readable and 'do_mmap()' ensures that + * FMODE_WRITE and VM_WRITE are coherent so the only remaining + * check is against VM_EXEC. + */ + if (vma->vm_flags & VM_EXEC) + return -EACCES; + + /* The entire region must be mapped */ + if (vma->vm_pgoff) + return -EINVAL; + + if (vma->vm_end - vma->vm_start != mapsize) + return -EINVAL; + + /* + * Don't allow mprotect(2) to relax permissions beyond what + * would have been allowed by this function. + * + * Mappings always readable and 'do_mmap()' ensures that + * FMODE_WRITE and VM_MAYWRITE are coherent so just clear + * VM_MAYEXEC here. + */ + vma->vm_flags &= ~VM_MAYEXEC; + vma->vm_flags |= VM_DONTCOPY; + + /* + * Initialize 'vma->vm_ops' to avoid vma_is_anonymous() false-positive. + */ + vma->vm_ops = &ghost_vm_ops; + return 0; +} + +/* + * Helper function for mapping status_word and similar regions into userspace. + * + * 'addr' must have been obtained from vmalloc_user(). + */ +int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, + void *addr, ulong mapsize) +{ + int error; + + error = _ghost_mmap_common(vma, mapsize); + if (!error) + error = remap_vmalloc_range(vma, addr, 0); + + return error; +} + +int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, + struct ghost_cpu_data **cpu_data, ulong mapsize) +{ + int error; + unsigned long uaddr; + + BUILD_BUG_ON(sizeof(struct ghost_cpu_data) != PAGE_SIZE); + + error = _ghost_mmap_common(vma, mapsize); + if (error) + return error; + + for (uaddr = vma->vm_start; uaddr < vma->vm_end; uaddr += PAGE_SIZE) { + error = remap_vmalloc_range_partial(vma, uaddr, *cpu_data++, + 0, PAGE_SIZE); + if (WARN_ON_ONCE(error)) { + /* + * do_mmap() will cleanup the partially populated VMA. + */ + break; + } + } + return error; +} + +struct queue_notifier { + struct ghost_agent_wakeup winfo[GHOST_MAX_WINFO]; + struct rcu_head rcu; + unsigned int wnum; /* number of valid entries in winfo[] */ +}; + +struct ghost_queue { + /* + * 'lock' protects 'refs' as well as the association between a + * message source and its queue and status_word. + * + * See go/ghost-queue-change for more details. + */ + spinlock_t lock; + ulong refs; + + struct ghost_enclave *enclave; + + /* 'ring' and 'nelems' are read-only after initialization */ + struct ghost_ring *ring; + uint32_t nelems; /* power-of-2 size of ghost_ring.msgs[] */ + + void *addr; /* + * address of vmalloc'ed region; this is + * deliberately a 'void *' instead of + * 'ghost_queue_header *' so we don't + * read it even inadvertently. + */ + + ulong mapsize; /* size of the vmalloc'ed region */ + + struct queue_notifier *notifier; /* rcu-protected agent wakeup info */ +#ifdef CONFIG_DEBUG_GHOST + int fd; /* Stash away fd for debug. */ +#endif +}; + +static inline int queue_decref(struct ghost_queue *q) +{ + ulong flags; + int error = 0; + + spin_lock_irqsave(&q->lock, flags); + if (WARN_ON_ONCE(!q->refs)) + error = -EINVAL; + else + q->refs--; + spin_unlock_irqrestore(&q->lock, flags); + return error; +} + +static inline void queue_incref(struct ghost_queue *q) +{ + ulong flags; + + spin_lock_irqsave(&q->lock, flags); + q->refs++; + spin_unlock_irqrestore(&q->lock, flags); +} + +/* Hold e->lock and the cpu_rsvp lock */ +static void __enclave_add_cpu(struct ghost_enclave *e, int cpu) +{ + struct ghost_txn *txn; + + VM_BUG_ON(!spin_is_locked(&e->lock)); + VM_BUG_ON(!spin_is_locked(&cpu_rsvp)); + + VM_BUG_ON(cpumask_test_cpu(cpu, &cpus_in_enclave)); + VM_BUG_ON(cpumask_test_cpu(cpu, &e->cpus)); + + txn = &e->cpu_data[cpu]->txn; + /* + * Reinitialize in case this txn was used by this enclave before, such + * as if a cpu was removed and readded. + */ + memset(txn, 0, sizeof(*txn)); + + txn->cpu = cpu; + txn->version = GHOST_TXN_VERSION; + txn->u.sync_group_owner = -1; + txn->state = GHOST_TXN_COMPLETE; + + cpumask_set_cpu(cpu, &cpus_in_enclave); + cpumask_set_cpu(cpu, &e->cpus); + rcu_assign_pointer(per_cpu(enclave, cpu), e); + rcu_assign_pointer(per_cpu(ghost_txn, cpu), txn); +} + +/* Hold e->lock. Caller must synchronize_rcu(). */ +static void __enclave_unpublish_cpu(struct ghost_enclave *e, int cpu) +{ + VM_BUG_ON(!spin_is_locked(&e->lock)); + + rcu_assign_pointer(per_cpu(enclave, cpu), NULL); + rcu_assign_pointer(per_cpu(ghost_txn, cpu), NULL); +} + +/* Hold e->lock and the cpu_rsvp lock. */ +static void __enclave_return_cpu(struct ghost_enclave *e, int cpu) +{ + VM_BUG_ON(!spin_is_locked(&e->lock)); + VM_BUG_ON(!spin_is_locked(&cpu_rsvp)); + + VM_BUG_ON(!cpumask_test_cpu(cpu, &cpus_in_enclave)); + VM_BUG_ON(!cpumask_test_cpu(cpu, &e->cpus)); + + cpumask_clear_cpu(cpu, &cpus_in_enclave); + cpumask_clear_cpu(cpu, &e->cpus); +} + +/* Hold e->lock and the cpu_rsvp lock. Caller must synchronize_rcu(). */ +static void __enclave_remove_cpu(struct ghost_enclave *e, int cpu) +{ + __enclave_unpublish_cpu(e, cpu); + /* cpu no longer belongs to any enclave */ + __enclave_return_cpu(e, cpu); + /* cpu can no be assigned to another enclave */ +} + +/* + * Undo anything done in ghost_create_enclave(). We also free the SW regions + * here, even though they were created at runtime. Once a swr is created, it + * persists until enclave_release. This keeps refcounting simpler. + */ +static void enclave_actual_release(struct work_struct *w) +{ + struct ghost_enclave *e = container_of(w, struct ghost_enclave, + enclave_actual_release); + struct ghost_sw_region *swr, *temp; + int cpu; + + list_for_each_entry_safe(swr, temp, &e->sw_region_list, list) + __sw_region_free(swr); + + for_each_possible_cpu(cpu) + vfree(e->cpu_data[cpu]); + + kfree(e->cpu_data); + kfree(e); +} + +void enclave_release(struct kref *k) +{ + struct ghost_enclave *e = container_of(k, struct ghost_enclave, kref); + + VM_BUG_ON(!cpumask_empty(&e->cpus)); + + /* + * enclave_release may be called from a non-sleepable context. However, + * freeing the enclave could block/sleep. vfree -> __vunmap -> + * remove_vm_area. Solve this by kicking a worker to do the actual + * cleanup. + */ + if (!schedule_work(&e->enclave_actual_release)) + WARN_ONCE(1, "enclave released twice?!?"); +} + +static void enclave_reap_tasks(struct work_struct *work) +{ + struct ghost_enclave *e = container_of(work, struct ghost_enclave, + task_reaper); + unsigned long irq_fl; + struct task_struct *t, *prev = NULL; + int ret = 0; + static const struct sched_param param = { .sched_priority = 0 }; + + spin_lock_irqsave(&e->lock, irq_fl); + WARN_ON_ONCE(!e->is_dying); + /* + * The moment we unlock, the task could exit. We can't lock the task + * either, since the lock ordering is pi_lock->e_lock. Once we let go + * of e->lock, we can't walk the list either. Assuming setscheduler + * succeeds, the task should be gone next time we look. That might not + * be true for TASK_DEAD, so we can skip those. + * + * The only error condition here is if we see the same non-dead task on + * the list after calling setscheduler. We'll abort and possibly leak + * resources, but at least we won't lock up the system. We exit the + * 'next_item' loop when: + * - the list is empty, + * - the list only contains TASK_DEAD, or + * - we abort + */ +next_item: + list_for_each_entry(t, &e->task_list, ghost.task_list) { + if (t->state == TASK_DEAD) + continue; + if (t == prev) { + WARN(1, "Reaper saw %d again, aborting!", t->pid); + break; + } + prev = t; + + get_task_struct(t); + spin_unlock_irqrestore(&e->lock, irq_fl); + + ret = sched_setscheduler_nocheck(t, SCHED_NORMAL, ¶m); + + put_task_struct(t); + spin_lock_irqsave(&e->lock, irq_fl); + + WARN_ONCE(ret && ret != -ESRCH, + "Failed to setsched pid %d, ret %d, state 0x%lx)!", + t->pid, ret, t->state); + + goto next_item; + } + + spin_unlock_irqrestore(&e->lock, irq_fl); + + kref_put(&e->kref, enclave_release); +} + +/* Helper work, which we can kick from IRQ context. */ +static void enclave_destroyer(struct work_struct *work) +{ + struct ghost_enclave *e = container_of(work, struct ghost_enclave, + enclave_destroyer); + + pr_info("Killing enclave %lu for a violation\n", e->id); + ghost_destroy_enclave(e); + kref_put(&e->kref, enclave_release); +} + +struct ghost_enclave *ghost_create_enclave(void) +{ + struct ghost_enclave *e = kzalloc(sizeof(*e), GFP_KERNEL); + bool vmalloc_failed = false; + int cpu; + + BUILD_BUG_ON(sizeof(struct ghost_cpu_data) != PAGE_SIZE); + + if (!e) + return NULL; + spin_lock_init(&e->lock); + kref_init(&e->kref); + INIT_LIST_HEAD(&e->sw_region_list); + INIT_LIST_HEAD(&e->task_list); + INIT_LIST_HEAD(&e->ew.link); + INIT_WORK(&e->task_reaper, enclave_reap_tasks); + INIT_WORK(&e->enclave_actual_release, enclave_actual_release); + INIT_WORK(&e->enclave_destroyer, enclave_destroyer); + + e->cpu_data = kcalloc(num_possible_cpus(), + sizeof(struct ghost_cpu_data *), GFP_KERNEL); + if (!e->cpu_data) { + kfree(e); + return NULL; + } + + for_each_possible_cpu(cpu) { + e->cpu_data[cpu] = vmalloc_user_node_flags(PAGE_SIZE, + cpu_to_node(cpu), + GFP_KERNEL); + if (e->cpu_data[cpu] == NULL) { + vmalloc_failed = true; + break; + } + } + + if (vmalloc_failed) { + for_each_possible_cpu(cpu) + vfree(e->cpu_data[cpu]); + kfree(e->cpu_data); + kfree(e); + return NULL; + } + + return e; +} + +/* + * Here we can trigger any destruction we want, such as killing agents, kicking + * tasks to CFS, etc. Undo things that were done at runtime. + * + * The enclave itself isn't freed until enclave_release(). + */ +void ghost_destroy_enclave(struct ghost_enclave *e) +{ + ulong flags; + int cpu; + + spin_lock_irqsave(&e->lock, flags); + if (e->is_dying) { + spin_unlock_irqrestore(&e->lock, flags); + return; + } + e->is_dying = true; + + /* + * At this point, no one can change the cpus or add new agents, since + * e->is_dying. + * + * Our goal here is to get the agents (if any) off the cpus. We could + * try kicking them to CFS, and yanking them out of rq->ghost.agent, but + * that is a little tricky. Are they blocked_in_run? Are they running + * ghost code that assumes they are the agent? + * + * Another approach is to kill them, then wait for them to finish. When + * they are done, we can return the cpu to the system. The trick is + * that we can't wait while holding locks: they could be blocked on I/O. + * They could also be *this* cpu. + * + * To handle this, we defer the cpu removal until the agent exits from + * ghost. Since our enclave is dying, we can unlock and not worry about + * any agent joining the enclave or the cpus changing. Since we don't + * give the cpu back to the system right away, we don't have to worry + * about anyone reusing the rq's agent fields. + * + * However, we *do* want to "unpublish" the assignment of the cpu to the + * enclave. This way, once we synchronize_rcu, no one can be using this + * cpu. We could defer that until release_from_ghost(), but that is not + * a convenient place to call synchronize_rcu() (holding RQ lock). + * + * In essence, __enclave_remove_cpu() is split: we unpublish the cpu + * here, then return it to the system when the agent leaves ghost. Note + * the agent could be leaving ghost of its own volition concurrently; + * the enclave lock ensures that if we see an agent, it will remove the + * cpu. And if not, we fully remove it here. We make that decision + * before we unlock in the loop below. + * + * Why do we unlock? Because send_sig grabs the pi_lock, and the lock + * ordering is pi -> rq -> e. This is safe, since the lock is no longer + * protecting invariants. The trick is that we rely on is_dying to + * prevent any of the changes to the enclave that we were concerned with + * (agent arrival or cpu changes). And once we decide that an agent + * should call __enclave_return_cpu(), that decision is set in stone. + */ + + for_each_cpu(cpu, &e->cpus) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *agent; + + if (!rq->ghost.agent) { + spin_lock(&cpu_rsvp); + __enclave_remove_cpu(e, cpu); + spin_unlock(&cpu_rsvp); + continue; + } + __enclave_unpublish_cpu(e, cpu); + rq->ghost.agent_remove_enclave_cpu = true; + /* + * We can't force_sig(). The task might be exiting and losing + * its sighand_struct. send_sig_info() checks for that. We + * also can't hold the enclave lock. + * + * As soon as we unlock, the agent could exit on its own too. + * We still need to poke it to make sure it dies, and since it + * might be dying on its own, we need to get a refcount while we + * poke it. + */ + agent = rq->ghost.agent; + get_task_struct(agent); + spin_unlock_irqrestore(&e->lock, flags); + send_sig_info(SIGKILL, SEND_SIG_PRIV, agent); + put_task_struct(agent); + spin_lock_irqsave(&e->lock, flags); + } + + spin_unlock_irqrestore(&e->lock, flags); + + kref_get(&e->kref); /* Reaper gets a kref */ + if (!schedule_work(&e->task_reaper)) + kref_put(&e->kref, enclave_release); + + synchronize_rcu(); /* Required after unpublishing a cpu */ + + /* + * Removes the enclave and all of its files from ghostfs. There may + * still be open FDs, each of which holds a reference. When the last FD + * is closed, we'll release and free. + */ + ghostfs_remove_enclave(e); + + kref_put(&e->kref, enclave_release); +} + +/* + * Sets the cpus of the enclave to `cpus`, adding and removing as necessary. + * Returns an error code on failure and does not change the enclave. Will fail + * if a cpu is in another enclave. + */ +int ghost_enclave_set_cpus(struct ghost_enclave *e, const struct cpumask *cpus) +{ + unsigned long flags; + unsigned int cpu; + int ret = 0; + cpumask_var_t add, del; + + if (!alloc_cpumask_var(&add, GFP_KERNEL)) + return -ENOMEM; + if (!alloc_cpumask_var(&del, GFP_KERNEL)) { + free_cpumask_var(add); + return -ENOMEM; + } + + spin_lock_irqsave(&e->lock, flags); + + if (e->is_dying) { + ret = -EXDEV; + goto out_e; + } + + cpumask_andnot(add, cpus, &e->cpus); + cpumask_andnot(del, &e->cpus, cpus); + + for_each_cpu(cpu, del) { + /* + * No need for an rq lock to read ghost.agent. Both the enclave + * and the rq locks are required to *write* rq->ghost.agent. + */ + if (cpu_rq(cpu)->ghost.agent) { + ret = -EEXIST; + goto out_e; + } + } + + spin_lock(&cpu_rsvp); + + if (cpumask_intersects(add, &cpus_in_enclave)) { + ret = -EBUSY; + goto out_rsvp; + } + for_each_cpu(cpu, add) { + if (!cpu_online(cpu)) { + ret = -ENODEV; + goto out_rsvp; + } + } + + for_each_cpu(cpu, add) + __enclave_add_cpu(e, cpu); + for_each_cpu(cpu, del) + __enclave_remove_cpu(e, cpu); + +out_rsvp: + spin_unlock(&cpu_rsvp); +out_e: + spin_unlock_irqrestore(&e->lock, flags); + + synchronize_rcu(); /* Required after unpublishing a cpu */ + + free_cpumask_var(add); + free_cpumask_var(del); + + return ret; +} + +void __init init_sched_ghost_class(void) +{ + int64_t cpu; + + for_each_possible_cpu(cpu) + per_cpu(sync_group_cookie, cpu) = cpu << SG_COOKIE_CPU_SHIFT; +} + +/* + * Helper to get an fd and translate it to an enclave. Ghostfs will maintain a + * kref on the enclave. That kref is increffed when the file was opened, and + * that kref is maintained by this call (a combination of fdget and + * kernfs_get_active). Once you call ghost_fdput_enclave(), the enclave kref + * can be released. + * + * Returns NULL if there is not a ghost_enclave for this FD, which could be due + * to concurrent enclave destruction. + * + * Caller must call ghost_fdput_enclave with e and &fd_to_put, even on error. + */ +struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) +{ + *fd_to_put = fdget(fd); + if (!fd_to_put->file) + return NULL; + return ghostfs_ctl_to_enclave(fd_to_put->file); +} + +/* + * Pairs with calls to ghost_fdget_enclave(). You can't call this while holding + * the rq lock. + */ +void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put) +{ + if (e) + ghostfs_put_enclave_ctl(fd_to_put->file); + fdput(*fd_to_put); +} + +static struct ghost_queue *fd_to_queue(struct fd f) +{ + if (!f.file) + return NULL; + if (f.file->f_op != &queue_fops) + return NULL; + return f.file->private_data; +} + +static void enclave_add_task(struct ghost_enclave *e, struct task_struct *p) +{ + ulong flags; + + if (p->ghost.agent) + return; + spin_lock_irqsave(&e->lock, flags); + if (e->is_dying) { + /* + * The task entered as the enclave was dying, and likely will + * miss the reaper. + */ + submit_enclave_work(e, task_rq(p), /*run_task_reaper=*/true, 0); + } + WARN_ON_ONCE(!list_empty(&p->ghost.task_list)); + list_add_tail(&p->ghost.task_list, &e->task_list); + spin_unlock_irqrestore(&e->lock, flags); +} + +/* Hold the enclave lock */ +static void __enclave_remove_task(struct ghost_enclave *e, + struct task_struct *p) +{ + lockdep_assert_held(&e->lock); + + if (p->ghost.agent) + return; + WARN_ON_ONCE(list_empty(&p->ghost.task_list)); + list_del_init(&p->ghost.task_list); +} + +static void enclave_add_sw_region(struct ghost_enclave *e, + struct ghost_sw_region *region) +{ + ulong flags; + + spin_lock_irqsave(&e->lock, flags); + list_add_tail(®ion->list, &e->sw_region_list); + spin_unlock_irqrestore(&e->lock, flags); +} + +static inline struct ghost_status_word * +first_status_word(struct ghost_sw_region_header *h) +{ + return (struct ghost_status_word *)((char *)h + h->start); +} + +static inline bool status_word_inuse(struct ghost_status_word *sw) +{ + return sw->flags & GHOST_SW_F_INUSE; +} + +static inline bool status_word_canfree(struct ghost_status_word *sw) +{ + return sw->flags & GHOST_SW_F_CANFREE; +} + +static int free_status_word_locked(struct ghost_enclave *e, + struct ghost_status_word *sw) +{ + int error = -ENOENT; + struct ghost_sw_region *region; + struct ghost_sw_region_header *header; + struct ghost_status_word *first, *limit; + + VM_BUG_ON(!spin_is_locked(&e->lock)); + list_for_each_entry(region, &e->sw_region_list, list) { + header = region->header; + first = first_status_word(header); + limit = &first[header->capacity]; + if (sw >= first && sw < limit) { + if (!status_word_inuse(sw)) { + /* + * Trying to free an already free status word? + */ + error = -EALREADY; + } else { + memset(sw, 0, sizeof(struct ghost_status_word)); + header->available++; + error = 0; /* success */ + } + break; + } + } + return error; +} + +static struct ghost_status_word *lookup_status_word_locked( + struct ghost_enclave *e, uint32_t id, uint32_t index) +{ + struct ghost_sw_region *region; + struct ghost_sw_region_header *header; + struct ghost_status_word *sw = NULL; + + VM_BUG_ON(!spin_is_locked(&e->lock)); + list_for_each_entry(region, &e->sw_region_list, list) { + header = region->header; + if (header->id == id) { + if (index < header->capacity) + sw = first_status_word(header) + index; + break; + } + } + return sw; +} + +static struct ghost_status_word * +alloc_status_word_locked(struct ghost_enclave *e, uint64_t gtid) +{ + struct ghost_sw_region *region; + struct ghost_sw_region_header *header = NULL; + struct ghost_status_word *sw, *found = NULL; + uint32_t n; + + VM_BUG_ON(!spin_is_locked(&e->lock)); + + /* Look for a sw_region that has at least one free status_word. */ + list_for_each_entry(region, &e->sw_region_list, list) { + if (region->header->available) { + header = region->header; /* got one */ + break; + } + } + + /* No free slots available in any status_word region. */ + if (!header) + goto done; + + sw = first_status_word(header); + for (n = 0; n < header->capacity; n++) { + found = &sw[region->alloc_scan_start++]; + /* Wrap around to the start if we are past the end. */ + if (region->alloc_scan_start >= header->capacity) + region->alloc_scan_start = 0; + + /* + * If this slot is free then claim it and adjust 'available'. + */ + if (!status_word_inuse(found)) { + found->flags = GHOST_SW_F_INUSE; + found->barrier = 0; + found->gtid = gtid; + found->runtime = 0; + header->available--; + break; + } + } + VM_BUG_ON(n >= header->capacity); +done: + return found; +} + +int64_t ghost_alloc_gtid(struct task_struct *p) +{ + gtid_t gtid; + uint64_t seqnum; + + static atomic64_t gtid_seqnum = ATOMIC64_INIT(0); + + BUILD_BUG_ON(PID_MAX_LIMIT > (1UL << GHOST_TID_PID_BITS)); + + do { + seqnum = atomic64_add_return(1, >id_seqnum) & + ((1UL << GHOST_TID_SEQNUM_BITS) - 1); + } while (!seqnum); + + gtid = ((gtid_t)p->pid << GHOST_TID_SEQNUM_BITS) | seqnum; + WARN_ON_ONCE(gtid <= 0); + return gtid; +} + +/* + * Prepare task 'p' to participate in ghost scheduling. + * + * Hold the enclave lock. + * + * The underlying 'task_struct' is stable because: + * - it is protected by 'p->pi_lock' (called via sched_setscheduler). + * - it is being created (called via sched_fork). + */ +static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, + struct ghost_queue *q, bool forked) +{ + struct ghost_status_word *sw; + int error = 0; + + WARN_ON_ONCE(p->ghost.status_word); + WARN_ON_ONCE(p->ghost.dst_q); + WARN_ON_ONCE(p->ghost.enclave); + + lockdep_assert_held(&e->lock); + + /* + * Clean up state from a previous incarnation (e.g. ghost->cfs->ghost). + */ + sched_ghost_entity_init(p); + + /* + * This prevents us from joining a dying enclave and grabbing the + * enclave's kref. However, it is possible for the enclave to die + * before we get enqueued (see uses of `new_task` and ghost_task_new()). + */ + if (e->is_dying) { + error = -EXDEV; + goto done; + } + if (!q) { + /* + * Associate task with the default queue. Agents typically pass + * in their own queue. + */ + q = e->def_q; + } + if (!q) { + error = -ENXIO; + goto done; + } + queue_incref(q); + p->ghost.dst_q = q; + + /* Allocate a status_word */ + sw = alloc_status_word_locked(e, gtid(p)); + if (sw == NULL) { + error = -ENOMEM; + goto done; + } + kref_get(&e->kref); + p->ghost.enclave = e; + p->ghost.status_word = sw; + p->ghost.new_task = forked; + +done: + if (error) { + if (p->ghost.dst_q) { + queue_decref(p->ghost.dst_q); + p->ghost.dst_q = NULL; + } + + if (p->ghost.status_word) { + free_status_word_locked(e, p->ghost.status_word); + p->ghost.status_word = NULL; + } + } + return error; +} + +/* Only called from enclave_release. */ +static int __sw_region_free(struct ghost_sw_region *swr) +{ + struct ghost_sw_region_header *h = swr->header; + + list_del(&swr->list); + + /* + * There cannot be any references to 'region' from either + * userspace or kernel so we don't need to take the lock. + */ + + if (h->available != h->capacity) { + /* + * XXX agent closed the file descriptor (voluntary or crashed) + * but there are message sources that point into it from the + * kernel. + * + * Until we understand this better just let it leak. + */ + WARN_ONCE(1, "%s: id/%u capacity/%u and avail/%d", + __func__, h->id, h->capacity, h->available); + } else { + vfree(h); + } + kfree(swr); + + /* XXX memcg uncharge */ + + return 0; +} + +static size_t calculate_sw_region_size(void) +{ + /* + * Arbitrary number, but if we subtract off the swr header, we're + * likely to get close to the number we want when we PAGE_ALIGN. + */ + #define GHOST_NR_SW_ELEMS (65536 - \ + DIV_ROUND_UP(sizeof(struct ghost_sw_region_header), \ + sizeof(struct ghost_status_word))) + size_t size; + + size = sizeof(struct ghost_sw_region_header); + size += GHOST_NR_SW_ELEMS * sizeof(struct ghost_status_word); + size = PAGE_ALIGN(size); + + return size; +} + +/* + * Returns a pointer to the region. Once created, a SW region will not be + * removed until the enclave is freed. + * + * Returns an ERR_PTR on error. + */ +struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, + unsigned int id, + unsigned int node) +{ + struct ghost_sw_region *swr; + struct ghost_sw_region_header *h; + + if (node < 0 || node >= nr_node_ids || !node_online(node)) + return ERR_PTR(-ENODEV); + + swr = kzalloc_node(sizeof(struct ghost_sw_region), GFP_KERNEL, node); + if (!swr) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&swr->list); + swr->mmap_sz = calculate_sw_region_size(); + swr->header = vmalloc_user_node_flags(swr->mmap_sz, node, GFP_KERNEL); + if (!swr->header) { + kfree(swr); + return ERR_PTR(-ENOMEM); + } + + h = swr->header; + h->id = id; + h->numa_node = node; + h->version = GHOST_SW_REGION_VERSION; + h->start = sizeof(struct ghost_sw_region_header); + h->capacity = (swr->mmap_sz - h->start) / + sizeof(struct ghost_status_word); + h->available = h->capacity; + + enclave_add_sw_region(e, swr); + swr->enclave = e; /* weak reference */ + + /* XXX memcg charge */ + + return swr; +} + +static int ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, + bool forked) +{ + int error; + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + error = __ghost_prep_task(e, p, /*q=*/NULL, forked); + spin_unlock_irqrestore(&e->lock, irq_fl); + + return error; +} + +/* + * 'p' and 'rq' are both locked. + */ +static int ghost_prep_agent(struct ghost_enclave *e, struct task_struct *p, + struct rq *rq, struct ghost_queue *q) +{ + int ret = 0; + + lockdep_assert_held(&rq->lock); + lockdep_assert_held(&p->pi_lock); + + WARN_ON_ONCE(!capable(CAP_SYS_NICE)); + + /* + * Bail if the task wants to be an agent but there is already + * an agent associated with this runqueue. + */ + if (rq->ghost.agent != NULL) + return -EBUSY; + + /* We hold the rq lock, which is already irqsaved. */ + spin_lock(&e->lock); + + if (!cpumask_test_cpu(rq->cpu, &e->cpus)) { + ret = -ENODEV; + goto out; + } + + ret = __ghost_prep_task(e, p, q, /*forked=*/false); + if (ret) + goto out; + + p->ghost.agent = true; + ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_TASK_IS_AGENT); + + /* + * The agent is not in the ghost class yet, but it will be before the RQ + * lock is unlocked. + * + * We set this now so that if you hold the enclave lock and a cpu + * belongs to the enclave, then you can *inspect* that cpu's + * rq->ghost.agent. Setting or clearing it requires both the enclave + * lock and the RQ lock. + */ + WARN_ON(rq->ghost.agent || rq->ghost.blocked_in_run); + rq->ghost.agent = p; + +out: + spin_unlock(&e->lock); + return ret; +} + +int ghost_sched_fork(struct task_struct *p) +{ + struct rq *rq; + struct rq_flags rf; + int ret; + struct ghost_enclave *parent_enclave; + + VM_BUG_ON(!task_has_ghost_policy(p)); + + /* + * Another task could be attempting to setsched current out of ghOSt. + * To keep current's enclave valid, we synchronize with the RQ lock. + */ + rq = task_rq_lock(current, &rf); + if (!ghost_policy(current->policy)) { + task_rq_unlock(rq, current, &rf); + /* It's not quite ECHILD, but it'll tell us where to look. */ + return -ECHILD; + } + parent_enclave = current->ghost.enclave; + VM_BUG_ON(!parent_enclave); + ret = ghost_prep_task(parent_enclave, p, true); + task_rq_unlock(rq, current, &rf); + + return ret; +} + +void ghost_sched_cleanup_fork(struct task_struct *p) +{ + struct ghost_queue *q; + struct ghost_status_word *sw; + struct ghost_enclave *e = p->ghost.enclave; + ulong flags; + int error; + + VM_BUG_ON(!task_has_ghost_policy(p)); + + /* + * Clear association between 'p' and the default queue. + */ + q = p->ghost.dst_q; + if (q != NULL) { + p->ghost.dst_q = NULL; + error = queue_decref(q); + VM_BUG_ON(error); + } + + sw = p->ghost.status_word; + if (sw != NULL) { + VM_BUG_ON(!e); + p->ghost.status_word = NULL; + spin_lock_irqsave(&e->lock, flags); + error = free_status_word_locked(e, sw); + spin_unlock_irqrestore(&e->lock, flags); + VM_BUG_ON(error); + } + if (e) { + kref_put(&e->kref, enclave_release); + p->ghost.enclave = NULL; + } +} + +/* + * For tasks attempting to join ghost, __sched_setscheduler() needs to pass us + * the enclave, and it needs to manage the reference counting on the fd and its + * underlying kn. This is because we can't call ghost_fdput_enclave directly + * while holding the rq lock, because it eventually calls kernfs_put_active, + * which grabs the rq lock. + * + * You'd think we could use a balance_callback, and pass it the fd_to_put + * (f_enc) and whether or not it had an enclave. However, there's no good place + * to put those arguments. We could stash them in struct ghost_rq, but for one + * small problem: the rq lock is released before balance_callback() runs. That + * means someone else could grab the lock, then setsched another task, thereby + * clobbering the args stored in ghost_rq. + * + * We can't dynamically allocate memory either, since we're holding the rq lock. + * I even considered reusing the sched_attr struct: cast it to some other struct + * and hang it off a linked list on the rq. The problem there is that although + * we will call balance_callback when we return to __sched_setscheduler(), + * balance_callback makes no guarantees about when the callback will run. If + * two threads call balance_callback(), one of them will run the callbacks and + * the other will return immediately. If we return immediately, then we can't + * use the schedattr. + * + * The most reasonable fix for all of this is to directly call + * ghost_fdput_enclave() from __sched_setscheduler(). + */ +int ghost_setscheduler(struct task_struct *p, struct rq *rq, + const struct sched_attr *attr, + struct ghost_enclave *new_e, + int *reset_on_fork) +{ + int oldpolicy = p->policy; + int newpolicy = attr->sched_policy; + int ret; + + if (WARN_ON_ONCE(!ghost_policy(oldpolicy) && !ghost_policy(newpolicy))) + return -EINVAL; + + /* + * If the process is dying, finish_task_switch will call task_dead + * *after* releasing the rq lock. We don't know if task_dead was called + * yet, and it will be called without holding any locks. This can break + * ghost for both task scheduling into ghost and out of ghost. + * - If we're entering ghost, but already ran task_dead from our old + * sched class, then we'll never run ghost_task_dead. + * - If we're leaving ghost, we need to run either ghost_task_dead xor + * setscheduler from ghost, but we have no nice way of knowing if we + * already ran ghost_task_dead. + */ + if (p->state == TASK_DEAD) + return -ESRCH; + /* Cannot change attributes for a ghost task after creation. */ + if (oldpolicy == newpolicy) + return -EPERM; + + /* Task 'p' is departing the ghost sched class. */ + if (ghost_policy(oldpolicy)) { + /* + * Don't allow an agent to move out of the ghost sched_class. + * There is no use case for this and more importantly we don't + * apply the side-effects of agent leaving the CPU (compare to + * task_dead_ghost() where we do). + */ + if (p->ghost.agent) + return -EPERM; + + invalidate_cached_tasks(rq, p); + return 0; + } + + if (!new_e) + return -EBADF; + + if (ghost_agent(attr)) { + int qfd = ghost_schedattr_to_queue_fd(attr); + struct fd f_que = {0}; + struct ghost_queue *q = NULL; + + if (qfd != -1) { + f_que = fdget(qfd); + q = fd_to_queue(f_que); + if (!q) { + fdput(f_que); + return -EBADF; + } + } + + /* It's OK to set reset_on_fork even if we fail. */ + *reset_on_fork = 1; + ret = ghost_prep_agent(new_e, p, rq, q); + + if (qfd != -1) + fdput(f_que); + } else { + ret = ghost_prep_task(new_e, p, false); + } + + return ret; +} + +int ghost_validate_sched_attr(const struct sched_attr *attr) +{ + /* + * A thread can only make a task an agent if the thread has the + * CAP_SYS_NICE capability. + */ + switch (attr->sched_priority) { + case GHOST_SCHED_TASK_PRIO: + return 0; + case GHOST_SCHED_AGENT_PRIO: + return capable(CAP_SYS_NICE) ? 0 : -EPERM; + default: + return -EINVAL; + } +} + +bool ghost_agent(const struct sched_attr *attr) +{ + return attr->sched_priority == GHOST_SCHED_AGENT_PRIO; +} + +/* Makes newq the default for e. */ +static void enclave_set_default_queue(struct ghost_enclave *e, + struct ghost_queue *newq) +{ + unsigned long irq_fl; + + /* + * All queues belong to an enclave, and they can only be default for + * their enclave. + */ + if (WARN_ON(newq->enclave != e)) + return; + + spin_lock_irqsave(&e->lock, irq_fl); + /* + * def_q is a weak (uncounted) reference. We explicitly clear it in + * queue_release when q no longer tracks this enclave. It is protected + * by the enclave lock. + */ + e->def_q = newq; + spin_unlock_irqrestore(&e->lock, irq_fl); +} + +/* Unsets q as default, if it was default. */ +static void enclave_maybe_del_default_queue(struct ghost_enclave *e, + struct ghost_queue *q) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->def_q == q) + e->def_q = NULL; + spin_unlock_irqrestore(&e->lock, irq_fl); +} + +static int queue_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct ghost_queue *q = file->private_data; + + return ghost_region_mmap(file, vma, q->addr, q->mapsize); +} + +static int queue_release(struct inode *inode, struct file *file) +{ + struct ghost_queue *q = file->private_data; + struct ghost_enclave *e = q->enclave; + ulong flags, refs; + + enclave_maybe_del_default_queue(e, q); + q->enclave = NULL; + kref_put(&e->kref, enclave_release); + + spin_lock_irqsave(&q->lock, flags); + refs = q->refs; + spin_unlock_irqrestore(&q->lock, flags); + + if (refs) { + /* + * XXX we could flag that this queue is not reachable + * from userspace anymore and can be freed when the + * last reference is dropped (e.g. when all referencing + * tasks die or change association). + * + * Just leak memory for now. + */ + WARN_ONCE(1, "%s: leaking queue with %lu refs", __func__, refs); + } else { + vfree(q->addr); + kfree(q->notifier); + kfree(q); + /* XXX memcg uncharge */ + } + return 0; +} + +static const struct file_operations queue_fops = { + .release = queue_release, + .mmap = queue_mmap, +}; + +static int ghost_create_queue(int elems, int node, int flags, + ulong __user *mapsize, int e_fd) +{ + ulong size; + int error = 0, fd; + struct ghost_queue *q; + struct ghost_queue_header *h; + struct ghost_enclave *e; + struct fd f_enc = {0}; + + const int valid_flags = 0; /* no flags for now */ + + /* + * Validate that 'head' and 'tail' are large enough to distinguish + * between an empty and full queue. In other words when the queue + * goes from empty to full we want to guarantee that 'head' will + * not rollover 'tail'. + */ + BUILD_BUG_ON( + GHOST_MAX_QUEUE_ELEMS >= + ((typeof(((struct ghost_ring *)0)->head))~0UL) + ); + + if (elems < 0 || elems > GHOST_MAX_QUEUE_ELEMS || !is_power_of_2(elems)) + return -EINVAL; + + if (flags & ~valid_flags) + return -EINVAL; + + if (node < 0 || node >= nr_node_ids || !node_online(node)) + return -EINVAL; + + size = sizeof(struct ghost_queue_header) + sizeof(struct ghost_ring); + size += elems * sizeof(struct ghost_msg); + size = PAGE_ALIGN(size); + + error = put_user(size, mapsize); + if (error) + return error; + + e = ghost_fdget_enclave(e_fd, &f_enc); + if (!e) { + error = -EBADF; + goto err_enc_fd; + } + + q = kzalloc_node(sizeof(struct ghost_queue), GFP_KERNEL, node); + if (!q) { + error = -ENOMEM; + goto err_alloc_queue; + } + + spin_lock_init(&q->lock); + q->addr = vmalloc_user_node_flags(size, node, GFP_KERNEL); + if (!q->addr) { + error = -ENOMEM; + goto err_vmalloc; + } + + h = q->addr; + h->version = GHOST_QUEUE_VERSION; + h->start = sizeof(struct ghost_queue_header); + h->nelems = elems; + + /* + * The queue mapping is writeable so we cannot trust anything + * in the header after it is mapped by the agent. + * + * Stash a pointer to the ring and number of elements below. + */ + q->ring = (struct ghost_ring *)((char *)h + h->start); + q->nelems = h->nelems; + q->mapsize = size; + + fd = anon_inode_getfd("[ghost_queue]", &queue_fops, q, + O_RDWR | O_CLOEXEC); + if (fd < 0) { + error = fd; + goto err_getfd; + } +#ifdef CONFIG_DEBUG_GHOST + q->fd = fd; +#endif + kref_get(&e->kref); + q->enclave = e; + + /* XXX memcg charge */ + ghost_fdput_enclave(e, &f_enc); + return fd; + +err_getfd: + vfree(q->addr); +err_vmalloc: + kfree(q); +err_alloc_queue: +err_enc_fd: + ghost_fdput_enclave(e, &f_enc); + return error; +} + +static struct task_struct *find_task_by_gtid(gtid_t gtid) +{ + struct task_struct *p; + pid_t pid = gtid >> GHOST_TID_SEQNUM_BITS; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "find_task_by_gtid() needs rcu_read_lock protection"); + + if (gtid < 0) + return NULL; + + if (gtid == 0) + return current; + + p = find_task_by_pid_ns(pid, &init_pid_ns); + if (!p) + return NULL; + + /* Make sure this really is the process we are looking for. */ + if (p->gtid != gtid) + return NULL; + + return p; +} + +static int _get_sw_info(struct ghost_enclave *e, + const struct ghost_status_word *sw, + struct ghost_sw_info *info) +{ + ulong flags; + int error = -ENOENT; + struct ghost_sw_region *region; + struct ghost_sw_region_header *header; + struct ghost_status_word *first, *limit; + + spin_lock_irqsave(&e->lock, flags); + list_for_each_entry(region, &e->sw_region_list, list) { + header = region->header; + first = first_status_word(header); + limit = &first[header->capacity]; + if (sw >= first && sw < limit) { + info->id = header->id; + info->index = sw - first; + error = 0; /* success */ + break; + } + } + spin_unlock_irqrestore(&e->lock, flags); + return error; +} + +int ghost_sw_get_info(struct ghost_enclave *e, + struct ghost_ioc_sw_get_info __user *arg) +{ + int error = -EINVAL; + struct task_struct *p; + struct ghost_msg_src src; + struct ghost_sw_info info = { 0 }; + + if (copy_from_user(&src, &arg->request, sizeof(struct ghost_msg_src))) + return -EFAULT; + + if (src.type == GHOST_AGENT) { + int want_cpu = src.arg, curr_cpu = get_cpu(); + struct rq *rq = this_rq(); + struct rq_flags rf; + + /* + * For now we restrict an agent to only query its + * own status_word. + */ + if (want_cpu != GHOST_THIS_CPU && want_cpu != curr_cpu) { + put_cpu(); + error = -ENOENT; + goto done; + } + + rq_lock_irq(rq, &rf); + if (is_agent(rq, current)) { + error = _get_sw_info(e, current->ghost.status_word, + &info); + } + rq_unlock_irq(rq, &rf); + put_cpu(); + } else if (src.type == GHOST_TASK) { + rcu_read_lock(); + p = find_task_by_gtid(src.arg); + if (p && p->ghost.status_word) + error = _get_sw_info(e, p->ghost.status_word, &info); + else + error = -ENOENT; + rcu_read_unlock(); + } + + if (!error) { + if (copy_to_user(&arg->response, &info, + sizeof(struct ghost_sw_info))) + error = -EFAULT; + } + +done: + return error; +} + +int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo) +{ + int error; + ulong flags; + struct ghost_status_word *sw; + struct ghost_sw_info info = { 0 }; + + if (copy_from_user(&info, uinfo, sizeof(struct ghost_sw_info))) + return -EFAULT; + + spin_lock_irqsave(&e->lock, flags); + + sw = lookup_status_word_locked(e, info.id, info.index); + if (sw == NULL) { + error = -ENOENT; + goto done; + } + if (!status_word_canfree(sw)) { + error = -EINVAL; + goto done; + } + error = free_status_word_locked(e, sw); +done: + spin_unlock_irqrestore(&e->lock, flags); + return error; +} + +static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, + int barrier, int flags, int *ustatus) +{ + int error = 0; + struct rq *rq; + struct file *file; + struct rq_flags rf; + struct ghost_msg_src src; + struct task_struct *p = NULL; + struct ghost_status_word *sw; + struct ghost_queue *oldq, *newq; + int status = 0; + + if (flags != 0) /* no flags for now */ + return -EINVAL; + + if (copy_from_user(&src, usrc, sizeof(struct ghost_msg_src))) + return -EFAULT; + + /* For now only allow changing task-to-queue association. */ + if (src.type != GHOST_TASK) + return -EINVAL; + + file = fget(fd); + if (!file) + return -EBADF; + + if (file->f_op != &queue_fops) { + error = -EBADF; + goto done; + } + + newq = file->private_data; + + rcu_read_lock(); + p = find_task_by_gtid(src.arg); + if (!p) { + rcu_read_unlock(); + error = -ENOENT; + goto done; + } + + /* Serialize with sched_setscheduler(), clone() and exit() */ + rq = task_rq_lock(p, &rf); + rcu_read_unlock(); + + oldq = p->ghost.dst_q; + sw = p->ghost.status_word; + if (unlikely(!oldq || !sw)) { + /* Task is dead or switched to another sched_class */ + WARN_ON_ONCE(p->state != TASK_DEAD && + ghost_class(p->sched_class)); + error = -ENOENT; + goto done; + } + + if (barrier_get(p) != barrier) { + error = -ESTALE; + goto done; + } + + /* + * Associating to the preexisting queue would be a noop, but telling + * userspace helps with in-place upgrade. In particular, the agent + * knows whether or not it may have received messages for a task that + * joined the enclave after the agent set its default queue. + */ + if (oldq == newq) + status |= GHOST_ASSOC_SF_ALREADY; + /* + * If a running task was setsched into ghost by a third party, and it + * hasn't blocked or been preempted yet, then its TASK_NEW has not been + * sent. That task may or may not have an assigned queue, so this is + * different than detecting oldq == newq. + */ + if (p->ghost.new_task) + status |= GHOST_ASSOC_SF_BRAND_NEW; + + if (ustatus && copy_to_user(ustatus, &status, sizeof(status))) { + error = -EFAULT; + goto done; + } + + queue_incref(newq); + p->ghost.dst_q = newq; + queue_decref(oldq); + +done: + if (p) + task_rq_unlock(rq, p, &rf); + fput(file); + return error; +} + +static int ghost_set_default_queue(int qfd) +{ + struct fd f_que = {0}; + struct ghost_queue *newq; + + f_que = fdget(qfd); + newq = fd_to_queue(f_que); + if (!newq) { + fdput(f_que); + return -EBADF; + } + /* The implied target enclave is whichever newq belongs to. */ + enclave_set_default_queue(newq->enclave, newq); + fdput(f_que); + return 0; +} + +/* + * Resolve the target CPU associated with a ghost_queue. + * + * The caller must guarantee that 'q' is stable. + * + * For e.g. the ghost queue associated with a task (p->ghost.dst_q) + * is protected by task_rq(p)->lock. It is the caller's responsibility + * to hold the 'rq->lock' when target_cpu(p->ghost.dst_q) is called. + * + * Returns the CPU of the ghost agent to wakeup or -1 if an eligible + * CPU is not found or configured. + */ +static int target_cpu(struct ghost_queue *q, int preferred_cpu) +{ + struct queue_notifier *notifier; + int cpu = -1, i; + + if (unlikely(!q)) + return -1; + + rcu_read_lock(); + notifier = rcu_dereference(q->notifier); + if (notifier) { + VM_BUG_ON(notifier->wnum > GHOST_MAX_WINFO); + for (i = 0; i < notifier->wnum; i++) { + const int wakeup_cpu = notifier->winfo[i].cpu; + + if (cpu < 0) + cpu = wakeup_cpu; + + /* + * If the preferred_cpu is one of the candidates then + * choose it. This is so that task state change msgs + * (e.g. TASK_BLOCKED) naturally wake the local agent. + */ + if (wakeup_cpu == preferred_cpu) { + cpu = preferred_cpu; + break; + } + } + } + rcu_read_unlock(); + return cpu; +} + +static int task_target_cpu(struct task_struct *p) +{ + struct rq *rq = task_rq(p); + + /* 'p->ghost.dst_q' is protected by 'rq->lock' */ + lockdep_assert_held(&rq->lock); + + /* + * It doesn't make sense to notify an agent about its own state change. + */ + if (unlikely(p->ghost.agent)) + return -1; + + return target_cpu(p->ghost.dst_q, task_cpu(p)); +} + +static int agent_target_cpu(struct rq *rq) +{ + struct task_struct *agent = rq->ghost.agent; + + lockdep_assert_held(&rq->lock); + VM_BUG_ON(!is_agent(rq, agent)); + + return target_cpu(agent->ghost.dst_q, task_cpu(agent)); +} + +static int ghost_config_queue_wakeup(int qfd, + struct ghost_agent_wakeup __user *w, + int ninfo, int flags) +{ + struct ghost_queue *q; + struct queue_notifier *qn, *old; + struct ghost_agent_wakeup wakeup[GHOST_MAX_WINFO]; + struct file *f; + ulong fl; + int cpu = -1, i; + int ret = 0; + + if (ninfo <= 0 || ninfo > GHOST_MAX_WINFO) + return -EINVAL; + + if (flags) + return -EINVAL; + + f = fget(qfd); + if (!f) + return -EBADF; + + if (f->f_op != &queue_fops) { + ret = -EBADF; + goto out_fput; + } + + q = f->private_data; + if (!q) { + ret = -EINVAL; + goto out_fput; + } + + if (copy_from_user(&wakeup, w, + sizeof(struct ghost_agent_wakeup) * ninfo)) { + ret = -EFAULT; + goto out_fput; + } + + for (i = 0; i < ninfo; i++) { + /* cpu == -1 implies that it is polling for messages. */ + cpu = wakeup[i].cpu; + + if (wakeup[i].prio || (cpu < -1) || (cpu >= nr_cpu_ids) || + !cpu_online(cpu)) { + ret = -EINVAL; + goto out_fput; + } + } + + qn = kzalloc(sizeof(struct queue_notifier), GFP_KERNEL); + memcpy(qn->winfo, wakeup, sizeof(qn->winfo)); + qn->wnum = ninfo; + + spin_lock_irqsave(&q->lock, fl); + old = rcu_dereference_protected(q->notifier, lockdep_is_held(&q->lock)); + rcu_assign_pointer(q->notifier, qn); + spin_unlock_irqrestore(&q->lock, fl); + + /* Wakeup agent on new CPU, in case 'q' has pending messages. */ + ghost_wake_agent_on(cpu); + + if (old) + kfree_rcu(old, rcu); +out_fput: + fput(f); + return ret; +} + +static int ghost_set_option(int option, ulong val1, ulong val2, ulong val3) +{ + switch (option) { + default: + return -EINVAL; + } + + return 0; +} + +static int ghost_get_cpu_time(gtid_t gtid, u64 __user *uruntime) +{ + struct task_struct *p; + u64 time; + + rcu_read_lock(); + p = find_task_by_gtid(gtid); + if (!p) { + rcu_read_unlock(); + return -ESRCH; + } + time = task_sched_runtime(p); + rcu_read_unlock(); + if (copy_to_user(uruntime, &time, sizeof(*uruntime))) + return -EFAULT; + return 0; +} + +/* + * 'ring_avail_slots' returns the available slots in the ring. + * + * More interestingly it splits the available slots into two contiguous + * segments. The first one starts at 'ring->head' and extends all the + * way to the end of the ring buffer. The second one starts at the + * beginning of the ring buffer and extends up to 'ring->tail'. + * + * The producer can then use this information to decide where to place + * a message. For e.g. assume that 'ring->head' is on the last slot of + * an empty ring buffer (i.e. ring->tail == ring->head). If a message + * requires two slots then the producer can skip the last slot with a + * NOP message and enqueue the actual message in the first two slots + * of the ring buffer. + */ +struct avail_slots { + uint32_t ahead; /* slots from 'head' to end of buffer */ + uint32_t behind; /* slots from beginning of buffer to 'tail' */ +}; + +static inline struct avail_slots +ring_avail_slots(struct ghost_ring *r, uint32_t maxslots) +{ + uint32_t used; + uint32_t head, tail; + struct avail_slots slots; + + head = READ_ONCE(r->head); + tail = READ_ONCE(r->tail); + used = head - tail; + + if (used >= maxslots) { + /* + * Ring is full (bad) or head and tail are out-of-sync (worse). + * + * This deserves a WARN and we rely on the caller to do it + * because they have more context about the queue, msg etc. + */ + slots.ahead = slots.behind = 0; + } else { + uint32_t avail = maxslots - used; + uint32_t hidx = head & (maxslots - 1); + uint32_t tidx = tail & (maxslots - 1); + + if (tidx <= hidx) { + slots.ahead = maxslots - hidx; + slots.behind = tidx; + } else { + slots.ahead = tidx - hidx; + slots.behind = 0; + } + VM_BUG_ON(avail != slots.ahead + slots.behind); + } + return slots; +} + +static int _produce(struct ghost_queue *q, uint32_t barrier, int type, + void *payload, int payload_size) +{ + struct ghost_ring *ring = q->ring; + uint32_t hidx, slots_needed, slots_skipped = 0; + struct avail_slots avail; + ulong flags; + int msglen; + + const int nelems = q->nelems; + const int slot_size = sizeof(struct ghost_msg); + + BUILD_BUG_ON_NOT_POWER_OF_2(slot_size); + + msglen = sizeof(struct ghost_msg) + payload_size; + if (WARN_ON_ONCE(msglen > USHRT_MAX)) + return -EINVAL; + slots_needed = ALIGN(msglen, slot_size) / slot_size; + + spin_lock_irqsave(&q->lock, flags); + + avail = ring_avail_slots(ring, nelems); + if (WARN_ON_ONCE(avail.ahead < slots_needed && + avail.behind < slots_needed)) { + ring->overflow++; + spin_unlock_irqrestore(&q->lock, flags); + return -EOVERFLOW; + } + + hidx = ring->head & (nelems - 1); + if (unlikely(avail.ahead < slots_needed)) { + /* + * Produce a NOP message to occupy all the slots up to + * the end of the ring buffer. + */ + ring->msgs[hidx].type = MSG_NOP; + ring->msgs[hidx].length = avail.ahead * slot_size; + + /* + * Reset 'hidx' so we now produce starting from the + * beginning of the ring buffer. + */ + hidx = 0; + slots_skipped = avail.ahead; + } + + ring->msgs[hidx].type = type; + ring->msgs[hidx].length = msglen; + ring->msgs[hidx].seqnum = barrier; + memcpy(&ring->msgs[hidx + 1], payload, payload_size); + smp_wmb(); /* 'msg' update must be visible before 'head' update */ + + ring->head += slots_skipped + slots_needed; + smp_wmb(); /* publish the new message */ + + spin_unlock_irqrestore(&q->lock, flags); + + return 0; +} + +static inline int produce_for_task(struct task_struct *p, int type, + void *payload, int payload_size) +{ + task_barrier_inc(task_rq(p), p); + return _produce(p->ghost.dst_q, task_barrier_get(p), + type, payload, payload_size); +} + +static inline int produce_for_agent(struct rq *rq, int type, void *payload, + int payload_size) +{ + struct task_struct *agent = rq->ghost.agent; + + agent_barrier_inc(rq); + return _produce(agent->ghost.dst_q, agent_barrier_get(agent), + type, payload, payload_size); +} + +/* + * There is a unique agent associated with each ghost CPU so we use + * the agent as a proxy for delivering CPU messages (specifically + * the 'dst_q' configuration). Not only is this convenient but the + * 'status_word.barrier' increment in produce_for_agent() pairs + * correctly with the 'agent_barrier' check in ghost_run(). + * + * CPU messages are produced with the 'rq' lock held to serialize + * with agent death and 'dst_q' changes via ghost_associate_queue(). + */ +static inline bool cpu_skip_message(struct rq *rq) +{ + struct task_struct *agent = rq->ghost.agent; + + lockdep_assert_held(&rq->lock); + + if (WARN_ON_ONCE(!agent)) + return true; + + if (WARN_ON_ONCE(!agent->ghost.dst_q)) + return true; + + return false; +} + +static inline bool cpu_deliver_msg_tick(struct rq *rq) +{ + struct ghost_msg_payload_cpu_tick payload; + + if (cpu_skip_message(rq)) + return false; + if (bpf_sched_ghost_skip_tick()) + return false; + + payload.cpu = cpu_of(rq); + + return !produce_for_agent(rq, MSG_CPU_TICK, &payload, sizeof(payload)); +} + +/* Returns true if MSG_CPU_TIMER_EXPIRED was produced and false otherwise */ +static inline bool cpu_deliver_timer_expired(struct rq *rq, uint64_t cookie) +{ + struct ghost_msg_payload_timer payload; + const int size = sizeof(payload); + + if (cpu_skip_message(rq)) + return false; + + payload.cpu = cpu_of(rq); + payload.cookie = cookie; + + return !produce_for_agent(rq, MSG_CPU_TIMER_EXPIRED, &payload, size); +} + +static bool cpu_deliver_msg_not_idle(struct rq *rq, struct task_struct *next) +{ + struct ghost_msg_payload_cpu_not_idle payload; + + if (cpu_skip_message(rq)) + return false; + + payload.cpu = cpu_of(rq); + payload.next_gtid = gtid(next); + + return !produce_for_agent(rq, MSG_CPU_NOT_IDLE, + &payload, sizeof(payload)); +} + +/* + * When called from pick_next_task() context returns 'true' if 'rq->cpu' + * is exiting switchto and 'false' otherwise (e.g. when producing the + * TASK_BLOCKED/TASK_YIELD/TASK_PREEMPT msgs). + * + * When called outside pick_next_task() context returns 'true' if 'rq->cpu' + * is currently in a switchto chain and 'false' otherwise (e.g. when producing + * TASK_DEPARTED msg for an oncpu ghost task). + * + * Technically this could be split into two APIs one for 'switchto_count < 0' + * and another for 'switchto_count > 0' but that feels like overkill. + */ +static bool ghost_in_switchto(struct rq *rq) +{ + return rq->ghost.switchto_count ? true : false; +} + +static inline bool task_skip_message(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + /* + * Ignore the agent's task_state changes. + * + * If the agent is not runnable then how do we tell it that + * it's not runnable. + */ + if (unlikely(is_agent(rq, p))) + return true; + + if (WARN_ON_ONCE(!p->ghost.dst_q)) + return true; + + return false; +} + +static bool task_deliver_msg_task_new(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_new payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + payload.runnable = task_on_rq_queued(p); + payload.runtime = p->se.sum_exec_runtime; + if (_get_sw_info(p->ghost.enclave, p->ghost.status_word, + &payload.sw_info)) { + WARN(1, "New task PID %d didn't have a status word!", p->pid); + return false; + } + + return !produce_for_task(p, MSG_TASK_NEW, &payload, sizeof(payload)); +} + +static bool task_deliver_msg_yield(struct rq *rq, struct task_struct *p) +{ + + struct ghost_msg_payload_task_yield payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + payload.runtime = p->se.sum_exec_runtime; + payload.cpu = cpu_of(rq); + payload.from_switchto = ghost_in_switchto(rq); + + return !produce_for_task(p, MSG_TASK_YIELD, &payload, sizeof(payload)); +} + +static bool task_deliver_msg_preempt(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_preempt payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + payload.runtime = p->se.sum_exec_runtime; + payload.cpu = cpu_of(rq); + payload.from_switchto = ghost_in_switchto(rq); + + return !produce_for_task(p, MSG_TASK_PREEMPT, &payload, + sizeof(payload)); +} + +static bool task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_blocked payload = {.gtid = gtid(p)}; + + if (task_skip_message(rq, p)) + return false; + + payload.runtime = p->se.sum_exec_runtime; + payload.cpu = cpu_of(rq); + payload.from_switchto = ghost_in_switchto(rq); + + return !produce_for_task(p, MSG_TASK_BLOCKED, &payload, + sizeof(payload)); +} + +static bool task_deliver_msg_dead(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_dead payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + return !produce_for_task(p, MSG_TASK_DEAD, &payload, sizeof(payload)); +} + +static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_departed payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + payload.cpu = cpu_of(rq); + if (task_current(rq, p) && ghost_in_switchto(rq)) + payload.from_switchto = true; + else + payload.from_switchto = false; + + return !produce_for_task(p, MSG_TASK_DEPARTED, &payload, + sizeof(payload)); +} + +static inline bool deferrable_wakeup(struct task_struct *p) +{ +#ifdef notyet + /* + * If 'p' held a lock while it was blocked then the wakeup + * is not deferrable since other tasks might be waiting on it. + */ + if (p->lockdep_depth) + return false; +#endif + + return p->sched_deferrable_wakeup; +} + +static bool task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_wakeup payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + payload.deferrable = deferrable_wakeup(p); + return !produce_for_task(p, MSG_TASK_WAKEUP, &payload, sizeof(payload)); +} + +static bool task_deliver_msg_switchto(struct rq *rq, struct task_struct *p) +{ + struct ghost_msg_payload_task_switchto payload; + + if (task_skip_message(rq, p)) + return false; + + payload.gtid = gtid(p); + payload.runtime = p->se.sum_exec_runtime; + payload.cpu = cpu_of(rq); + + return !produce_for_task(p, MSG_TASK_SWITCHTO, &payload, + sizeof(payload)); +} + +static void release_from_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_enclave *e = p->ghost.enclave; + bool wake_agent; + ulong flags; + + VM_BUG_ON(!e); + + lockdep_assert_held(&rq->lock); + lockdep_assert_held(&p->pi_lock); + + WARN_ON_ONCE(is_cached_task(rq, p)); + + spin_lock_irqsave(&e->lock, flags); + + /* Remove before potentially clearing p->ghost.agent. */ + __enclave_remove_task(e, p); + + /* + * Annotate the status_word so it can be freed by the agent. + * + * N.B. this must be ordered before the TASK_DEAD message is + * visible to the agent. + * + * N.B. we grab the enclave lock to prevent a misbehaving agent + * from freeing the status_word (or its enclosing sw_region) + * before delivery of the TASK_DEAD message below. + */ + ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_F_CANFREE); + if (p->state == TASK_DEAD) + wake_agent = task_deliver_msg_dead(rq, p); + else + wake_agent = task_deliver_msg_departed(rq, p); + + /* status_word is off-limits to the kernel */ + p->ghost.status_word = NULL; + + if (unlikely(is_agent(rq, p))) { + /* We don't allow agents to setsched away from ghost */ + WARN_ON_ONCE(p->state != TASK_DEAD); + rq->ghost.agent = NULL; + rq->ghost.run_flags = 0; + WRITE_ONCE(rq->ghost.agent_barrier, 0); + p->ghost.agent = false; + VM_BUG_ON(rq->ghost.blocked_in_run); + VM_BUG_ON(wake_agent); + + /* See ghost_destroy_enclave() */ + if (rq->ghost.agent_remove_enclave_cpu) { + rq->ghost.agent_remove_enclave_cpu = false; + spin_lock(&cpu_rsvp); + __enclave_return_cpu(e, rq->cpu); + spin_unlock(&cpu_rsvp); + } + } + + /* + * Will drop the kref for p->ghost.enclave, possibly as soon as we drop + * rq and e locks. + */ + submit_enclave_work(e, rq, false, /*nr_decrefs=*/ 1); + p->ghost.enclave = NULL; + + spin_unlock_irqrestore(&e->lock, flags); + + if (wake_agent) + ghost_wake_agent_of(p); + + /* Release reference to 'dst_q' */ + VM_BUG_ON(!p->ghost.dst_q); + queue_decref(p->ghost.dst_q); + p->ghost.dst_q = NULL; +} + +static void task_dead_ghost(struct task_struct *p) +{ + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + release_from_ghost(rq, p); + task_rq_unlock(rq, p, &rf); +} + +/* + * Update the scheduling state used by pick_next_task_ghost(). + */ +static void ghost_set_pnt_state(struct rq *rq, struct task_struct *p, + int run_flags) +{ + trace_sched_ghost_latched(rq->ghost.latched_task, p); + + lockdep_assert_held(&rq->lock); + + if (rq->ghost.latched_task != p) { + /* + * We're overwriting a 'latched_task' that never got oncpu. + * If we ever require the kernel to send TASK_PREEMPTED for + * all preemptions, we'll need to do so here. + */ + ; + } + rq->ghost.latched_task = p; + rq->ghost.must_resched = false; + rq->ghost.run_flags = run_flags; +} + +/* + * The essence of waking an agent: increment the barrier, regardless of if it + * was sleeping, to ensure we catch the agent while it is attempting to block + * (it will return ESTALE). + * + * If it was blocked_in_run, clear it and reschedule, which ensures it wakes up. + */ +void ghost_wake_agent_on(int cpu) +{ + struct rq *dst_rq; + + if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu)) + return; + + dst_rq = cpu_rq(cpu); + + /* + * This is not the SW barrier. The RQ agent_barrier can get ahead of + * the SW barrier, so we'll need to update the SW for userspace to see + * this update. + * + * agent_barrier_inc() holds the RQ lock and can clobber this increment. + * It's OK for us if *this* increment is lost. We just need *some* + * increment to happen to prevent the agent from sleeping. However, our + * update probably needs to be atomic. If we do a non-atomic increment, + * it's possible for us to clobber agent_barrier_inc()'s locked update, + * and essentially rollback an agent barrier - possibly after it's been + * exposed to userspace. + * + * This sync operation is a full barrier. This write must happen before + * the read of blocked_in_run. + */ + (void)__sync_fetch_and_add(&dst_rq->ghost.agent_barrier, 1); + + /* + * The barrier ordered this read already. The READ_ONCE is harmless and + * mostly for paranoia. + */ + if (!READ_ONCE(dst_rq->ghost.blocked_in_run)) + return; + + /* + * We can't write blocked_in_run, since we don't hold the RQ lock. + * Instead, we'll tell the cpu to do it in pick_next_task (agent), by + * setting agent_should_wake and kicking the cpu. The initial READ is + * an optimization to reduce the number of rescheds. + */ + if (READ_ONCE(dst_rq->ghost.agent_should_wake)) + return; + WRITE_ONCE(dst_rq->ghost.agent_should_wake, true); + + /* Write must come before the IPI/resched */ + smp_wmb(); + + if (cpu == smp_processor_id()) { + set_tsk_need_resched(dst_rq->curr); + set_preempt_need_resched(); + } else { + resched_cpu_unlocked(cpu); + } +} + +void ghost_wake_agent_of(struct task_struct *p) +{ + ghost_wake_agent_on(task_target_cpu(p)); +} + +void ghost_task_preempted(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(task_rq(p) != rq); + + /* + * When TASK_PREEMPTED is produced before returning from pick_next_task + * (e.g. via pick_next_ghost_agent) we don't have an up-to-date runtime + * since put_prev_task() hasn't been called yet. + * + * Therefore if 'p == rq->curr' we must do update_curr_ghost() by hand. + */ + if (p == rq->curr) + update_curr_ghost(rq); + + /* Produce MSG_TASK_PREEMPT into 'p->ghost.dst_q' */ + task_deliver_msg_preempt(rq, p); + + /* + * Wakeup agent on this CPU. + * + * In some scheduling models (e.g. uber-agent) this may be unnecessary + * and cause redundant context switches to the agent. + * + * Return to local agent if it has expressed interest in this edge. + */ + if (rq->ghost.run_flags & RTLA_ON_PREEMPT) + schedule_agent(rq, false); +} + +static void ghost_task_new(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(task_rq(p) != rq); + + enclave_add_task(p->ghost.enclave, p); + + /* See explanation in ghost_task_preempted() */ + if (p == rq->curr) + update_curr_ghost(rq); + + task_deliver_msg_task_new(rq, p); +} + +static void ghost_task_yield(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(task_rq(p) != rq); + + /* See explanation in ghost_task_preempted() */ + if (p == rq->curr) + update_curr_ghost(rq); + + task_deliver_msg_yield(rq, p); +} + +static void ghost_task_blocked(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(task_rq(p) != rq); + + task_deliver_msg_blocked(rq, p); +} + +void ghost_wait_for_rendezvous(struct rq *rq) +{ + int64_t target; + + WARN_ON_ONCE(preemptible()); + + /* agent does not rendezvous with anybody */ + if (is_agent(rq, rq->curr)) + return; + + while (!need_resched()) { + target = smp_load_acquire(&rq->ghost.rendezvous); +#ifdef CONFIG_DEBUG_GHOST + if (target == GHOST_NO_RENDEZVOUS) + WARN_ON_ONCE(!need_resched()); +#endif + if (rendezvous_reached(target)) { + /* + * Reschedule immediately if rendezvous is poisoned + * (some other txn in the sync_group failed to commit). + * + * N.B. rescheduling here could produce TASK_PREEMPTED + * msg from 'rq->curr'. + */ + if (unlikely(rendezvous_poisoned(target))) { + struct rq_flags rf; + /* + * We could race with a remote cpu doing a sync + * commit while waiting for rendezvous. This is + * usually not a problem because a successful + * commit results in an IPI to remote CPUs to + * apply the new scheduling state (except when + * the task is already running on the remote + * cpu in which case we elide the IPI). + * + * In cases where we don't get an IPI we have + * take care to not clobber updated scheduling + * state. For e.g. T1 started running on CPU1 + * but its rendezvous was poisoned (another + * cpu in its sync_group failed to commit). By + * the time T1 acquired the rq->lock there was + * newer sync_group commit that committed T1 + * on CPU1 (this one was successful). The new + * commit would detect that T1 was already on + * CPU1 and elide the IPI. Thus we check for + * poisoned rendezvous under 'rq->lock' lest + * we force T1 offcpu inadvertently! + */ + rq_lock_irqsave(rq, &rf); + + /* + * Validate everything after acquiring rq->lock + * since everything (including our sched_class) + * could have changed underneath us. + */ + if (!ghost_need_rendezvous(rq)) { + rq_unlock_irqrestore(rq, &rf); + return; + } + + target = smp_load_acquire( + &rq->ghost.rendezvous); + if (!rendezvous_poisoned(target)) { + rq_unlock_irqrestore(rq, &rf); + continue; + } + + force_offcpu(rq, /*resched=*/true); + rq_unlock_irqrestore(rq, &rf); + VM_BUG_ON(!need_resched()); + } + break; + } + + cpu_relax(); + } +} + +static inline bool same_process(struct task_struct *p, struct task_struct *q) +{ + return p->group_leader == q->group_leader; +} + +static inline bool run_flags_valid(int run_flags, int unsupported_flags) +{ + const int valid_run_flags = RTLA_ON_PREEMPT | + RTLA_ON_BLOCKED | + RTLA_ON_YIELD | + RTLA_ON_IDLE | + NEED_L1D_FLUSH | + NEED_CPU_NOT_IDLE | + ALLOW_TASK_ONCPU; + + if (run_flags & ~valid_run_flags) + return false; + + if (run_flags & unsupported_flags) + return false; + + return true; +} + +/* + * ghOSt API to yield local cpu or ping a remote cpu. + * + * Return 0 on success and -1 on failure. + */ +SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, + u32, task_barrier, int, run_cpu, int, run_flags) +{ + struct task_struct *agent; + struct rq_flags rf; + struct rq *rq; + int error = 0; + int this_cpu; + + if (!capable(CAP_SYS_NICE)) + return -EPERM; + + if (run_cpu < 0 || run_cpu >= nr_cpu_ids || !cpu_online(run_cpu)) + return -EINVAL; + + preempt_disable(); + this_cpu = raw_smp_processor_id(); + + switch (gtid) { + case GHOST_NULL_GTID: /* yield on local cpu */ + if ((run_flags & ~(RTLA_ON_IDLE | NEED_CPU_NOT_IDLE)) || + (run_cpu != this_cpu)) + error = -EINVAL; + break; + case GHOST_AGENT_GTID: /* ping agent */ + if (run_flags) + error = -EINVAL; + break; + default: + error = -EINVAL; + break; + } + + if (error) + goto done; + + rq = cpu_rq(run_cpu); + rq_lock_irq(rq, &rf); + + agent = rq->ghost.agent; + if (unlikely(!agent)) { + error = -EINVAL; + rq_unlock_irq(rq, &rf); + goto done; + } + + /* ping agent */ + if (gtid == GHOST_AGENT_GTID) { + if (same_process(agent, current)) { + /* "serialize" with remote-agent doing a local run */ + agent_barrier_inc(rq); + schedule_agent(rq, true); + } else { + error = -EINVAL; + } + rq_unlock_irq(rq, &rf); + goto done; + } + + /* + * Yield local cpu. Agent is voluntarily blocked in ghost_run() and + * returns when: + * + * - another agent "pings" it via ghost_run(AGENT_GTID). + * - a message is produced into a queue that the agent has expressed + * an interest in (via ghost_wake_agent()). + * - a scheduling edge from an oncpu ghost task (e.g. blocked edge + * or yield edge). + */ + VM_BUG_ON(gtid != GHOST_NULL_GTID); + VM_BUG_ON(run_cpu != this_cpu); + + /* + * If an agent is going to block then we must be absolutely sure + * that its state is up-to-date (e.g. ping, remote-run or wakeup). + * + * The 'agent_barrier' check below guarantees this. + * + * The mb orders the blocked_in_run write before the barrier + * check in case of a concurrent wakeup. + */ + smp_store_mb(rq->ghost.blocked_in_run, true); + if (unlikely(agent_barrier_get(agent) != agent_barrier)) { + rq->ghost.blocked_in_run = false; + rq_unlock_irq(rq, &rf); + error = -ESTALE; + goto done; + } + + /* + * Reuse the latched task in case it was updated by a remote agent + * (before this local yield). + */ + ghost_set_pnt_state(rq, rq->ghost.latched_task, run_flags); + + rq_unlock_irq(rq, &rf); + + sched_preempt_enable_no_resched(); + + schedule(); + + /* + * Disable preemption to ensure that runqueue remains stable. + */ + preempt_disable(); + rq = this_rq(); + + /* + * The agent is per-cpu and must always schedule on that CPU. + * + * In other words it cannot block or switch to 'next' on one + * CPU and wake up on a different one. + */ + VM_BUG_ON(rq->cpu != this_cpu); + + /* + * We reach this point after 'pick_next_task_ghost()' selected + * this agent to run and now we are returning back to userspace. + * + * 'blocked_in_run' must reflect that we are leaving ghost_run(). + */ + VM_BUG_ON(rq->ghost.blocked_in_run); +done: + preempt_enable_no_resched(); + + return error; +} + +static inline bool _ghost_txn_ready(int cpu, int *commit_flags) +{ + struct ghost_txn *txn; + bool ready = false; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(cpu < 0 || cpu >= nr_cpu_ids); + + rcu_read_lock(); + txn = rcu_dereference(per_cpu(ghost_txn, cpu)); + if (txn) { + ready = smp_load_acquire(&txn->state) == GHOST_TXN_READY; + if (commit_flags != NULL) + *commit_flags = READ_ONCE(txn->commit_flags); + } + rcu_read_unlock(); + + return ready; +} + +static inline bool ghost_txn_ready(int cpu) +{ + return _ghost_txn_ready(cpu, NULL); +} + +static inline bool ghost_txn_greedy(int cpu) +{ + int commit_flags; + + return _ghost_txn_ready(cpu, &commit_flags) && commit_flags == 0; +} + +/* + * Try to claim txn iff the point-of-commit matches what was requested + * via 'commit_flags'. Note that 'commit_flags == 0' indicates a greedy + * commit and matches all commit points. + * + * The only exception is when a commit is explicitly requested by + * the agent via GHOST_COMMIT_TXN in which case 'txn->commit_flags' + * is ignored (indicated by where = -1). + * + * Returns 'true' if txn was claimed and 'false' otherwise. + * + * N.B. 'where' is deliberately an 'int' as opposed to 'enum txn_commit_at' + * otherwise the compiler optimizes away the 'where >= 0' conditional below + * since it "knows" all possible enum values. + */ +static inline bool ghost_claim_txn(int cpu, int where) +{ + struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, cpu)); + + VM_BUG_ON(preemptible()); + VM_BUG_ON(cpu < 0 || cpu >= nr_cpu_ids); + + if (!txn || smp_load_acquire(&txn->state) != GHOST_TXN_READY) + return false; + + if (where >= 0 && txn->commit_flags != 0 && txn->commit_flags != where) + return false; + + /* + * TODO: don't claim transaction if commit is going to fail + * (for e.g. if 'current' is in a higher priority sched_class). + * + * This behavior can be finessed using 'txn->flags'. + */ + return cmpxchg_acquire(&txn->state, + GHOST_TXN_READY, raw_smp_processor_id()) == GHOST_TXN_READY; +} + +static inline bool txn_commit_allowed(struct rq *rq, gtid_t gtid, bool sync) +{ + /* + * Asynchronous commit is instigated by kernel and thus always + * allowed (e.g. return-to-user). + */ + if (!sync) + return true; + + /* An agent is always allowed to commit synchronously. */ + if (current->ghost.agent) + return true; + + /* + * A non-agent task is allowed to ping an agent as long as both + * belong to the same process. + */ + if (gtid == GHOST_AGENT_GTID && same_process(rq->ghost.agent, current)) + return true; + + return false; +} + +/* Returns 'true' if txn commit is instigated by the agent on its own cpu */ +static inline bool _local_commit(struct rq *rq, bool sync) +{ + int run_cpu = cpu_of(rq); + int this_cpu = raw_smp_processor_id(); + + lockdep_assert_held(&rq->lock); + WARN_ON_ONCE(preemptible()); + + return sync && this_cpu == run_cpu && is_agent(rq, current); +} + +/* Returns 'true' if the agent is pinging itself */ +static inline bool ping_self(struct rq *rq, bool sync, gtid_t gtid) +{ + return _local_commit(rq, sync) && gtid == GHOST_AGENT_GTID; +} + +/* + * Returns 'true' if agent is giving up the CPU because it is switching + * to another ghost task or idling. + */ +static inline bool blocking_run(struct rq *rq, bool sync, gtid_t gtid) +{ + return _local_commit(rq, sync) && gtid != GHOST_AGENT_GTID; +} + +/* + * Caller is responsible for claiming txn (before calling this function) + * and finalizing it (after this function returns). + */ +static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, + int *commit_state, bool *need_rendezvous) +{ + gtid_t gtid; + struct rq *rq; + struct rq_flags rf; + struct task_struct *next; + bool local_run, resched = false; + int run_flags, state = GHOST_TXN_COMPLETE; + struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, run_cpu)); + + VM_BUG_ON(preemptible()); + VM_BUG_ON(commit_state == NULL); + VM_BUG_ON(need_rendezvous == NULL); + VM_BUG_ON(run_cpu < 0 || run_cpu >= nr_cpu_ids); + + *need_rendezvous = false; + + if (!txn || txn->state != raw_smp_processor_id()) { + state = GHOST_TXN_INVALID_CPU; + goto done; + } + + if (txn->version != GHOST_TXN_VERSION) { + state = GHOST_TXN_UNSUPPORTED_VERSION; + goto done; + } + + if (txn->cpu != run_cpu) { + state = GHOST_TXN_INVALID_CPU; + goto done; + } + + gtid = READ_ONCE(txn->gtid); + run_flags = READ_ONCE(txn->run_flags); + if (!run_flags_valid(run_flags, 0)) { + state = GHOST_TXN_INVALID_FLAGS; + goto done; + } + + if ((run_flags & NEED_CPU_NOT_IDLE) && + (gtid != GHOST_NULL_GTID && gtid != GHOST_IDLE_GTID)) { + state = GHOST_TXN_INVALID_FLAGS; + goto done; + } + + if (!cpu_online(run_cpu)) { + state = GHOST_TXN_CPU_OFFLINE; + goto done; + } + + if (likely(gtid > 0)) { + rcu_read_lock(); + next = find_task_by_gtid(gtid); + if (next == NULL || next->ghost.agent) { + rcu_read_unlock(); + state = next ? GHOST_TXN_INVALID_TARGET : + GHOST_TXN_TARGET_NOT_FOUND; + goto done; + } + + rq = task_rq_lock(next, &rf); + rcu_read_unlock(); + + if (validate_next_task(rq, next, txn->task_barrier, &state)) { + task_rq_unlock(rq, next, &rf); + goto done; + } + + if (!(run_flags & ALLOW_TASK_ONCPU) || + (task_cpu(next) != run_cpu)) { + if (validate_next_offcpu(rq, next, &state)) { + task_rq_unlock(rq, next, &rf); + goto done; + } + } else if (task_running(rq, next) && + rq->ghost.must_resched && + test_tsk_need_resched(next)) { + /* + * 'next' is running but its oncpu days are numbered + * due to TIF_NEED_RESCHED. Specifically this handles + * the the race with ghost_wait_for_rendezvous() where + * it calls force_offcpu() due to an earlier poisoned + * rendezvous but hasn't scheduled yet (either hasn't + * reached a scheduling point or waiting for 'rq->lock' + * in __schedule). + * + * Fail the commit until 'next' gets fully offcpu. + */ + state = GHOST_TXN_TARGET_STALE; + task_rq_unlock(rq, next, &rf); + goto done; + } + + rq = ghost_move_task(rq, next, run_cpu, &rf); + + raw_spin_unlock(&next->pi_lock); /* irqs still disabled */ + } else { + if (gtid < GHOST_IDLE_GTID) { + state = GHOST_TXN_INVALID_TARGET; + goto done; + } + next = NULL; + rq = cpu_rq(run_cpu); + rq_lock_irqsave(rq, &rf); + + if (gtid == GHOST_IDLE_GTID) + next = rq->idle; + + if (!(run_flags & ALLOW_TASK_ONCPU)) { + if (validate_next_offcpu(rq, next, &state)) + goto unlock_rq; + } + } + + /* + * Update 'rq->ghost_rq' latch state for pick_next_task() to use + * when making a decision. + */ + if (unlikely(!rq->ghost.agent)) { + state = GHOST_TXN_NO_AGENT; + goto unlock_rq; + } + + if (unlikely(!txn_commit_allowed(rq, gtid, sync))) { + state = GHOST_TXN_NOT_PERMITTED; + goto unlock_rq; + } + + if (next && !ghost_can_schedule(rq, gtid)) { + /* + * Transaction cannot be committed if CPU is not available + * (but only if 'next' is a task that can run elsewhere). + * + * Specifically if 'txn->gtid' is GHOST_AGENT_GTID there is + * no point returning an error because the agent cannot run + * anywhere else. + */ + state = GHOST_TXN_CPU_UNAVAIL; + goto unlock_rq; + } + + local_run = blocking_run(rq, sync, gtid); + if (local_run) { + /* + * Agent is doing a synchronous commit on its local cpu and + * caller will schedule() on return. We ensure that its + * state is up-to-date via the 'agent_barrier' check below. + * + * The mb orders the blocked_in_run write before the barrier + * check in case of a concurrent wakeup. + */ + smp_store_mb(rq->ghost.blocked_in_run, true); + if (agent_barrier_get(rq->ghost.agent) != + READ_ONCE(txn->agent_barrier)) { + rq->ghost.blocked_in_run = false; + state = GHOST_TXN_AGENT_STALE; + goto unlock_rq; + } + } else { + /* + * We do not assert a barrier match for the remote run case: + * + * In a remote run case we validate that that task_barrier + * is up-to-date (i.e. we have consumed the latest message for + * that task). We don't care so much about the agent_barrier + * because the agent is not blocking anyways. + * + * There is a danger in insisting that scheduling decisions be + * made on the most up-to-date state. In the limit this could + * lead to a livelock where the agent keeps making scheduling + * decisions (but never gets to act on it). Practically we need + * to consider the likelihood of a scheduling decision changing + * after consuming a pending message. + */ + } + + *need_rendezvous = true; + resched = ghost_can_schedule(rq, gtid); + + if (next && task_running(rq, next)) { + /* 'next' is already oncpu */ + VM_BUG_ON(!(run_flags & ALLOW_TASK_ONCPU)); + resched = false; + + /* + * If 'next' is already oncpu and rendezvous is !poisoned + * then don't update rendezvous. Here's why: + * - if the sync_commit succeeds there is no benefit in + * updating the rendezvous from one !poisoned value to + * another !poisoned value (this is true even if 'next' + * hasn't entered wait_for_rendezvous). + * - if the sync_commit fails (due to some other cpu) and + * 'next' hasn't entered wait_for_rendezvous yet then + * updating rendezvous will force 'next' offcpu + * (benign but unnecessary context switch). + * + * The basic idea is that if 'next' was able to get oncpu + * successfully then keep it there until it is explicitly + * preempted by the agent (yielding to CFS or bound to a + * different eg_core). + */ + WARN_ON_ONCE(!rendezvous_reached(rq->ghost.rendezvous)); + if (!rendezvous_poisoned(rq->ghost.rendezvous)) + *need_rendezvous = false; + + /* + * 'next' is already oncpu but a different task may be latched + * in 'latched_task' (for e.g. remote cpu has disabled irqs + * and a resched_ipi is pending). Alternately the remote cpu + * could be running the idle task organically and therefore + * 'rq->ghost.run_flags' needs to be updated. + * + * Set 'latched_task' to NULL to recreate the state right after + * 'next' got oncpu (see pick_next_task_ghost()). + */ + ghost_set_pnt_state(rq, NULL, run_flags); + } else { + /* + * 'next' may be NULL if 'txn->gtid' is one of the special + * encodings. + */ + + /* "serialize" with remote-agent doing a local run */ + agent_barrier_inc(rq); + + /* + * Update latched task unless this is a ping in which case + * we'll get the agent running without clobbering an already + * latched task. + */ + if (likely(gtid != GHOST_AGENT_GTID)) + ghost_set_pnt_state(rq, next, run_flags); + + if (!next) { + /* Handle special gtid encodings (e.g. ping) */ + schedule_next(rq, gtid, false); + } + } + + if (!local_run) { + if (!sync) { + VM_BUG_ON(run_cpu != raw_smp_processor_id()); + /* + * Agents have absolute priority over normal ghost + * tasks so no need to reschedule when the txn is + * asynchronously committed in agent context (e.g. + * return-to-user path). + */ + if (is_agent(rq, current)) + resched = false; + } else { + /* Don't schedule if agent is pinging its own CPU */ + if (ping_self(rq, sync, gtid)) + resched = false; + } + + /* + * Elide the reschedule IPI if TIF_NEED_RESCHED is already set + * or if the target CPU is polling for it. + */ + if (resched) { + resched = !test_tsk_need_resched(rq->curr) && + set_nr_and_not_polling(rq->curr); + } + } + VM_BUG_ON(local_run && !resched); /* local_run must reschedule */ + + /* + * Release store to ensure that TIF_NEED_RESCHED is visible on the + * remote cpu before GHOST_NO_RENDEZVOUS so we don't release an + * earlier sync-group transaction inadvertently. + */ + if (*need_rendezvous) + smp_store_release(&rq->ghost.rendezvous, rendezvous); + +unlock_rq: + rq_unlock_irqrestore(rq, &rf); +done: + *commit_state = state; + return resched; +} + +static inline struct ghost_txn *_ghost_get_txn_ptr(int cpu) +{ + int this_cpu = raw_smp_processor_id(); + struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, cpu)); + + VM_BUG_ON(preemptible()); + VM_BUG_ON(cpu < 0 || cpu >= nr_cpu_ids); + if (txn && txn->state != this_cpu) { + /* A buggy agent can trip this. */ + pr_info("txn->state for cpu %d was not %d!", txn->state, + this_cpu); + } + + return txn; +} + +static inline void _ghost_set_txn_state(struct ghost_txn *txn, + enum ghost_txn_state state) +{ + smp_store_release(&txn->state, state); +} + +static void ghost_set_txn_state(int cpu, enum ghost_txn_state state) +{ + struct ghost_txn *txn = _ghost_get_txn_ptr(cpu); + + if (txn) + _ghost_set_txn_state(txn, state); +} + +static inline void ghost_poison_txn(int cpu) +{ + ghost_set_txn_state(cpu, GHOST_TXN_POISONED); +} + +static inline bool ghost_txn_succeeded(int state) +{ + return state == GHOST_TXN_COMPLETE; +} + +static bool ghost_commit_txn(int run_cpu, bool sync, int *commit_state) +{ + int state; + bool resched, need_rendezvous; + + resched = _ghost_commit_txn(run_cpu, sync, GHOST_NO_RENDEZVOUS, &state, + &need_rendezvous); + if (commit_state) + *commit_state = state; + + ghost_set_txn_state(run_cpu, state); + return resched; +} + +#if defined(CONFIG_X86_64) || defined(CONFIG_X86) +static inline void ghost_send_reschedule(struct cpumask *mask) +{ + int cpu; + + for_each_cpu(cpu, mask) { + if (try_ipiless_wakeup(cpu)) + __cpumask_clear_cpu(cpu, mask); + } + + if (!cpumask_empty(mask)) + apic->send_IPI_mask(mask, RESCHEDULE_VECTOR); + +#ifdef CONFIG_DEBUG_GHOST + /* + * 'mask' can be modified non-deterministically due to ipiless wakeup + * above and callers must not assume that 'mask' is same before and + * after the call. + * + * Weed out such callers by clobbering 'mask' in debug builds. + */ + cpumask_clear(mask); +#endif +} +#else +static inline void ghost_send_reschedule(struct cpumask *mask) +{ + int cpu; + + for_each_cpu(cpu, mask) + smp_send_reschedule(cpu); +} +#endif + +static bool _ghost_commit_pending_txn(int cpu, enum txn_commit_at where) +{ + if (unlikely(ghost_claim_txn(cpu, where))) + return ghost_commit_txn(cpu, false, NULL); + return false; +} + +static void ghost_commit_pending_txn(enum txn_commit_at where) +{ + int cpu = get_cpu(); + + rcu_read_lock(); + _ghost_commit_pending_txn(cpu, where); + rcu_read_unlock(); + + put_cpu(); +} + +extern void ghost_commit_greedy_txn(void) +{ + ghost_commit_pending_txn(0); /* Commit a greedy pending txn */ +} + +void ghost_commit_all_greedy_txns(void) +{ + int cpu, this_cpu; + struct ghost_enclave *e; + cpumask_var_t ipimask; + + /* + * 'cpumask_var_t' currently allocates CPU masks on the stack since + * CONFIG_CPUMASK_OFFSTACK is not set. If this flag does become set in + * the future, 'cpumask_var_t' becomes a pointer and + * 'zalloc_cpumask_var' allocates memory rather than act as a no-op. In + * that case, we will likely want to preallocate 'ipimask' for each CPU + * when the transaction region is set up rather than allocate memory on + * every timer tick. This warning will get our attention if the flag is + * set. + */ +#ifdef CONFIG_CPUMASK_OFFSTACK + WARN_ONCE(1, "Pre-allocate cpumasks"); +#endif + + if (!zalloc_cpumask_var(&ipimask, GFP_ATOMIC)) + return; + + this_cpu = get_cpu(); + rcu_read_lock(); + e = rcu_dereference(per_cpu(enclave, this_cpu)); + if (!e) + goto done; + /* + * Note that e's cpu mask could be changed concurrently, with cpus added + * or removed. This is benign. First, any commits will look at the + * rcu-protected ghost_txn pointer. That's what really matters, and + * any caller to __enclave_unpublish_cpu() will synchronize_rcu(). + * + * Furthermore, if a cpu is added while we are looking (which is not + * protected by RCU), it's not a big deal. This is a greedy commit, and + * we'll catch it on the next tick. + */ + for_each_cpu(cpu, &e->cpus) { + /* + * Send a resched IPI to remote CPUs (and do not commit those + * txns) but commit a txn for this CPU. + */ + if (cpu == this_cpu) + ghost_commit_greedy_txn(); + else if (ghost_txn_greedy(cpu)) + __cpumask_set_cpu(cpu, ipimask); + } + /* Poke the remote CPUs so that they see their pending txns. */ + ghost_send_reschedule(ipimask); +done: + free_cpumask_var(ipimask); + rcu_read_unlock(); + put_cpu(); +} + +static void ghost_reached_rendezvous(int cpu, int64_t target) +{ + struct rq *rq; + + WARN_ON_ONCE(preemptible()); + + rq = cpu_rq(cpu); + smp_store_release(&rq->ghost.rendezvous, target); +} + +/* + * A sync-group cookie uniquely identifies a sync-group commit. + * + * It is useful to identify the initiator and participants of the sync-group. + */ +static inline int64_t ghost_sync_group_cookie(void) +{ + int64_t val; + + WARN_ON_ONCE(preemptible()); + BUILD_BUG_ON(NR_CPUS > (1U << SG_COOKIE_CPU_BITS)); + + val = __this_cpu_inc_return(sync_group_cookie); + VM_BUG_ON((val <= 0) || (val & GHOST_POISONED_RENDEZVOUS)); + + return val; +} + +static int gsys_ghost_sync_group(ulong __user *user_mask_ptr, + uint user_mask_len, int flags) +{ + int64_t target; + bool failed = false; + bool local_resched = false; + int cpu, this_cpu, error, state; + cpumask_var_t cpumask, ipimask, rendmask; + + const int valid_flags = 0; + + if (flags & ~valid_flags) + return -EINVAL; + + if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) + return -ENOMEM; + + error = get_user_cpu_mask(user_mask_ptr, user_mask_len, cpumask); + if (error) { + free_cpumask_var(cpumask); + return error; + } + + if (!zalloc_cpumask_var(&ipimask, GFP_KERNEL)) { + free_cpumask_var(cpumask); + return -ENOMEM; + } + + if (!zalloc_cpumask_var(&rendmask, GFP_KERNEL)) { + free_cpumask_var(cpumask); + free_cpumask_var(ipimask); + return -ENOMEM; + } + + preempt_disable(); + this_cpu = raw_smp_processor_id(); + target = ghost_sync_group_cookie(); + + rcu_read_lock(); + + /* + * Claim all transactions. We have the following invariant at the + * end of the loop: + * 'failed' 'cpumask' + * false all sync_group cpus. + * true subset of sync_group cpus with claimed txns. + */ + for_each_cpu(cpu, cpumask) { + if (WARN_ON_ONCE(!ghost_claim_txn(cpu, -1))) { + /* + * This is not expected and points at a programming + * error in the agent (e.g. txn was async committed + * on another cpu). + */ + failed = true; + __cpumask_clear_cpu(cpu, cpumask); + } + } + + /* + * Commit transactions. We have the following invariant at the end + * of the loop: + * 'failed' 'cpumask' + * false all sync_group cpus. + * true sync_group subset with successfully committed txns. + * + * In either case 'ipimask' contains the CPUs that must be interrupted + * to observe the updated scheduling state. It is always a a subset of + * 'cpumask' because: + * - TIF_NEED_RESCHED already set or remote cpu is polling for it. + * - reschedule on the local cpu is captured in 'local_resched'. + * + * Note that for_each_cpu_wrap() guarantees that 'this_cpu' is the + * last cpu visited in the loop. This lets _ghost_commit_txn() set + * 'blocked_in_run=true' for local commits safely (otherwise the + * caller needs to clear 'blocked_in_run' if the overall sync_group + * fails to commit subsequently). + */ + for_each_cpu_wrap(cpu, cpumask, this_cpu + 1) { + bool resched, need_rendezvous; + /* + * No point in committing this txn if we know a prior txn + * failed to commit. + */ + if (failed) { + ghost_poison_txn(cpu); + __cpumask_clear_cpu(cpu, cpumask); + continue; + } + + resched = _ghost_commit_txn(cpu, true, -target, &state, + &need_rendezvous); + if (!ghost_txn_succeeded(state)) { + VM_BUG_ON(resched); + VM_BUG_ON(need_rendezvous); + failed = true; + ghost_set_txn_state(cpu, state); + __cpumask_clear_cpu(cpu, cpumask); + } else { + if (resched) { + VM_BUG_ON(!need_rendezvous); + if (cpu == this_cpu) + local_resched = true; + else + __cpumask_set_cpu(cpu, ipimask); + } + + if (need_rendezvous) + __cpumask_set_cpu(cpu, rendmask); + } + } + + /* + * Send resched IPI to CPUs that traverse the need_resched edge (0->1). + * + * _ghost_commit_txn() assumes that if need_resched is set then an IPI + * must have been already sent. + */ + ghost_send_reschedule(ipimask); + + /* + * The overall sync_group failed but we may have successfully updated + * scheduling state on a subset of CPUs. Poison 'target' to get these + * cpus to reschedule immediately after reaching rendezvous. + */ + if (failed) + target |= GHOST_POISONED_RENDEZVOUS; + + for_each_cpu(cpu, rendmask) + ghost_reached_rendezvous(cpu, target); + + state = failed ? GHOST_TXN_POISONED : GHOST_TXN_COMPLETE; + for_each_cpu(cpu, cpumask) { + struct ghost_txn *txn = _ghost_get_txn_ptr(cpu); + + if (!txn) { + /* + * We've committed, but the cpu has since been removed + * from the enclave. Possibly it's destroyed. + */ + continue; + } + _ghost_set_txn_state(txn, state); + if (!failed) { + /* + * Release ownership in case we end up scheduling on + * this cpu (thus precluding the agent from doing so + * itself). When control eventually returns to the + * agent the return value can be used to detect a + * successful commit. + * + * N.B. even though not strictly necessary the txn + * ownership is released even if the local cpu is + * not scheduling. This is intentional to keep the + * userspace code consistent (it doesn't need to + * distinguish between local/remote sync commits). + */ + smp_store_release(&txn->u.sync_group_owner, -1); + } else { + /* + * The commit failed but the agent retains ownership + * of the transactions in the sync_group. The agent + * can release ownership after it has inspected the + * reason for failure. + */ + } + } + + rcu_read_unlock(); + + /* Reschedule (potentially switching to 'latched_task'). */ + if (local_resched) { + WARN_ON_ONCE(failed); + ghost_agent_schedule(); + } + + WARN_ON_ONCE(this_rq()->ghost.blocked_in_run); + + preempt_enable_no_resched(); + free_cpumask_var(cpumask); + free_cpumask_var(ipimask); + free_cpumask_var(rendmask); + return !failed; +} + +static int gsys_ghost_commit_txn(ulong __user *user_mask_ptr, + uint user_mask_len, int flags) +{ + int error, state; + int cpu, this_cpu; + int failed_commits = 0; + bool local_resched = false; + cpumask_var_t cpumask, ipimask; + + const int valid_flags = 0; + + if (flags & ~valid_flags) + return -EINVAL; + + if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) + return -ENOMEM; + + error = get_user_cpu_mask(user_mask_ptr, user_mask_len, cpumask); + if (error) { + free_cpumask_var(cpumask); + return error; + } + + if (!zalloc_cpumask_var(&ipimask, GFP_KERNEL)) { + free_cpumask_var(cpumask); + return -ENOMEM; + } + +#ifdef notyet + /* Restrict 'cpumask' to the current enclave. */ + ghost_enclave_cpus_allowed(current, cpus_allowed); + cpumask_and(cpumask, cpumask, cpus_allowed); +#endif + + preempt_disable(); + this_cpu = raw_smp_processor_id(); + + rcu_read_lock(); + for_each_cpu_wrap(cpu, cpumask, this_cpu + 1) { + int commit_flags; + bool greedy_commit, inline_commit; + + if (!_ghost_txn_ready(cpu, &commit_flags)) + continue; + + greedy_commit = (commit_flags == 0); + inline_commit = (commit_flags == COMMIT_AT_TXN_COMMIT); + + if (cpu != this_cpu && !inline_commit) { + /* + * Only send an IPI if transaction can be committed by + * the remote CPU. Notably this excludes transactions + * that must be committed in specific code paths (for + * e.g. COMMIT_AT_SCHEDULE used for a pre-staged txn). + */ + if (greedy_commit) + __cpumask_set_cpu(cpu, ipimask); + + continue; + } + + /* + * Agent wants us to commit a transaction whose 'commit_flags' + * indicate that it should be committed elsewhere (for e.g. + * COMMIT_AT_SCHEDULE for a pre-staged transaction). + */ + WARN_ONCE(!greedy_commit && !inline_commit, + "Unexpected txn->commit_flags %#x on cpu %d", + commit_flags, cpu); + + /* + * Commit txn locally either because it targets 'this_cpu' + * or caller wants it committed inline. + */ + if (!ghost_claim_txn(cpu, -1)) { + failed_commits++; + continue; + } + + /* + * If there are failed commits then poison the local commit + * so we'll return from this syscall immediately (as opposed + * to switching to a task which would delay reaction to the + * failed commits). + * + * Note that for_each_cpu_wrap() guarantees that 'this_cpu' + * is the last cpu visited in the loop so 'failed_commits' + * is final. + */ + if (failed_commits > 0 && cpu == this_cpu) { + ghost_poison_txn(cpu); + continue; + } + + if (ghost_commit_txn(cpu, true, &state)) { + WARN_ON_ONCE(!ghost_txn_succeeded(state)); + if (cpu == this_cpu) + local_resched = true; + else + __cpumask_set_cpu(cpu, ipimask); + } else if (!ghost_txn_succeeded(state)) { + failed_commits++; + } else { + /* commit succeeded, resched not needed */ + } + } + rcu_read_unlock(); + + ghost_send_reschedule(ipimask); + + /* Reschedule (potentially switching to 'latched_task'). */ + if (local_resched) { + WARN_ON_ONCE(failed_commits); + ghost_agent_schedule(); + } + + WARN_ON_ONCE(this_rq()->ghost.blocked_in_run); + + preempt_enable_no_resched(); + free_cpumask_var(cpumask); + free_cpumask_var(ipimask); + return 0; +} + +static int ghost_timerfd_validate(struct timerfd_ghost *timerfd_ghost) +{ + int cpu; + struct rq *rq; + struct rq_flags rf; + const int valid_flags = TIMERFD_GHOST_ENABLED; + + if (timerfd_ghost->flags & ~valid_flags) + return -EINVAL; + + /* ghost interaction disabled for this timerfd */ + if (!(timerfd_ghost->flags & TIMERFD_GHOST_ENABLED)) + return 0; + + cpu = timerfd_ghost->cpu; + if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -EINVAL; + + rq = cpu_rq(cpu); + rq_lock_irqsave(rq, &rf); + + if (!rq->ghost.agent || !same_process(rq->ghost.agent, current)) { + rq_unlock_irqrestore(rq, &rf); + return -EINVAL; + } + + rq_unlock_irqrestore(rq, &rf); + return 0; +} + +/* fs/timerfd.c */ +int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, + struct itimerspec64 *old, struct timerfd_ghost *tfdl); + +static int ghost_timerfd_settime(int timerfd, int flags, + const struct __kernel_itimerspec __user *utmr, + struct __kernel_itimerspec __user *otmr, + struct timerfd_ghost __user *utfdl) +{ + struct timerfd_ghost timerfd_ghost; + struct itimerspec64 new, old; + int ret; + + if (get_itimerspec64(&new, utmr)) + return -EFAULT; + + if (copy_from_user(&timerfd_ghost, utfdl, sizeof(timerfd_ghost))) + return -EFAULT; + + ret = ghost_timerfd_validate(&timerfd_ghost); + if (ret) + return ret; + + ret = do_timerfd_settime(timerfd, flags, &new, &old, &timerfd_ghost); + if (ret) + return ret; + + if (otmr && put_itimerspec64(&old, otmr)) + return -EFAULT; + + return ret; +} + +/* Called from timerfd_triggered() on timer expiry */ +void ghost_timerfd_triggered(struct timerfd_ghost *timerfd_ghost) +{ + struct rq *rq; + struct rq_flags rf; + int cpu = timerfd_ghost->cpu; + + if (WARN_ON_ONCE(cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu))) + return; + + rq = cpu_rq(cpu); + rq_lock_irqsave(rq, &rf); + + if (cpu_deliver_timer_expired(rq, timerfd_ghost->cookie)) + ghost_wake_agent_on(agent_target_cpu(rq)); + + rq_unlock_irqrestore(rq, &rf); +} + +static int ghost_gtid_lookup(int64_t id, int op, int flags, int64_t __user *out) +{ + int error = 0; + int64_t result; + struct task_struct *target; + + if (flags) + return -EINVAL; + + rcu_read_lock(); + target = find_task_by_gtid(id); + if (!target) { + rcu_read_unlock(); + return -ESRCH; + } + + switch (op) { + case GHOST_GTID_LOOKUP_TGID: + result = task_tgid_nr(target); + break; + default: + error = -EINVAL; + break; + } + rcu_read_unlock(); + + if (!error) { + if (copy_to_user(out, &result, sizeof(result))) + error = -EFAULT; + } + + return error; +} + +static int ghost_get_gtid(int64_t __user *out) +{ + gtid_t gtid = current->gtid; + + if (copy_to_user(out, >id, sizeof(gtid))) + return -EFAULT; + + return 0; +} + +/* Null syscall for benchmarking */ +static int ghost_null(bool lookup_e, int e_fd) +{ + struct ghost_enclave *e; + struct fd f_enc = {0}; + int ret = 0; + + if (!lookup_e) + return 0; + e = ghost_fdget_enclave(e_fd, &f_enc); + if (!e) + ret = -EBADF; + ghost_fdput_enclave(e, &f_enc); + return ret; +} + +/* + * TODO: Access to the ghost syscalls needs to be restricted, probably via a + * capability. + */ +SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, + u64, arg3, u64, arg4, u64, arg5) +{ + bool be_nice = true; + + if (op == GHOST_GET_GTID_10 || op == GHOST_GET_GTID_11 || + op == GHOST_BASE_GET_GTID) + be_nice = false; + + if (be_nice && !capable(CAP_SYS_NICE)) + return -EPERM; + + switch (op) { + case GHOST_NULL: + return ghost_null(arg1, arg2); + case GHOST_CREATE_QUEUE: + return ghost_create_queue(arg1, arg2, arg3, + (ulong __user *)arg4, arg5); + case GHOST_ASSOCIATE_QUEUE: + return ghost_associate_queue(arg1, (void *)arg2, arg3, arg4, + (int __user *)arg5); + case GHOST_SET_DEFAULT_QUEUE: + return ghost_set_default_queue(arg1); + case GHOST_CONFIG_QUEUE_WAKEUP: + return ghost_config_queue_wakeup(arg1, + (struct ghost_agent_wakeup __user *) arg2, + arg3, arg4); + case GHOST_SET_OPTION: + return ghost_set_option(arg1, arg2, arg3, arg4); + case GHOST_GET_CPU_TIME: + return ghost_get_cpu_time((gtid_t)arg1, (u64 __user *)arg2); + case GHOST_COMMIT_TXN: + return gsys_ghost_commit_txn((ulong __user *)arg1, arg2, arg3); + case GHOST_SYNC_GROUP_TXN: + return gsys_ghost_sync_group((ulong __user *)arg1, arg2, arg3); + case GHOST_TIMERFD_SETTIME: + return ghost_timerfd_settime(arg1, arg2, + (const struct __kernel_itimerspec __user *)arg3, + (struct __kernel_itimerspec __user *)arg4, + (struct timerfd_ghost __user *)arg5); + case GHOST_GTID_LOOKUP: + return ghost_gtid_lookup(arg1, arg2, arg3, + (int64_t __user *)arg4); + case GHOST_GET_GTID_10: + case GHOST_GET_GTID_11: + case GHOST_BASE_GET_GTID: + return ghost_get_gtid((int64_t __user *)arg1); + default: + if (op >= _GHOST_BASE_OP_FIRST) + return -EOPNOTSUPP; + else + return -EINVAL; + } +} + +#ifndef SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK +#define SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK 0x10000 +#endif + +void ghost_switchto(struct rq *rq, struct task_struct *prev, + struct task_struct *next, int switchto_flags) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(prev != rq->curr); + VM_BUG_ON(prev->state == TASK_RUNNING); + VM_BUG_ON(next->state == TASK_RUNNING); + VM_BUG_ON(!ghost_class(prev->sched_class)); + VM_BUG_ON(!ghost_class(next->sched_class)); + VM_BUG_ON(rq->ghost.check_prev_preemption); + VM_BUG_ON(rq->ghost.switchto_count < 0); + + if (switchto_flags & SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK) { + next->se.exec_start = prev->se.exec_start; + } else { + update_curr_ghost(rq); + next->se.exec_start = rq_clock_task(rq); + } + + list_del_init(&prev->ghost.run_list); + ghost_sw_clear_flag(prev->ghost.status_word, GHOST_SW_TASK_RUNNABLE); + + list_add_tail(&next->ghost.run_list, &rq->ghost.tasks); + next->ghost.last_runnable_at = 0; /* we're on_cpu */ + ghost_sw_set_flag(next->ghost.status_word, + GHOST_SW_TASK_RUNNABLE | GHOST_SW_TASK_ONCPU); + + if (++rq->ghost.switchto_count == 1) { + /* + * Produce MSG_TASK_SWITCHTO but don't wake up the agent. + * In per-cpu models, agent wakeup will preempt the task + * and break the switchto chain before it even gets started. + */ + task_deliver_msg_switchto(rq, prev); + } +} + +void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next) +{ + if (cpu_deliver_msg_not_idle(rq, next)) + ghost_wake_agent_on(agent_target_cpu(rq)); +} + +unsigned long ghost_cfs_added_load(struct rq *rq) +{ + int ghost_nr_running = rq->ghost.ghost_nr_running; + struct task_struct *curr; + bool add_load = false; + + /* No ghost tasks; nothing to contribute load. */ + if (!ghost_nr_running) + return 0; + + /* + * We have a few cases where we want to add load: + * (a): We have a local agent that is not blocked_in_run. + * (b): Currently have a non-agent ghost task running. + * (c): Have a latched task that is not yet running. We + * treat this the same as case (b), since this is really + * just a race over getting through schedule() (modulo possible + * preemption by another sched_class). + */ + + if (ghost_nr_running > __ghost_extra_nr_running(rq)) { + /* (a) */ + add_load = true; + goto out; + } + + rcu_read_lock(); + curr = READ_ONCE(rq->curr); + if (task_has_ghost_policy(curr) && !is_agent(rq, curr) && + curr->state == TASK_RUNNING) { + /* (b) */ + add_load = true; + } + curr = NULL; /* don't use outside of RCU */ + rcu_read_unlock(); + if (add_load) + goto out; + + if (rq->ghost.latched_task) { + /* (c) */ + add_load = true; + } + +out: + if (add_load) + return sysctl_ghost_cfs_load_added; + return 0; +} diff --git a/kernel/sched/ghost.h b/kernel/sched/ghost.h new file mode 100644 index 000000000000..9093627f89a2 --- /dev/null +++ b/kernel/sched/ghost.h @@ -0,0 +1,18 @@ +/* + * Copyright 2021 Google LLC + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* Stub functions for open source ghOSt code. */ + +/* These functions should not be open sourced. */ +static inline void kvm_register_core_conflict(int cpu) {} +static inline bool try_ipiless_wakeup(int cpu) { return false; } diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c new file mode 100644 index 000000000000..5a4aecbef564 --- /dev/null +++ b/kernel/sched/ghostfs.c @@ -0,0 +1,903 @@ +/* + * Copyright 2021 Google LLC + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "sched.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../fs/kernfs/kernfs-internal.h" + +/* Helper for when you "echo foo > ctl" without the -n. */ +static void strip_slash_n(char *buf, size_t count) +{ + char *n = strnchr(buf, count, '\n'); + + if (n) + *n = '\0'; +} + +static struct kernfs_root *ghost_kfs_root; + +struct gf_dirent { + char *name; + umode_t mode; + struct kernfs_ops *ops; + loff_t size; + bool is_dir; +}; + +/* + * Finds a specific file in a directory table. This is useful for runtime + * initialization, though keep in mind the dirtab is a global structure. + */ +static struct gf_dirent *gf_find_file(struct gf_dirent *dirtab, + const char *name) +{ + struct gf_dirent *gft; + + for (gft = dirtab; gft->name; gft++) { + if (!strcmp(gft->name, name)) + return gft; + } + return NULL; +} + +/* + * Every open file in an enclave holds a kref on the enclave. This includes the + * ctl, txn, the status words, etc. + * + * Each of those file's priv points to the enclave struct, either directly or + * indirectly. For instance, the SW files have their own object that points to + * the enclave. We can differentiate which file is which based on the kernfs + * ops. In some cases, we can reuse ops and differentiate on the object's name. + * + * This does not include the enclave directory; it's priv is NULL, and it has no + * function ops. + * + * Destroying the enclave involves removing it from the filesystem by writing + * "destroy" into ctl. That triggers kernfs_remove() and whatever things we + * want to do for the enclave itself, like killing the agents. The enclave + * object will exist until the last FD is closed. Note that when we call + * kernfs_remove(), it'll "drain" any open files, which involves unmapping any + * mmapped files. + * + * Most of the heavy lifting for destroying an enclave is done in + * ghost_destroy_enclave(). The kernel scheduler may call that function at any + * moment if it determines an enclave misbehaved. + */ + +static struct ghost_enclave *of_to_e(struct kernfs_open_file *of) +{ + return of->kn->priv; +} + +static struct ghost_enclave *seq_to_e(struct seq_file *sf) +{ + struct kernfs_open_file *of = sf->private; + + return of_to_e(of); +} + +/* For enclave files whose priv directly points to a ghost enclave. */ +static int gf_e_open(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + + kref_get(&e->kref); + return 0; +} + +/* For enclave files whose priv directly points to a ghost enclave. */ +static void gf_e_release(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + + kref_put(&e->kref, enclave_release); +} + +/* + * There can be at most one writable open, and there can be any number of + * read-only opens. + * + * If there is a writable open, that means userspace has advertised this enclave + * as having an agent_online, and the value of agent_online will change + * accordingly. + */ +static int gf_agent_online_open(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long irq_fl; + + if (!(of->file->f_mode & FMODE_WRITE)) + goto done; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->agent_online) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + e->agent_online = true; + spin_unlock_irqrestore(&e->lock, irq_fl); + + kernfs_notify(of->kn); + +done: + kref_get(&e->kref); + return 0; +} + +static void gf_agent_online_release(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long irq_fl; + + if (!(of->file->f_mode & FMODE_WRITE)) + goto done; + + spin_lock_irqsave(&e->lock, irq_fl); + WARN_ONCE(!e->agent_online, + "closing RW agent_online for enclave_%lu, but it's not online!", + e->id); + e->agent_online = false; + spin_unlock_irqrestore(&e->lock, irq_fl); + + /* + * Kicks any epoll/notifiers about the change in state. This is the + * "agent death edge". + * + * Note that this doesn't necessarily mean the enclave will be + * destroyed, merely that userspace no longer thinks there is a valid + * agent. This FD was probably closed because the agent crashed. + */ + kernfs_notify(of->kn); + +done: + kref_put(&e->kref, enclave_release); +} + +static int gf_agent_online_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%u", e->agent_online ? 1 : 0); + return 0; +} + +static ssize_t gf_agent_online_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + /* We need a write op so kernfs allows us to be opened for writing. */ + return -EINVAL; +} + +static struct kernfs_ops gf_ops_e_agent_online = { + .open = gf_agent_online_open, + .release = gf_agent_online_release, + .seq_show = gf_agent_online_show, + .write = gf_agent_online_write, +}; + +static int gf_cpu_data_mmap(struct kernfs_open_file *of, + struct vm_area_struct *vma) +{ + struct ghost_enclave *e = of_to_e(of); + + return ghost_cpu_data_mmap(of->file, vma, e->cpu_data, + of->kn->attr.size); +} + +static struct kernfs_ops gf_ops_e_cpu_data = { + .open = gf_e_open, + .release = gf_e_release, + .mmap = gf_cpu_data_mmap, +}; + +/* Returns an ASCII list of the cpus in the enclave */ +static ssize_t gf_cpulist_read(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long fl; + cpumask_var_t cpus; + char *pagebuf; + ssize_t strlen; + + if (off > PAGE_SIZE) + return -EINVAL; + bytes = min_t(size_t, bytes, PAGE_SIZE - off); + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + spin_lock_irqsave(&e->lock, fl); + memcpy(cpus, &e->cpus, cpumask_size()); + spin_unlock_irqrestore(&e->lock, fl); + + pagebuf = (char *)get_zeroed_page(GFP_KERNEL); + if (!pagebuf) { + free_cpumask_var(cpus); + return -ENOMEM; + } + + strlen = cpumap_print_to_pagebuf(/*list=*/true, pagebuf, cpus); + bytes = memory_read_from_buffer(buf, bytes, &off, pagebuf, strlen); + + free_page((unsigned long)pagebuf); + free_cpumask_var(cpus); + + return bytes; +} + + +static ssize_t __cpu_set_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off, bool is_list) +{ + cpumask_var_t cpus; + int err; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + if (is_list) + err = cpulist_parse(buf, cpus); + else + err = cpumask_parse(buf, cpus); + if (err) { + free_cpumask_var(cpus); + return err; + } + err = ghost_enclave_set_cpus(of_to_e(of), cpus); + free_cpumask_var(cpus); + return err ? err : bytes; +} + +/* Sets the enclave's cpus to buf, an ASCII list of cpus. */ +static ssize_t gf_cpulist_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + return __cpu_set_write(of, buf, bytes, off, /*is_list=*/true); +} + +static struct kernfs_ops gf_ops_e_cpulist = { + .open = gf_e_open, + .release = gf_e_release, + .read = gf_cpulist_read, + .write = gf_cpulist_write, +}; + +/* Returns an ASCII hex mask of the cpus in the enclave */ +static ssize_t gf_cpumask_read(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long fl; + cpumask_var_t cpus; + char *mask_str; + int len; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + spin_lock_irqsave(&e->lock, fl); + memcpy(cpus, &e->cpus, cpumask_size()); + spin_unlock_irqrestore(&e->lock, fl); + + /* + * +1 for the \0. We won't return the \0 to userspace, but the string + * will be null-terminated while in the kernel. + */ + len = snprintf(NULL, 0, "%*pb\n", cpumask_pr_args(cpus)) + 1; + mask_str = kmalloc(len, GFP_KERNEL); + if (!mask_str) { + free_cpumask_var(cpus); + return -ENOMEM; + } + len = snprintf(mask_str, len, "%*pb\n", cpumask_pr_args(cpus)); + bytes = memory_read_from_buffer(buf, bytes, &off, mask_str, len); + + kfree(mask_str); + free_cpumask_var(cpus); + + return bytes; +} + +/* Sets the enclave's cpus to buf, an ASCII hex cpumask. */ +static ssize_t gf_cpumask_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + return __cpu_set_write(of, buf, bytes, off, /*is_list=*/false); +} + +static struct kernfs_ops gf_ops_e_cpumask = { + .open = gf_e_open, + .release = gf_e_release, + .read = gf_cpumask_read, + .write = gf_cpumask_write, +}; + +static int gf_ctl_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%lu", e->id); + return 0; +} + +/* Called from the scheduler when it destroys the enclave. */ +void ghostfs_remove_enclave(struct ghost_enclave *e) +{ + kernfs_remove(e->enclave_dir); +} + +static void destroy_enclave(struct kernfs_open_file *ctl_of) +{ + struct kernfs_node *ctl = ctl_of->kn; + struct ghost_enclave *e = of_to_e(ctl_of); + + /* + * kernfs_remove() works recursively, removing the entire enclave + * directory. We need to remove the ctl file first, since we're in the + * middle of a kn op. + * + * Multiple threads can call kernfs_remove_self() at once. Whichever + * succeeds will remove the directory and release e. + */ + if (!kernfs_remove_self(ctl)) + return; + if (WARN_ON(ctl->parent != e->enclave_dir)) + return; + ghost_destroy_enclave(e); +} + +static struct ghost_sw_region *of_to_swr(struct kernfs_open_file *of) +{ + return of->kn->priv; +} + +/* + * swr_kns refcount the enclave, even though they point to the swr. An swr will + * never be deleted until the enclave is released. + */ +static int gf_swr_open(struct kernfs_open_file *of) +{ + struct ghost_sw_region *swr = of_to_swr(of); + + kref_get(&swr->enclave->kref); + return 0; +} + +static void gf_swr_release(struct kernfs_open_file *of) +{ + struct ghost_sw_region *swr = of_to_swr(of); + + kref_put(&swr->enclave->kref, enclave_release); +} + +static int gf_swr_mmap(struct kernfs_open_file *of, struct vm_area_struct *vma) +{ + struct ghost_sw_region *swr = of_to_swr(of); + + if (vma->vm_flags & VM_WRITE) + return -EINVAL; + vma->vm_flags &= ~VM_MAYWRITE; + + return ghost_region_mmap(of->file, vma, swr->header, of->kn->attr.size); +} + +static struct kernfs_ops gf_ops_e_swr = { + .open = gf_swr_open, + .release = gf_swr_release, + .mmap = gf_swr_mmap, +}; + +static int create_sw_region(struct kernfs_open_file *ctl_of, unsigned int id, + unsigned int numa_node) +{ + struct ghost_enclave *e = of_to_e(ctl_of); + struct kernfs_node *dir, *swr_kn; + struct ghost_sw_region *swr; + char name[31]; + int err; + + dir = kernfs_find_and_get(e->enclave_dir, "sw_regions"); + if (WARN_ON_ONCE(!dir)) + return -EINVAL; + + if (snprintf(name, sizeof(name), "sw_%u", id) >= sizeof(name)) { + err = -ENOSPC; + goto err_snprintf; + } + + swr_kn = kernfs_create_file(dir, name, 0440, 0, &gf_ops_e_swr, e); + if (IS_ERR(swr_kn)) { + err = PTR_ERR(swr_kn); + goto err_create_kn; + } + /* swr_kn is inaccessible until kernfs_activate. */ + + swr = ghost_create_sw_region(e, id, numa_node); + if (IS_ERR(swr)) { + err = PTR_ERR(swr); + goto err_create_swr; + } + + swr_kn->attr.size = swr->mmap_sz; + swr_kn->priv = swr; + kernfs_activate(swr_kn); + + return 0; +err_create_swr: + kernfs_remove(swr_kn); +err_create_kn: +err_snprintf: + kernfs_put(dir); + return err; +} + +static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + unsigned int arg1, arg2; + int err; + + /* + * Ignore the offset for ctl commands, so userspace doesn't have to + * worry about lseeking after each command. Userspace should submit a + * single write per command too, and not write("des"), write ("troy"). + * Otherwise they'll fail. + */ + + strip_slash_n(buf, len); + + if (!strcmp(buf, "destroy")) { + destroy_enclave(of); + } else if (sscanf(buf, "create sw_region %u %u", &arg1, &arg2) == 2) { + err = create_sw_region(of, arg1, arg2); + if (err) + return err; + } else { + pr_err("%s: bad cmd :%s:", __func__, buf); + return -EINVAL; + } + + return len; +} + +static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, + unsigned long arg) +{ + struct ghost_enclave *e = of_to_e(of); + + switch (cmd) { + case GHOST_IOC_NULL: + return 0; + case GHOST_IOC_SW_GET_INFO: + return ghost_sw_get_info(e, + (struct ghost_ioc_sw_get_info __user *)arg); + case GHOST_IOC_SW_FREE: + return ghost_sw_free(e, (void __user *)arg); + } + return -ENOIOCTLCMD; +} + +static struct kernfs_ops gf_ops_e_ctl = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_ctl_show, + .write = gf_ctl_write, + .ioctl = gf_ctl_ioctl, +}; + +/* + * Returns the enclave for f, if f is a ghostfs ctl file. We could support + * other file types, but since this is a backdoor into the FS, we only need to + * support ctl. + * + * Successful callers must call ghostfs_put_ctl_enclave(f). + * + * The kernfs_ops don't need this helper. kernfs manages the the refcounts. We + * need to do it manually here, because this is is a "backdoor" function to get + * the enclave pointer. That pointer is kept alive by kernfs. + */ +struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) +{ + struct kernfs_node *kn; + + kn = kernfs_node_from_file(f); + if (!kn) + return NULL; + if (kernfs_type(kn) != KERNFS_FILE) + return NULL; + if (kn->attr.ops != &gf_ops_e_ctl) + return NULL; + if (!kernfs_get_active(kn)) + return NULL; + WARN_ON(!kn->priv); + return kn->priv; +} + +/* Pair this with a successful ghostfs_ctl_to_enclave call. */ +void ghostfs_put_enclave_ctl(struct file *f) +{ + struct kernfs_node *kn; + + kn = kernfs_node_from_file(f); + if (WARN_ON(!kn)) + return; + if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) + return; + if (WARN_ON(kn->attr.ops != &gf_ops_e_ctl)) + return; + kernfs_put_active(kn); +} + +static int gf_runnable_timeout_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%lld", ktime_to_ms(READ_ONCE(e->max_unscheduled))); + return 0; +} + +static ssize_t gf_runnable_timeout_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + unsigned long msec; + + err = kstrtoul(buf, 0, &msec); + if (err) + return -EINVAL; + + WRITE_ONCE(e->max_unscheduled, ms_to_ktime(msec)); + + return len; +} + +static struct kernfs_ops gf_ops_e_runnable_timeout = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_runnable_timeout_show, + .write = gf_runnable_timeout_write, +}; + +static int gf_status_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + unsigned long fl; + bool is_active; + + /* + * Userspace uses this to find any active enclave, since they don't have + * any other methods yet to know which enclave to use. + */ + spin_lock_irqsave(&e->lock, fl); + /* + * We don't need to lock to read agent_online, but eventually we'll + * check for the presence of an interstitial scheduler too. This status + * is for the *enclave*, not the *agent*. + */ + is_active = e->agent_online; + spin_unlock_irqrestore(&e->lock, fl); + + seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); + return 0; +} + +static struct kernfs_ops gf_ops_e_status = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_status_show, +}; + +static struct gf_dirent enclave_dirtab[] = { + { + .name = "sw_regions", + .mode = 0555, + .is_dir = true, + }, + { + .name = "agent_online", + .mode = 0664, + .ops = &gf_ops_e_agent_online, + }, + { + .name = "cpu_data", + .mode = 0660, + .ops = &gf_ops_e_cpu_data, + }, + { + .name = "cpulist", + .mode = 0664, + .ops = &gf_ops_e_cpulist, + }, + { + .name = "cpumask", + .mode = 0664, + .ops = &gf_ops_e_cpumask, + }, + { + .name = "ctl", + .mode = 0664, + .ops = &gf_ops_e_ctl, + }, + { + .name = "runnable_timeout", + .mode = 0664, + .ops = &gf_ops_e_runnable_timeout, + }, + { + .name = "status", + .mode = 0444, + .ops = &gf_ops_e_status, + }, + {0}, +}; + +/* Caller is responsible for cleanup. Removing the parent will suffice. */ +static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, + struct ghost_enclave *priv) +{ + struct gf_dirent *gft; + struct kernfs_node *kn; + + for (gft = dirtab; gft->name; gft++) { + if (gft->is_dir) { + kn = kernfs_create_dir(parent, gft->name, gft->mode, + NULL); + } else { + kn = kernfs_create_file(parent, gft->name, gft->mode, + gft->size, gft->ops, priv); + } + if (IS_ERR(kn)) + return PTR_ERR(kn); + } + return 0; +} + +static int make_enclave(struct kernfs_node *parent, unsigned long id) +{ + struct kernfs_node *dir; + struct ghost_enclave *e; + char name[31]; + int ret; + + /* + * ghost_create_enclave() is mostly just "alloc and initialize". + * Anything done by it gets undone in enclave_release, and it is not + * discoverable, usable, or otherwise hooked into the kernel until + * kernfs_active(). + */ + e = ghost_create_enclave(); + if (!e) + return -ENOMEM; + e->id = id; + if (snprintf(name, sizeof(name), "enclave_%lu", id) >= sizeof(name)) { + ret = -ENOSPC; + goto out_e; + } + + dir = kernfs_create_dir(parent, name, 0555, NULL); + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + goto out_e; + } + e->enclave_dir = dir; + + ret = gf_add_files(dir, enclave_dirtab, e); + if (ret) + goto out_dir; + + /* + * Once the enclave has been activated, it is available to userspace and + * can be used for scheduling. After that, we must destroy it by + * calling ghost_destroy_enclave(), not by releasing the reference. + */ + kernfs_activate(dir); /* recursive */ + + return 0; + +out_dir: + kernfs_remove(dir); /* recursive */ +out_e: + kref_put(&e->kref, enclave_release); + return ret; +} + +static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct kernfs_node *ctl = of->kn; + struct kernfs_node *top_dir = ctl->parent; + unsigned long x; + int ret; + + strip_slash_n(buf, len); + + /* This will ignore any extra digits or characters beyond the %u. */ + ret = sscanf(buf, "create %lu", &x); + if (ret == 1) { + ret = make_enclave(top_dir, x); + return ret ? ret : len; + } + + return -EINVAL; +} + +static struct kernfs_ops gf_ops_top_ctl = { + .write = gf_top_ctl_write, +}; + +static int gf_top_version_show(struct seq_file *sf, void *v) +{ + seq_printf(sf, "%u", GHOST_VERSION); + return 0; +} + +static struct kernfs_ops gf_ops_top_version = { + .seq_show = gf_top_version_show, +}; + +static struct gf_dirent top_dirtab[] = { + { + .name = "ctl", + .mode = 0660, + .ops = &gf_ops_top_ctl, + }, + { + .name = "version", + .mode = 0444, + .ops = &gf_ops_top_version, + }, + {0} +}; + +/* + * Most gf_dirent file sizes are not known at compile time. Most don't matter + * for sysfs and we can leave them as 0. But for anything that gets mmapped, + * it's convenient for userspace for the kernel to say how big it is. + */ +static void runtime_adjust_dirtabs(void) +{ + struct gf_dirent *enc_txn; + + enc_txn = gf_find_file(enclave_dirtab, "cpu_data"); + if (WARN_ON(!enc_txn)) + return; + enc_txn->size = GHOST_CPU_DATA_REGION_SIZE; +} + +static int __init ghost_setup_root(void) +{ + int ret; + struct kernfs_root *fs_root; + + fs_root = kernfs_create_root(NULL, KERNFS_ROOT_CREATE_DEACTIVATED | + KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, NULL); + if (IS_ERR(fs_root)) + return PTR_ERR(fs_root); + + ret = gf_add_files(fs_root->kn, top_dirtab, NULL); + if (ret) { + kernfs_destroy_root(fs_root); + return ret; + } + + ghost_kfs_root = fs_root; + + runtime_adjust_dirtabs(); + + kernfs_activate(ghost_kfs_root->kn); + + return ret; +} + +static int ghost_get_tree(struct fs_context *fc) +{ + int ret; + + ret = kernfs_get_tree(fc); + if (ret) + return ret; + + return 0; +} + +static void ghost_fs_context_free(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + + kernfs_free_fs_context(fc); + kfree(kfc); +} + +static const struct fs_context_operations ghost_fs_context_ops = { + .free = ghost_fs_context_free, + .get_tree = ghost_get_tree, +}; + +static int ghost_init_fs_context(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc; + + /* Technically, this should be in uapi/linux/magic.h. */ + #define GHOST_SUPER_MAGIC 0xBAD1DEA2 + + kfc = kzalloc(sizeof(struct kernfs_fs_context), GFP_KERNEL); + if (!kfc) + return -ENOMEM; + + kfc->root = ghost_kfs_root; + kfc->magic = GHOST_SUPER_MAGIC; + fc->fs_private = kfc; + fc->ops = &ghost_fs_context_ops; + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(&init_user_ns); + fc->global = true; + return 0; +} + +static void ghost_kill_sb(struct super_block *sb) +{ + kernfs_kill_sb(sb); +} + +static struct file_system_type ghost_fs_type = { + .name = "ghost", + .init_fs_context = ghost_init_fs_context, + .kill_sb = ghost_kill_sb, +}; + +static int __init ghostfs_init(void) +{ + int ret = 0; + + ret = ghost_setup_root(); + if (ret) + return ret; + + ret = sysfs_create_mount_point(fs_kobj, "ghost"); + if (ret) + goto cleanup_root; + + ret = register_filesystem(&ghost_fs_type); + if (ret) + goto cleanup_mountpoint; + + return 0; + +cleanup_mountpoint: + sysfs_remove_mount_point(fs_kobj, "ghost"); +cleanup_root: + kernfs_destroy_root(ghost_kfs_root); + ghost_kfs_root = NULL; + + return ret; +} + +static void __exit ghostfs_exit(void) +{ + unregister_filesystem(&ghost_fs_type); + sysfs_remove_mount_point(fs_kobj, "ghost"); + kernfs_destroy_root(ghost_kfs_root); +} + +late_initcall(ghostfs_init); +__exitcall(ghostfs_exit); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bb09988451a0..0efec2eec0a8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -99,6 +99,186 @@ extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); extern void call_trace_sched_update_nr_running(struct rq *rq, int count); + +#ifdef CONFIG_SCHED_CLASS_GHOST + +#include + +struct ghost_rq { + struct task_struct *agent; /* protected by e->lock and rq->lock */ + uint32_t agent_barrier; + bool agent_remove_enclave_cpu; /* protected by e->lock */ + bool blocked_in_run; /* agent is blocked in 'ghost_run()' */ + bool agent_should_wake; + bool must_resched; /* rq->curr must reschedule in PNT */ + bool check_prev_preemption; /* see 'ghost_prepare_task_switch()' */ + int ghost_nr_running; + int run_flags; /* flags passed to 'ghost_run()' */ + + /* For deferring work to the balance_callback */ + struct list_head enclave_work; /* work to do */ + struct callback_head ew_head; /* callback management */ + + struct list_head tasks; + + struct task_struct *latched_task; /* task returned by pick_next_task */ + + long switchto_count; + + /* + * zero not participating in a sync-group rendezvous. + * negative sync-group in process of committing. + * positive sync-group successfully committed. + * + * Thus a CPU must not return from __schedule() as long as + * 'rq->ghost.rendezvous' is negative. + */ + int64_t rendezvous; +}; + +struct ghost_sw_region { + struct list_head list; /* ghost_enclave glue */ + uint32_t alloc_scan_start; /* allocator starts scan here */ + struct ghost_sw_region_header *header; /* pointer to vmalloc memory */ + size_t mmap_sz; /* size of mmapped region */ + struct ghost_enclave *enclave; +}; + +#define GHOST_MAX_SW_REGIONS 64 +#define GHOST_CPU_DATA_REGION_SIZE \ + (sizeof(struct ghost_cpu_data) * num_possible_cpus()) + +struct enclave_work { + struct list_head link; + unsigned int nr_decrefs; + bool run_task_reaper; +}; + +/* + * ghost_enclave is a container for the agents, queues and sw_regions + * that express the scheduling policy for a set of CPUs. + */ +struct ghost_enclave { + /* + * 'lock' serializes mutation of 'sw_region_list' as well as + * allocation and freeing of status words within a region. + * + * 'lock' also serializes mutation of 'def_q'. + * + * 'lock' requires the irqsave variant of spin_lock because + * it is called in code paths with the 'rq->lock' held and + * interrupts disabled. + */ + spinlock_t lock; + struct kref kref; + struct list_head sw_region_list; + ulong sw_region_ids[BITS_TO_LONGS(GHOST_MAX_SW_REGIONS)]; + + struct ghost_cpu_data **cpu_data; + struct cpumask cpus; + + struct ghost_queue *def_q; /* default queue */ + + struct list_head task_list; /* all non-agent tasks in the enclave */ + struct work_struct task_reaper; + struct enclave_work ew; /* to defer work while holding locks */ + struct work_struct enclave_actual_release;/* work for enclave_release */ + + /* + * max_unscheduled: How long a task can be runnable, but unscheduled, + * before the kernel thinks the enclave failed and queues the + * enclave_destroyer. + */ + ktime_t max_unscheduled; + struct work_struct enclave_destroyer; + + unsigned long id; + bool is_dying; + bool agent_online; /* userspace says agent can schedule. */ + struct kernfs_node *enclave_dir; +}; + +/* In kernel/sched/ghostfs.c */ +extern struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); +extern void ghostfs_put_enclave_ctl(struct file *f); +extern void ghostfs_remove_enclave(struct ghost_enclave *e); + +/* In kernel/sched/ghost.c */ +extern bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev); +extern struct ghost_enclave *ghost_create_enclave(void); +extern void enclave_release(struct kref *k); +extern void ghost_destroy_enclave(struct ghost_enclave *e); +extern int ghost_enclave_set_cpus(struct ghost_enclave *e, + const struct cpumask *cpus); +extern int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, + void *addr, ulong mapsize); +extern int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, + struct ghost_cpu_data **cpu_data, ulong mapsize); +extern struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, + unsigned int id, + unsigned int node); +extern int ghost_sw_get_info(struct ghost_enclave *e, + struct ghost_ioc_sw_get_info __user *arg); +extern int ghost_sw_free(struct ghost_enclave *e, + struct ghost_sw_info __user *uinfo); +extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); +extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); + +extern void init_sched_ghost_class(void); +extern void init_ghost_rq(struct ghost_rq *ghost_rq); +extern bool ghost_agent(const struct sched_attr *attr); +extern int ghost_validate_sched_attr(const struct sched_attr *attr); +extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, + const struct sched_attr *attr, + struct ghost_enclave *new_e, + int *reset_on_fork); +extern int ghost_sched_fork(struct task_struct *p); +extern void ghost_sched_cleanup_fork(struct task_struct *p); +extern void ghost_latched_task_preempted(struct rq *rq); +extern void ghost_task_preempted(struct rq *rq, struct task_struct *prev); +extern unsigned long ghost_cfs_added_load(struct rq *rq); +extern void ghost_wake_agent_on(int cpu); +extern void ghost_wake_agent_of(struct task_struct *p); +extern void ghost_agent_schedule(void); +extern bool bpf_sched_ghost_skip_tick(void); +extern void ghost_wait_for_rendezvous(struct rq *rq); +extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); +extern void ghost_tick(struct rq *rq); +int64_t ghost_alloc_gtid(struct task_struct *p); +extern void ghost_switchto(struct rq *rq, struct task_struct *prev, + struct task_struct *next, int switchto_flags); + +static inline void sched_ghost_entity_init(struct task_struct *p) +{ + memset(&p->ghost, 0, sizeof(p->ghost)); + INIT_LIST_HEAD(&p->ghost.run_list); + INIT_LIST_HEAD(&p->ghost.task_list); +} + +static inline void ghost_sw_set_flag(struct ghost_status_word *sw, + uint32_t flag) { + smp_store_release(&sw->flags, sw->flags | flag); +} + +static inline void ghost_sw_clear_flag(struct ghost_status_word *sw, + uint32_t flag) { + smp_store_release(&sw->flags, sw->flags & ~flag); +} + +static inline int ghost_schedattr_to_enclave_fd(const struct sched_attr *attr) +{ + return attr->sched_runtime; +} + +static inline int ghost_schedattr_to_queue_fd(const struct sched_attr *attr) +{ + return attr->sched_deadline; +} + +#else +static inline unsigned long ghost_cfs_added_load(struct rq *rq) { return 0; } +#endif /* CONFIG_SCHED_CLASS_GHOST */ + /* * Helpers for converting nanosecond timing to jiffy resolution */ @@ -175,10 +355,17 @@ static inline int dl_policy(int policy) { return policy == SCHED_DEADLINE; } + +static inline bool ghost_policy(int policy) +{ + return policy == SCHED_GHOST; +} + static inline bool valid_policy(int policy) { return idle_policy(policy) || fair_policy(policy) || - rt_policy(policy) || dl_policy(policy); + rt_policy(policy) || dl_policy(policy) || + ghost_policy(policy); } static inline int task_has_idle_policy(struct task_struct *p) @@ -196,6 +383,11 @@ static inline int task_has_dl_policy(struct task_struct *p) return dl_policy(p->policy); } +static inline int task_has_ghost_policy(struct task_struct *p) +{ + return ghost_policy(p->policy); +} + #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) static inline void update_avg(u64 *avg, u64 sample) @@ -414,6 +606,10 @@ struct task_group { struct cfs_bandwidth cfs_bandwidth; +#ifdef CONFIG_SCHED_CLASS_GHOST + bool ghost_enabled; +#endif + #ifdef CONFIG_UCLAMP_TASK_GROUP /* The two decimal precision [%] value requested from user-space */ unsigned int uclamp_pct[UCLAMP_CNT]; @@ -933,6 +1129,9 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; struct dl_rq dl; +#ifdef CONFIG_SCHED_CLASS_GHOST + struct ghost_rq ghost; +#endif #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this CPU: */ @@ -1051,6 +1250,10 @@ struct rq { unsigned int ttwu_local; #endif +#ifdef CONFIG_SMP + int resched_ipi_work; +#endif + #ifdef CONFIG_CPU_IDLE /* Must be inspected within a rcu lock section */ struct cpuidle_state *idle_state; @@ -1417,11 +1620,15 @@ queue_balance_callback(struct rq *rq, { lockdep_assert_held(&rq->lock); + /* + * The last element on the list points to itself, so we can always + * detect if head is already enqueued. + */ if (unlikely(head->next || rq->balance_callback == &balance_push_callback)) return; head->func = (void (*)(struct callback_head *))func; - head->next = rq->balance_callback; + head->next = rq->balance_callback ?: head; rq->balance_callback = head; } @@ -1750,6 +1957,7 @@ static inline int task_on_rq_migrating(struct task_struct *p) #define WF_SYNC 0x10 /* Waker goes to sleep after wakeup */ #define WF_MIGRATED 0x20 /* Internal use, task got migrated */ #define WF_ON_CPU 0x40 /* Wakee is on_cpu */ +#define WF_DEFERRABLE_WAKEUP 0x80000 #ifdef CONFIG_SMP static_assert(WF_EXEC == SD_BALANCE_EXEC); @@ -1873,6 +2081,18 @@ struct sched_class { #endif }; +#ifdef CONFIG_SMP +static inline bool rq_has_resched_ipi_work(struct rq *rq) +{ + return rq->resched_ipi_work; +} +#else +static inline bool rq_has_resched_ipi_work(struct rq *rq) +{ + return false; +} +#endif + static inline void put_prev_task(struct rq *rq, struct task_struct *prev) { WARN_ON_ONCE(rq->curr != prev); @@ -1917,6 +2137,79 @@ extern const struct sched_class dl_sched_class; extern const struct sched_class rt_sched_class; extern const struct sched_class fair_sched_class; extern const struct sched_class idle_sched_class; +#ifdef CONFIG_SCHED_CLASS_GHOST +extern const struct sched_class ghost_agent_sched_class; +extern const struct sched_class ghost_sched_class; + +static inline bool ghost_class(const struct sched_class *class) +{ + return class == &ghost_sched_class; +} + +bool is_agent(struct rq *rq, struct task_struct *p); + +/* + * Contents of rq->ghost.rendezvous field: + * + * We have 1 bit for , 11 bits for , 1 bit for and + * 51 bits for + * + * Assuming a call rate of once per usec we get ~71 years before + * the 51-bit counter overflows. + */ +#define GHOST_NO_RENDEZVOUS 0 +#define GHOST_POISONED_RENDEZVOUS (1LL << 51) + +static inline bool rendezvous_reached(int64_t target) +{ + return target >= 0; +} + +static inline bool rendezvous_poisoned(int64_t target) +{ + if (rendezvous_reached(target)) { + /* + * We must have reached rendezvous before evaluating whether + * or not it is poisoned. This is actually an important check + * to avoid a false positive (an in-progress rendezvous is a + * negative number and GHOST_POISONED_RENDEZVOUS bit is set). + * + * For e.g. -2 is 0xfffffffffffffffe + */ + return target & GHOST_POISONED_RENDEZVOUS; + } + return false; +} + +static inline bool ghost_need_rendezvous(struct rq *rq) +{ + int64_t r; + + if (!ghost_class(rq->curr->sched_class)) + return false; + + r = smp_load_acquire(&rq->ghost.rendezvous); + return !rendezvous_reached(r) || rendezvous_poisoned(r); +} + +static inline bool skip_fair_idle_balance(struct cfs_rq *cfs_rq, + struct task_struct *prev) +{ + /* + * Skip fair idle balance iff: + * - there are no runnable CFS tasks on this cpu. + * - CFS was not already running on this cpu. + * + * In other words avoid attracting CFS tasks when a cpu is traversing + * the ghost->idle or idle->ghost edges. + */ + if (!cfs_rq->nr_running && prev->sched_class != &fair_sched_class) + return true; + else + return false; +} + +#endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) { @@ -2012,6 +2305,10 @@ extern void reweight_task(struct task_struct *p, int prio); extern void resched_curr(struct rq *rq); extern void resched_cpu(int cpu); +bool set_nr_and_not_polling(struct task_struct *p); +#ifdef CONFIG_SMP +extern void resched_cpu_unlocked(int cpu); +#endif extern struct rt_bandwidth def_rt_bandwidth; extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); @@ -2087,6 +2384,47 @@ static inline void sub_nr_running(struct rq *rq, unsigned count) sched_update_tick_dependency(rq); } +static inline int __ghost_extra_nr_running(struct rq *rq) +{ +#ifdef CONFIG_SCHED_CLASS_GHOST + int agent_active = 0; + + /* + * If a blocked ghost agent becomes runnable (blocked_in_run == false) + * when idle_balance() has dropped the rq->lock, it's possible that the + * Idle load balancer pulls CFS tasks which run before the agent gets a + * chance. In order to intercept this path and let the agent begin an + * inter-agent handoff before losing its CPU, we leave the agent's + * contribution in rq->nr_running. This causes the CFS pick_next_task to + * trigger a re-entry to the global pick_next_task loop, from where we + * can return back to the agent to initiate handoff. + * + * ghost_nr_running and rq->nr_running account for the agent + other + * ghost threads. Keep the agent accounted for in rq->nr_running, only + * while it is actively scheduling. + */ + if (rq->ghost.agent) { + if (task_on_rq_queued(rq->ghost.agent) && + !rq->ghost.blocked_in_run) + agent_active = 1; + } + + return rq->ghost.ghost_nr_running - agent_active; +#else + return 0; +#endif +} + +static inline int extra_nr_running(struct rq *rq) +{ + return __ghost_extra_nr_running(rq); +} + +static inline unsigned int rq_adj_nr_running(struct rq *rq) +{ + return rq->nr_running - extra_nr_running(rq); +} + extern void activate_task(struct rq *rq, struct task_struct *p, int flags); extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c9fbdd848138..745b75099967 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3139,6 +3139,37 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, +#endif +#ifdef CONFIG_SCHED_CLASS_GHOST + { + .procname = "ghost_cfs_load_added", + .data = &sysctl_ghost_cfs_load_added, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = SYSCTL_ZERO, + }, + { + .procname = "ghost_wake_on_waker_cpu", + .data = &sysctl_ghost_wake_on_waker_cpu, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "ghost_switchto_disable", + .data = &sysctl_ghost_switchto_disable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "ghost_commit_at_tick", + .data = &sysctl_ghost_commit_at_tick, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #endif { } }; diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 743c852e10f2..d74cc8fd1643 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1871,7 +1871,7 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod struct restart_block *restart; do { - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE|__TASK_DEFERRABLE_WAKEUP); hrtimer_sleeper_start_expires(t, mode); if (likely(t->task)) diff --git a/mm/vmalloc.c b/mm/vmalloc.c index e6f352bf0498..189b281942d8 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -2730,6 +2730,26 @@ void *vzalloc_node(unsigned long size, int node) } EXPORT_SYMBOL(vzalloc_node); +/** + * vmalloc_user_node_flags - allocate memory for userspace on a specific node + * @size: allocation size + * @node: numa node + * @flags: flags for the page level allocator + * + * The resulting memory area is zeroed so it can be mapped to userspace + * without leaking data. + * + * Return: pointer to the allocated memory or %NULL on error + */ +void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags) +{ + return __vmalloc_node_range(size, SHMLBA, VMALLOC_START, VMALLOC_END, + flags | __GFP_ZERO, PAGE_KERNEL, + VM_USERMAP, node, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(vmalloc_user_node_flags); + #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) From c9a5c55c2b8a0c6966dfe9093b14f4b95b3a1671 Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Sat, 12 Jun 2021 12:27:34 -0700 Subject: [PATCH 002/165] Add README.md Change-Id: I243e9d328aed1fef8629a713a5734677a4fd9556 --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000000..2d69d3e33fe4 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# ghOSt: Fast & Flexible User-Space Delegation of Linux Scheduling + +ghOSt is a general-purpose delegation of scheduling policy implemented on top of +the Linux kernel. The ghOSt framework provides a rich API that receives +scheduling decisions for processes from userspace and actuates them as +transactions. Programmers can use any language or tools to develop policies, +which can be upgraded without a machine reboot. ghOSt supports policies for a +range of scheduling objectives, from µs-scale latency, to throughput, to energy +efficiency, and beyond, and incurs low overheads for scheduling actions. Many +policies are just a few hundred lines of code. Overall, ghOSt provides a +performant framework for delegation of thread scheduling policy to userspace +processes that enables policy optimization, non-disruptive upgrades, and fault +isolation. + +You must compile and install this kernel in order to use the [ghOSt userspace +component](https://www.github.com/google/ghost-userspace). + +This kernel is Linux 5.11. We recommend installing Ubuntu 20.04 LTS, which ships +with Linux 5.4 by default, as the distribution. Compile this kernel on Ubuntu +20.04 and then replace the existing 5.4 kernel with this one. + +This is not an officially supported Google product. From 06a37c889e54657bbd007f4012a6ba6bf1057129 Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Mon, 21 Jun 2021 13:54:59 -0700 Subject: [PATCH 003/165] Bug fix Change-Id: I7bf91efc96dfaf513ebb08f9408f36c57e5d3dec --- kernel/sched/fair.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ba187a863fb7..bc9e83f6b2cd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7150,12 +7150,12 @@ done: __maybe_unused; return p; idle: + if (!rf) + return NULL; #ifdef CONFIG_SCHED_CLASS_GHOST if (skip_fair_idle_balance(cfs_rq, prev)) return NULL; #endif - if (!rf) - return NULL; new_tasks = newidle_balance(rq, rf); From ee0f851d542e9c57cc565b11fb11c7190d31ce9f Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Mon, 21 Jun 2021 16:33:22 -0700 Subject: [PATCH 004/165] Bug fix Change-Id: Ia7fcc08f30bc2fe1097e3ff8dd1b879568c9ffc8 --- kernel/sched/core.c | 4 ++-- kernel/sched/ghost.c | 17 +++++++++++++++++ kernel/sched/sched.h | 3 +++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 52ef3198d183..4057f1083661 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4137,7 +4137,7 @@ struct callback_head balance_push_callback = { .func = (void (*)(struct callback_head *))balance_push, }; -static inline struct callback_head *splice_balance_callbacks(struct rq *rq) +inline struct callback_head *splice_balance_callbacks(struct rq *rq) { struct callback_head *head = rq->balance_callback; @@ -4153,7 +4153,7 @@ static void __balance_callbacks(struct rq *rq) do_balance_callbacks(rq, splice_balance_callbacks(rq)); } -static inline void balance_callbacks(struct rq *rq, struct callback_head *head) +inline void balance_callbacks(struct rq *rq, struct callback_head *head) { unsigned long flags; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 68d3f94f947a..10f802fae899 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3553,10 +3553,27 @@ static void task_dead_ghost(struct task_struct *p) { struct rq_flags rf; struct rq *rq; + struct callback_head *head; + + WARN_ON_ONCE(preemptible()); rq = task_rq_lock(p, &rf); release_from_ghost(rq, p); + head = splice_balance_callbacks(rq); task_rq_unlock(rq, p, &rf); + /* + * 'rq_pin_lock' issues a warning when the there are pending callback + * functions for the runqueue. The point of this warning is to ensure + * that callbacks are run in a timely manner + * (https://lkml.org/lkml/2020/9/11/1027). + * + * When 'release_from_ghost' adds a callback to the balance queue in the + * task_dead path, there is no subsequent call to 'balance_callbacks' + * before 'rq_pin_lock' is called. This causes the warning to be issued. + * + * To avoid the warning, we manually call 'balance_callbacks' here. + */ + balance_callbacks(rq, head); } /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0efec2eec0a8..3f9c630c715a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -279,6 +279,9 @@ static inline int ghost_schedattr_to_queue_fd(const struct sched_attr *attr) static inline unsigned long ghost_cfs_added_load(struct rq *rq) { return 0; } #endif /* CONFIG_SCHED_CLASS_GHOST */ +struct callback_head *splice_balance_callbacks(struct rq *rq); +void balance_callbacks(struct rq *rq, struct callback_head *head); + /* * Helpers for converting nanosecond timing to jiffy resolution */ From d556edb33dcfe13040d3396df5810fe72fe96a31 Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Thu, 8 Jul 2021 16:42:27 -0700 Subject: [PATCH 005/165] Apply latest KCLs (up through and including kcl/395125) Change-Id: I0bdedf1467048bef0bf83c2722dfb1026d6f8e3b --- include/linux/bpf_types.h | 4 +- include/linux/sched.h | 46 ++- include/uapi/linux/bpf.h | 6 +- include/uapi/linux/ghost.h | 92 ++++-- kernel/bpf/syscall.c | 20 +- kernel/bpf/verifier.c | 2 +- kernel/sched/bpf.c | 210 +++++++----- kernel/sched/ghost.c | 642 +++++++++++++++++++++++++++---------- kernel/sched/ghostfs.c | 3 + kernel/sched/sched.h | 29 +- 10 files changed, 749 insertions(+), 305 deletions(-) diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 1a24c4d0686c..c3f020349925 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -77,8 +77,8 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm, void *, void *) #endif /* CONFIG_BPF_LSM */ #endif -BPF_PROG_TYPE(BPF_PROG_TYPE_SCHEDULER, scheduler, struct bpf_scheduler, - struct bpf_scheduler_kern) +BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_SCHED, ghost_sched, struct bpf_ghost_sched, + struct bpf_ghost_sched_kern) BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) diff --git a/include/linux/sched.h b/include/linux/sched.h index eb87eefdf9d7..7d05ee963aa8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -608,7 +608,39 @@ struct sched_ghost_entity { uint new_task : 1; uint agent : 1; + /* + * Locking of 'twi' is awkward: + * 1. wake_up_new_task: both select_task_rq() and task_woken_ghost() + * are called with 'pi->lock' held. + * 2. ttwu_do_activate: both select_task_rq() and task_woken_ghost() + * are called with 'pi->lock' held when called via ttwu_queue() + * (i.e. not a remote wakeup). + * 3. ttwu_do_activate: only 'rq->lock' is held when called via + * sched_ttwu_pending (i.e. indirectly via ttwu_queue_remote). + * + * (1) and (2) are easy because 'p->pi_lock' is held across both + * select_task_rq() and task_woken_ghost(). + * + * (3) is tricky because 'p->pi_lock' is held when select_task_rq() + * is called on the waker's cpu while 'rq->lock' is held when + * task_woken_ghost() is called on the remote cpu. We rely on the + * following constraints: + * a. Once a task is woken up there cannot be another wakeup until + * it gets oncpu and blocks (thus another wakeup cannot happen + * until task_woken_ghost() has been called). + * b. flush_smp_call_function_queue()->llist_del_all() pairs with + * __ttwu_queue_wakelist()->llist_add() to guarantee visiblity + * of changes made to 'p->ghost.twi' on the waker's cpu when + * ttwu_do_activate() is called on the remote cpu. + */ + struct { + int last_ran_cpu; + int wake_up_cpu; + int waker_cpu; + } twi; /* twi = task_wakeup_info */ + struct list_head task_list; + struct rcu_head rcu; }; extern void ghost_commit_greedy_txn(void); @@ -1423,18 +1455,18 @@ struct task_struct { */ }; -struct bpf_scheduler_kern { +struct bpf_ghost_sched_kern { }; struct bpf_prog; union bpf_attr; enum bpf_prog_type; -int scheduler_bpf_prog_attach(const union bpf_attr *attr, - struct bpf_prog *prog); -int scheduler_bpf_prog_detach(const union bpf_attr *attr, - enum bpf_prog_type ptype); -int scheduler_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog); +int ghost_sched_bpf_prog_attach(const union bpf_attr *attr, + struct bpf_prog *prog); +int ghost_sched_bpf_prog_detach(const union bpf_attr *attr, + enum bpf_prog_type ptype); +int ghost_sched_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog); static inline struct pid *task_pid(struct task_struct *task) { diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 0f0448ca3af6..650e93ef4b5a 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -200,7 +200,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_EXT, BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, - BPF_PROG_TYPE_SCHEDULER = 35, + BPF_PROG_TYPE_GHOST_SCHED = 35, }; enum bpf_attach_type { @@ -242,7 +242,7 @@ enum bpf_attach_type { BPF_XDP_CPUMAP, BPF_SK_LOOKUP, BPF_XDP, - BPF_SCHEDULER_TICK = 50, + BPF_GHOST_SCHED_SKIP_TICK = 50, __MAX_BPF_ATTACH_TYPE }; @@ -5153,7 +5153,7 @@ enum { BTF_F_ZERO = (1ULL << 3), }; -struct bpf_scheduler { +struct bpf_ghost_sched { }; #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 4cc099211501..d824877dcf39 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 30 +#define GHOST_VERSION 35 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -116,6 +116,7 @@ struct ghost_status_word { uint32_t barrier; uint32_t flags; uint64_t gtid; + int64_t switch_time; /* time at which task was context-switched onto CPU */ uint64_t runtime; /* total time spent on the CPU in nsecs */ } __attribute__((__aligned__(32))); @@ -172,6 +173,7 @@ enum { MSG_TASK_YIELD, MSG_TASK_DEPARTED, MSG_TASK_SWITCHTO, + MSG_TASK_AFFINITY_CHANGED, /* cpu messages */ MSG_CPU_TICK = _MSG_CPU_FIRST, @@ -218,9 +220,27 @@ struct ghost_msg_payload_task_departed { char from_switchto; }; +struct ghost_msg_payload_task_affinity_changed { + uint64_t gtid; +}; + struct ghost_msg_payload_task_wakeup { uint64_t gtid; char deferrable; /* bool: 0 or 1 */ + + int last_ran_cpu; /* + * CPU that task last ran on (may be different + * than where it was last scheduled by the + * agent due to switchto). + */ + + int wake_up_cpu; /* + * CPU where the task was woken up (this is + * typically where the task last ran but it + * may also be the waker's cpu). + */ + + int waker_cpu; /* CPU of the waker task */ }; struct ghost_msg_payload_task_switchto { @@ -354,6 +374,37 @@ enum txn_commit_at { /* ghost transaction */ +/* + * (txn->state == GHOST_TXN_READY) transaction is ready + * (txn->state >= 0 && txn->state < nr_cpu_ids) transaction is claimed + * (txn->state < 0) transaction is committed + */ +enum ghost_txn_state { + GHOST_TXN_COMPLETE = INT_MIN, + GHOST_TXN_ABORTED, + GHOST_TXN_TARGET_ONCPU, + GHOST_TXN_TARGET_STALE, + GHOST_TXN_TARGET_NOT_FOUND, + GHOST_TXN_TARGET_NOT_RUNNABLE, + GHOST_TXN_AGENT_STALE, + GHOST_TXN_CPU_OFFLINE, + GHOST_TXN_CPU_UNAVAIL, + GHOST_TXN_INVALID_FLAGS, + GHOST_TXN_INVALID_TARGET, + GHOST_TXN_NOT_PERMITTED, + GHOST_TXN_INVALID_CPU, + GHOST_TXN_NO_AGENT, + GHOST_TXN_UNSUPPORTED_VERSION, + GHOST_TXN_POISONED, + + /* + * Values [0-nr_cpu_ids) indicates that the transaction is claimed + * by the specific CPU. + */ + + GHOST_TXN_READY = INT_MAX, +}; + /* * _ghost_txn_state_t is not expected to be used anywhere except as the type * of ghost_txn::state below. See cl/350179823 as an example of unintentional @@ -361,7 +412,12 @@ enum txn_commit_at { */ #ifdef __cplusplus #include -typedef std::atomic _ghost_txn_state_t; +typedef std::atomic _ghost_txn_state_t; +/* + * To be safe, check that the size of '_ghost_txn_state_t' is equal to the size + * of 'int32_t', which is the size that the kernel assumes the state is. + */ +static_assert(sizeof(_ghost_txn_state_t) == sizeof(int32_t)); typedef std::atomic _ghost_txn_owner_t; #else typedef int32_t _ghost_txn_state_t; @@ -378,6 +434,7 @@ struct ghost_txn { uint8_t commit_flags; uint8_t unused; int64_t gtid; + int64_t commit_time; /* the time that the txn commit succeeded/failed */ /* * Context-dependent fields. */ @@ -393,37 +450,6 @@ struct ghost_cpu_data { #define GHOST_TXN_VERSION 0 -/* - * (txn->state == GHOST_TXN_READY) transaction is ready - * (txn->state >= 0 && txn->state < nr_cpu_ids) transaction is claimed - * (txn->state < 0) transaction is committed - */ -enum ghost_txn_state { - GHOST_TXN_COMPLETE = INT_MIN, - GHOST_TXN_ABORTED, - GHOST_TXN_TARGET_ONCPU, - GHOST_TXN_TARGET_STALE, - GHOST_TXN_TARGET_NOT_FOUND, - GHOST_TXN_TARGET_NOT_RUNNABLE, - GHOST_TXN_AGENT_STALE, - GHOST_TXN_CPU_OFFLINE, - GHOST_TXN_CPU_UNAVAIL, - GHOST_TXN_INVALID_FLAGS, - GHOST_TXN_INVALID_TARGET, - GHOST_TXN_NOT_PERMITTED, - GHOST_TXN_INVALID_CPU, - GHOST_TXN_NO_AGENT, - GHOST_TXN_UNSUPPORTED_VERSION, - GHOST_TXN_POISONED, - - /* - * Values [0-nr_cpu_ids) indicates that the transaction is claimed - * by the specific CPU. - */ - - GHOST_TXN_READY = INT_MAX, -}; - /* GHOST_TIMERFD_SETTIME */ struct timerfd_ghost { int cpu; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 7980c5a5e1db..dddfe6f5c51c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2002,9 +2002,9 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, if (expected_attach_type == BPF_SK_LOOKUP) return 0; return -EINVAL; - case BPF_PROG_TYPE_SCHEDULER: + case BPF_PROG_TYPE_GHOST_SCHED: switch (expected_attach_type) { - case BPF_SCHEDULER_TICK: + case BPF_GHOST_SCHED_SKIP_TICK: return 0; default: return -EINVAL; @@ -2959,8 +2959,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) return BPF_PROG_TYPE_SK_LOOKUP; case BPF_XDP: return BPF_PROG_TYPE_XDP; - case BPF_SCHEDULER_TICK: - return BPF_PROG_TYPE_SCHEDULER; + case BPF_GHOST_SCHED_SKIP_TICK: + return BPF_PROG_TYPE_GHOST_SCHED; default: return BPF_PROG_TYPE_UNSPEC; } @@ -3016,8 +3016,8 @@ static int bpf_prog_attach(const union bpf_attr *attr) case BPF_PROG_TYPE_SOCK_OPS: ret = cgroup_bpf_prog_attach(attr, ptype, prog); break; - case BPF_PROG_TYPE_SCHEDULER: - ret = scheduler_bpf_prog_attach(attr, prog); + case BPF_PROG_TYPE_GHOST_SCHED: + ret = ghost_sched_bpf_prog_attach(attr, prog); break; default: ret = -EINVAL; @@ -3055,8 +3055,8 @@ static int bpf_prog_detach(const union bpf_attr *attr) case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_SOCK_OPS: return cgroup_bpf_prog_detach(attr, ptype); - case BPF_PROG_TYPE_SCHEDULER: - return scheduler_bpf_prog_detach(attr, ptype); + case BPF_PROG_TYPE_GHOST_SCHED: + return ghost_sched_bpf_prog_detach(attr, ptype); default: return -EINVAL; } @@ -4074,8 +4074,8 @@ static int link_create(union bpf_attr *attr) ret = bpf_xdp_link_attach(attr, prog); break; #endif - case BPF_PROG_TYPE_SCHEDULER: - ret = scheduler_bpf_link_attach(attr, prog); + case BPF_PROG_TYPE_GHOST_SCHED: + ret = ghost_sched_bpf_link_attach(attr, prog); break; default: ret = -EINVAL; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 033f3a32a73b..f9ec4be8de2d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7981,7 +7981,7 @@ static int check_return_code(struct bpf_verifier_env *env) case BPF_PROG_TYPE_SK_LOOKUP: range = tnum_range(SK_DROP, SK_PASS); break; - case BPF_PROG_TYPE_SCHEDULER: + case BPF_PROG_TYPE_GHOST_SCHED: break; case BPF_PROG_TYPE_EXT: /* freplace program can return anything as its return value diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 588ec7e292c8..9e8ae143bfde 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -14,88 +14,98 @@ #include "sched.h" #include -static DEFINE_MUTEX(sched_tick); -static struct bpf_prog *tick_prog; +#ifdef CONFIG_SCHED_CLASS_GHOST -bool bpf_sched_ghost_skip_tick(void) +bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { - struct bpf_scheduler_kern ctx = {}; + struct bpf_ghost_sched_kern ctx = {}; struct bpf_prog *prog; - int ret; - rcu_read_lock(); + lockdep_assert_held(&rq->lock); - prog = rcu_dereference(tick_prog); - if (!prog) { - rcu_read_unlock(); + prog = rcu_dereference(e->bpf_tick); + if (!prog) return false; - } /* prog returns 1 if we want a tick on this cpu. */ - ret = BPF_PROG_RUN(prog, &ctx); - - rcu_read_unlock(); - - return ret != 1; + return BPF_PROG_RUN(prog, &ctx) != 1; } -static int sched_tick_attach(struct bpf_prog *prog) +static int ghost_sched_tick_attach(struct ghost_enclave *e, + struct bpf_prog *prog) { - mutex_lock(&sched_tick); - if (tick_prog) { - mutex_unlock(&sched_tick); + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->bpf_tick) { + spin_unlock_irqrestore(&e->lock, irq_fl); return -EBUSY; } - rcu_assign_pointer(tick_prog, prog); - mutex_unlock(&sched_tick); + rcu_assign_pointer(e->bpf_tick, prog); + spin_unlock_irqrestore(&e->lock, irq_fl); return 0; } -static void sched_tick_detach(struct bpf_prog *prog) +static void ghost_sched_tick_detach(struct ghost_enclave *e, + struct bpf_prog *prog) { - mutex_lock(&sched_tick); - rcu_replace_pointer(tick_prog, NULL, lockdep_is_held(&sched_tick)); - mutex_unlock(&sched_tick); + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + rcu_replace_pointer(e->bpf_tick, NULL, lockdep_is_held(&e->lock)); + spin_unlock_irqrestore(&e->lock, irq_fl); } -struct bpf_sched_link { +struct bpf_ghost_sched_link { struct bpf_link link; + struct ghost_enclave *e; enum bpf_attach_type ea_type; }; -static void bpf_sched_link_release(struct bpf_link *link) +static void bpf_ghost_sched_link_release(struct bpf_link *link) { - struct bpf_sched_link *sc_link = - container_of(link, struct bpf_sched_link, link); + struct bpf_ghost_sched_link *sc_link = + container_of(link, struct bpf_ghost_sched_link, link); + struct ghost_enclave *e = sc_link->e; + + if (WARN_ONCE(!e, "Missing enclave for bpf link ea_type %d!", + sc_link->ea_type)) + return; switch (sc_link->ea_type) { - case BPF_SCHEDULER_TICK: - sched_tick_detach(link->prog); + case BPF_GHOST_SCHED_SKIP_TICK: + ghost_sched_tick_detach(e, link->prog); break; default: + WARN_ONCE(1, "Unexpected release for ea_type %d", + sc_link->ea_type); break; }; - + kref_put(&e->kref, enclave_release); + sc_link->e = NULL; } -static void bpf_sched_link_dealloc(struct bpf_link *link) +static void bpf_ghost_sched_link_dealloc(struct bpf_link *link) { - struct bpf_sched_link *sc_link = - container_of(link, struct bpf_sched_link, link); + struct bpf_ghost_sched_link *sc_link = + container_of(link, struct bpf_ghost_sched_link, link); kfree(sc_link); } -static const struct bpf_link_ops bpf_sched_link_ops = { - .release = bpf_sched_link_release, - .dealloc = bpf_sched_link_dealloc, +static const struct bpf_link_ops bpf_ghost_sched_link_ops = { + .release = bpf_ghost_sched_link_release, + .dealloc = bpf_ghost_sched_link_dealloc, }; -int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) +int ghost_sched_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog) { struct bpf_link_primer link_primer; - struct bpf_sched_link *sc_link; + struct bpf_ghost_sched_link *sc_link; enum bpf_attach_type ea_type; + struct ghost_enclave *e; + struct fd f_enc = {0}; int err; if (attr->link_create.flags) @@ -105,7 +115,7 @@ int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) ea_type = prog->expected_attach_type; switch (ea_type) { - case BPF_SCHEDULER_TICK: + case BPF_GHOST_SCHED_SKIP_TICK: break; default: return -EINVAL; @@ -115,7 +125,7 @@ int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) if (!sc_link) return -ENOMEM; bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, - &bpf_sched_link_ops, prog); + &bpf_ghost_sched_link_ops, prog); sc_link->ea_type = ea_type; err = bpf_link_prime(&sc_link->link, &link_primer); @@ -124,9 +134,26 @@ int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) return -EINVAL; } + e = ghost_fdget_enclave(attr->link_create.target_fd, &f_enc); + if (!e) { + ghost_fdput_enclave(e, &f_enc); + /* bpf_link_cleanup() triggers .dealloc, but not .release. */ + bpf_link_cleanup(&link_primer); + return -EBADF; + } + /* + * On success, sc_link will hold a kref on the enclave, which will get + * put when the link's FD is closed (and thus bpf_link_put -> + * bpf_link_free -> our release). This is similar to how ghostfs files + * hold a kref on the enclave. Release is not called on failure. + */ + kref_get(&e->kref); + sc_link->e = e; + ghost_fdput_enclave(e, &f_enc); + switch (ea_type) { - case BPF_SCHEDULER_TICK: - err = sched_tick_attach(prog); + case BPF_GHOST_SCHED_SKIP_TICK: + err = ghost_sched_tick_attach(e, prog); break; default: pr_warn("bad sched bpf ea_type %d, should be unreachable", @@ -135,7 +162,9 @@ int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) break; }; if (err) { + /* bpf_link_cleanup() triggers .dealloc, but not .release. */ bpf_link_cleanup(&link_primer); + kref_put(&e->kref, enclave_release); return err; } @@ -143,32 +172,33 @@ int scheduler_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) } /* netns does this to have a packed array of progs[type]. might do this for the - * task type only, or maybe for all sched types. + * task type only, or maybe for all ghost types. */ -enum sched_bpf_attach_type { - SCHED_BPF_INVALID = -1, - SCHED_BPF_TICK = 0, +enum ghost_sched_bpf_attach_type { + GHOST_SCHED_BPF_INVALID = -1, + GHOST_SCHED_BPF_TICK = 0, MAX_SCHED_BPF_ATTACH_TYPE }; -static inline enum sched_bpf_attach_type -to_sched_bpf_attach_type(enum bpf_attach_type attach_type) +static inline enum ghost_sched_bpf_attach_type +to_ghost_sched_bpf_attach_type(enum bpf_attach_type attach_type) { switch (attach_type) { - case BPF_SCHEDULER_TICK: - return SCHED_BPF_TICK; + case BPF_GHOST_SCHED_SKIP_TICK: + return GHOST_SCHED_BPF_TICK; default: - return SCHED_BPF_INVALID; + return GHOST_SCHED_BPF_INVALID; } } -int scheduler_bpf_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) +int ghost_sched_bpf_prog_attach(const union bpf_attr *attr, + struct bpf_prog *prog) { - enum sched_bpf_attach_type type; + enum ghost_sched_bpf_attach_type type; if (attr->target_fd || attr->attach_flags || attr->replace_bpf_fd) return -EINVAL; - type = to_sched_bpf_attach_type(attr->attach_type); + type = to_ghost_sched_bpf_attach_type(attr->attach_type); if (type < 0) return -EINVAL; @@ -177,16 +207,16 @@ int scheduler_bpf_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) return -1; } -int scheduler_bpf_prog_detach(const union bpf_attr *attr, - enum bpf_prog_type ptype) +int ghost_sched_bpf_prog_detach(const union bpf_attr *attr, + enum bpf_prog_type ptype) { struct bpf_prog *prog; - enum sched_bpf_attach_type type; + enum ghost_sched_bpf_attach_type type; if (attr->attach_flags) return -EINVAL; - type = to_sched_bpf_attach_type(attr->attach_type); + type = to_ghost_sched_bpf_attach_type(attr->attach_type); if (type < 0) return -EINVAL; @@ -207,7 +237,7 @@ int scheduler_bpf_prog_detach(const union bpf_attr *attr, } static const struct bpf_func_proto * -scheduler_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { default: @@ -215,13 +245,14 @@ scheduler_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) } } -static bool scheduler_is_valid_access(int off, int size, - enum bpf_access_type type, - const struct bpf_prog *prog, - struct bpf_insn_access_aux *info) +static bool ghost_sched_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) { /* The verifier guarantees that size > 0. */ - if (off < 0 || off + size > sizeof(struct bpf_scheduler) || off % size) + if (off < 0 || off + size > sizeof(struct bpf_ghost_sched) + || off % size) return false; switch (off) { @@ -231,14 +262,15 @@ static bool scheduler_is_valid_access(int off, int size, } #define SCHEDULER_ACCESS_FIELD(T, F) \ - T(BPF_FIELD_SIZEOF(struct bpf_scheduler_kern, F), \ + T(BPF_FIELD_SIZEOF(struct bpf_ghost_sched_kern, F), \ si->dst_reg, si->src_reg, \ - offsetof(struct bpf_scheduler_kern, F)) + offsetof(struct bpf_ghost_sched_kern, F)) -static u32 scheduler_convert_ctx_access(enum bpf_access_type type, - const struct bpf_insn *si, - struct bpf_insn *insn_buf, - struct bpf_prog *prog, u32 *target_size) +static u32 ghost_sched_convert_ctx_access(enum bpf_access_type type, + const struct bpf_insn *si, + struct bpf_insn *insn_buf, + struct bpf_prog *prog, + u32 *target_size) { struct bpf_insn *insn = insn_buf; @@ -253,11 +285,35 @@ static u32 scheduler_convert_ctx_access(enum bpf_access_type type, return insn - insn_buf; } -const struct bpf_verifier_ops scheduler_verifier_ops = { - .get_func_proto = scheduler_func_proto, - .is_valid_access = scheduler_is_valid_access, - .convert_ctx_access = scheduler_convert_ctx_access, +const struct bpf_verifier_ops ghost_sched_verifier_ops = { + .get_func_proto = ghost_sched_func_proto, + .is_valid_access = ghost_sched_is_valid_access, + .convert_ctx_access = ghost_sched_convert_ctx_access, }; -const struct bpf_prog_ops scheduler_prog_ops = { +const struct bpf_prog_ops ghost_sched_prog_ops = { }; +#else /* !CONFIG_SCHED_CLASS_GHOST */ + +const struct bpf_verifier_ops ghost_sched_verifier_ops = {}; +const struct bpf_prog_ops ghost_sched_prog_ops = {}; + +int ghost_sched_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog) +{ + return -EINVAL; +} + +int ghost_sched_bpf_prog_attach(const union bpf_attr *attr, + struct bpf_prog *prog) +{ + return -EINVAL; +} + +int ghost_sched_bpf_prog_detach(const union bpf_attr *attr, + enum bpf_prog_type ptype) +{ + return -EINVAL; +} + +#endif /* !CONFIG_SCHED_CLASS_GHOST */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 10f802fae899..72215e76be2c 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -66,16 +66,21 @@ static void ghost_task_new(struct rq *rq, struct task_struct *p); static void ghost_task_yield(struct rq *rq, struct task_struct *p); static void ghost_task_blocked(struct rq *rq, struct task_struct *p); static void task_dead_ghost(struct task_struct *p); -static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p); -static bool task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p); +static void task_deliver_msg_affinity_changed(struct rq *rq, + struct task_struct *p); +static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p); +static void task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p); static bool cpu_deliver_msg_tick(struct rq *rq); static int task_target_cpu(struct task_struct *p); static int agent_target_cpu(struct rq *rq); static inline bool ghost_txn_ready(int cpu); +static bool _ghost_commit_pending_txn(int cpu, enum txn_commit_at where); +static inline void ghost_claim_and_kill_txn(int cpu, enum ghost_txn_state err); static void ghost_commit_all_greedy_txns(void); static void ghost_commit_pending_txn(enum txn_commit_at where); static inline int queue_decref(struct ghost_queue *q); static void release_from_ghost(struct rq *rq, struct task_struct *p); +static void ghost_wake_agent_on(int cpu); struct rq *context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf); @@ -89,6 +94,28 @@ static void __enclave_remove_task(struct ghost_enclave *e, static int __sw_region_free(struct ghost_sw_region *region); static const struct file_operations queue_fops; +/* enclave::is_dying */ +#define ENCLAVE_IS_DYING (1U << 0) +#define ENCLAVE_IS_REAPABLE (1U << 1) + +static inline bool enclave_is_dying(struct ghost_enclave *e) +{ + lockdep_assert_held(&e->lock); + + return e->is_dying & ENCLAVE_IS_DYING; +} + +static inline bool enclave_is_reapable(struct ghost_enclave *e) +{ + lockdep_assert_held(&e->lock); + + if (e->is_dying & ENCLAVE_IS_REAPABLE) { + WARN_ON_ONCE(!enclave_is_dying(e)); + return true; + } + return false; +} + /* * There are certain things we can't do in the scheduler while holding rq locks * (or enclave locks, which are ordered after the rq lock), such as @@ -192,6 +219,7 @@ static void submit_enclave_work(struct ghost_enclave *e, struct rq *rq, bool need_work = false; VM_BUG_ON(nr_decrefs < 0); + WARN_ON_ONCE(run_task_reaper && !enclave_is_reapable(e)); lockdep_assert_held(&rq->lock); lockdep_assert_held(&e->lock); @@ -329,8 +357,36 @@ static inline void invalidate_cached_tasks(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); - if (unlikely(rq->ghost.latched_task == p)) - rq->ghost.latched_task = NULL; + if (unlikely(rq->ghost.latched_task == p)) { + if (rq->ghost.skip_latched_preemption) { + /* + * We cannot produce a TASK_PREEMPT msg here. + * + * This is called via ghost_move_task() during txn + * commit after validating the task to be committed. + * Thus if we produce a TASK_PREEMPT msg it won't be + * caught by the task->barrier and prevent the txn + * from committing. + * + * Then it is possible to observe back-to-back + * TASK_PREEMPT msgs in the agent (the second + * would be due to a real preemption) which will + * trigger a 'task->preempted' CHECK-fail in the agent. + */ + rq->ghost.latched_task = NULL; + } else { + /* + * This is called via non-ghost move_queued_task() + * callers (e.g. sched_setaffinity). We produce a + * TASK_PREEMPT msg to let the agent know that the + * task is no longer latched (without this the task + * would be stranded). + */ + ghost_latched_task_preempted(rq); + } + } + + rq->ghost.skip_latched_preemption = false; } static inline bool is_cached_task(struct rq *rq, struct task_struct *p) @@ -386,7 +442,7 @@ static inline void force_offcpu(struct rq *rq, bool resched) schedule_next(rq, GHOST_NULL_GTID, resched); } -static void update_curr_ghost(struct rq *rq) +static void __update_curr_ghost(struct rq *rq, bool update_sw) { struct task_struct *curr = rq->curr; struct ghost_status_word *sw = curr->ghost.status_word; @@ -406,15 +462,20 @@ static void update_curr_ghost(struct rq *rq) now = rq_clock_task(rq); delta = now - curr->se.exec_start; - if (unlikely((s64)delta <= 0)) - return; + if ((s64)delta > 0) { + curr->se.sum_exec_runtime += delta; + account_group_exec_runtime(curr, delta); + cgroup_account_cputime(curr, delta); + curr->se.exec_start = now; + } - curr->se.sum_exec_runtime += delta; - account_group_exec_runtime(curr, delta); - cgroup_account_cputime(curr, delta); - curr->se.exec_start = now; + if (update_sw) + WRITE_ONCE(sw->runtime, curr->se.sum_exec_runtime); +} - WRITE_ONCE(sw->runtime, curr->se.sum_exec_runtime); +static void update_curr_ghost(struct rq *rq) +{ + __update_curr_ghost(rq, true); } static void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) @@ -435,8 +496,10 @@ static void switched_to_ghost(struct rq *rq, struct task_struct *p) { struct ghost_status_word *status_word = p->ghost.status_word; - if (task_running(rq, p)) + if (task_running(rq, p)) { + ghost_sw_set_time(status_word, ktime_get_ns()); ghost_sw_set_flag(status_word, GHOST_SW_TASK_ONCPU); + } if (task_on_rq_queued(p)) ghost_sw_set_flag(status_word, GHOST_SW_TASK_RUNNABLE); @@ -659,7 +722,7 @@ void ghost_tick(struct rq *rq) * Called from the timer tick handler while holding the rq->lock. Called only * if a ghost task is current. */ -void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) +static void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) { struct task_struct *agent = rq->ghost.agent; @@ -671,7 +734,7 @@ void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) if (unlikely(!agent)) return; - update_curr_ghost(rq); + __update_curr_ghost(rq, false); if (cpu_deliver_msg_tick(rq)) ghost_wake_agent_on(agent_target_cpu(rq)); @@ -708,11 +771,13 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, return rq->ghost.latched_task || rq_adj_nr_running(rq); } -int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) +static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) { + int waker_cpu = smp_processor_id(); + /* For anything but wake ups, just return the task_cpu */ if (!(wake_flags & (WF_TTWU | WF_FORK))) - goto out; + return task_cpu(p); /* * We have at least a couple of obvious choices here: @@ -736,9 +801,14 @@ int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) * and the default is to keep the task on the same CPU it last ran on. */ if (sysctl_ghost_wake_on_waker_cpu) - return smp_processor_id(); -out: - return task_cpu(p); + p->ghost.twi.wake_up_cpu = waker_cpu; + else + p->ghost.twi.wake_up_cpu = task_cpu(p); + + p->ghost.twi.waker_cpu = waker_cpu; + p->ghost.twi.last_ran_cpu = task_cpu(p); + + return p->ghost.twi.wake_up_cpu; } static inline bool task_is_dead(struct rq *rq, struct task_struct *p) @@ -794,7 +864,8 @@ static int validate_next_task(struct rq *rq, struct task_struct *next, return 0; } -int validate_next_offcpu(struct rq *rq, struct task_struct *next, int *state) +static int validate_next_offcpu(struct rq *rq, struct task_struct *next, + int *state) { lockdep_assert_held(&rq->lock); @@ -864,6 +935,7 @@ static inline void ghost_prepare_switch(struct rq *rq, struct task_struct *prev, next->se.exec_start = rq_clock_task(rq); sw = next->ghost.status_word; WARN_ON_ONCE(sw->flags & GHOST_SW_TASK_ONCPU); + ghost_sw_set_time(sw, ktime_get_ns()); ghost_sw_set_flag(sw, GHOST_SW_TASK_ONCPU); } else { WARN_ON_ONCE(rq->ghost.must_resched && @@ -932,102 +1004,104 @@ bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev) void ghost_latched_task_preempted(struct rq *rq) { - struct task_struct *latched = rq->ghost.latched_task; + struct task_struct *latched = rq->ghost.latched_task; - WARN_ON_ONCE(!rq->ghost.agent); + WARN_ON_ONCE(!rq->ghost.agent); - if (task_has_ghost_policy(latched)) { - ghost_task_preempted(rq, latched); - ghost_wake_agent_of(latched); - } else { - /* - * Idle task was latched and agent must have made - * other arrangements to be notified that CPU is - * no longer idle (for e.g. NEED_CPU_NOT_IDLE). - */ - WARN_ON_ONCE(latched != rq->idle); - } - rq->ghost.latched_task = NULL; + if (task_has_ghost_policy(latched)) { + ghost_task_preempted(rq, latched); + ghost_wake_agent_of(latched); + } else { + /* + * Idle task was latched and agent must have made + * other arrangements to be notified that CPU is + * no longer idle (for e.g. NEED_CPU_NOT_IDLE). + */ + WARN_ON_ONCE(latched != rq->idle); + } + rq->ghost.latched_task = NULL; } static inline void ghost_update_boost_prio(struct task_struct *p, - bool preempted) + bool preempted) { - if (preempted) - ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_BOOST_PRIO); - else - ghost_sw_clear_flag(p->ghost.status_word, GHOST_SW_BOOST_PRIO); + if (preempted) + ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_BOOST_PRIO); + else + ghost_sw_clear_flag(p->ghost.status_word, GHOST_SW_BOOST_PRIO); } static struct task_struct *pick_next_ghost_agent(struct rq *rq) { struct task_struct *agent = rq->ghost.agent; - struct task_struct *next = NULL; - bool preempted; + struct task_struct *next = NULL; + int nr_running; + bool preempted; struct task_struct *prev = rq->curr; if (!agent || !agent->on_rq) - goto done; + goto done; - /* - * Evaluate after produce_prev_msgs() in case it wakes up the local - * agent. - */ + /* + * Evaluate after produce_prev_msgs() in case it wakes up the local + * agent. + */ next = pick_agent(rq); - /* - * 'preempted' is true if a higher priority sched_class (e.g. CFS) - * has runnable tasks. We use it as follows: - * - * 1. indicate via GHOST_SW_BOOST_PRIO that the agent must yield - * the CPU asap. - * - * 2. produce a TASK_PREEMPTED msg on behalf of 'prev' (note that the - * TASK_PREEMPTED msg could trigger a local agent wakeup; producing - * the msg here as opposed to after pick_next_task() allows us to - * eliminate one redundant context switch from 'ghost->CFS->agent' - * to 'ghost->agent'. - */ - WARN_ON_ONCE(rq->nr_running < rq->ghost.ghost_nr_running); - preempted = (rq->nr_running > rq->ghost.ghost_nr_running); - - if (rq->ghost.check_prev_preemption) { - if (next || preempted) { - /* - * Preempted by local agent or another sched_class. - * - * Paranoia: force_offcpu guarantees that 'prev' does - * not stay oncpu after producing TASK_PREEMPTED(prev). - */ - ghost_task_preempted(rq, prev); - ghost_wake_agent_of(prev); - force_offcpu(rq, false); - rq->ghost.check_prev_preemption = false; - - /* did the preemption msg wake up the local agent? */ - if (!next) - next = pick_agent(rq); - } - } + /* + * 'preempted' is true if a higher priority sched_class (e.g. CFS) + * has runnable tasks. We use it as follows: + * + * 1. indicate via GHOST_SW_BOOST_PRIO that the agent must yield + * the CPU asap. + * + * 2. produce a TASK_PREEMPTED msg on behalf of 'prev' (note that the + * TASK_PREEMPTED msg could trigger a local agent wakeup; producing + * the msg here as opposed to after pick_next_task() allows us to + * eliminate one redundant context switch from 'ghost->CFS->agent' + * to 'ghost->agent'. + */ + nr_running = rq->nr_running; + WARN_ON_ONCE(nr_running < rq->ghost.ghost_nr_running); + preempted = (nr_running > rq->ghost.ghost_nr_running); - /* - * CPU is switching to a non-ghost task while a task is latched. - * - * Treat this like latched_task preemption because we don't know when - * the CPU will be available again so no point in keeping it latched. - */ - if (rq->ghost.latched_task && preempted) { - ghost_latched_task_preempted(rq); + if (rq->ghost.check_prev_preemption) { + if (next || preempted) { + /* + * Preempted by local agent or another sched_class. + * + * Paranoia: force_offcpu guarantees that 'prev' does + * not stay oncpu after producing TASK_PREEMPTED(prev). + */ + ghost_task_preempted(rq, prev); + ghost_wake_agent_of(prev); + force_offcpu(rq, false); + rq->ghost.check_prev_preemption = false; + + /* did the preemption msg wake up the local agent? */ + if (!next) + next = pick_agent(rq); + } + } - /* did the preemption msg wake up the local agent? */ - if (!next) - next = pick_agent(rq); - } + /* + * CPU is switching to a non-ghost task while a task is latched. + * + * Treat this like latched_task preemption because we don't know when + * the CPU will be available again so no point in keeping it latched. + */ + if (rq->ghost.latched_task && preempted) { + ghost_latched_task_preempted(rq); - if (!next) - goto done; + /* did the preemption msg wake up the local agent? */ + if (!next) + next = pick_agent(rq); + } - ghost_update_boost_prio(next, preempted); + if (!next) + goto done; + + ghost_update_boost_prio(next, preempted); ghost_prepare_switch(rq, prev, next); done: @@ -1235,11 +1309,32 @@ static void yield_task_ghost(struct rq *rq) static void set_cpus_allowed_ghost(struct task_struct *p, const struct cpumask *newmask, u32 flags) { + struct rq_flags rf; + struct rq *rq = task_rq(p); + bool locked = false; + /* * Agents: not allowed (rejected in __set_cpus_allowed_ptr); - * Normal ghost tasks: WARN until we figure out what to do here. + * Normal ghost tasks: Notify the agents. + */ + WARN_ON_ONCE(is_agent(rq, p)); + + /* + * do_set_cpus_allowed() mentions a case where we could arrive here + * without the rq->lock held, but message delivery requires rq->lock + * held. */ - WARN_ON_ONCE(1); + if (!raw_spin_is_locked(&rq->lock)) { + __task_rq_lock(p, &rf); + locked = true; + } + + task_deliver_msg_affinity_changed(rq, p); + + if (locked) + __task_rq_unlock(rq, &rf); + + set_cpus_allowed_common(p, newmask, flags); } static void task_woken_ghost(struct rq *rq, struct task_struct *p) @@ -1320,6 +1415,16 @@ static struct rq *ghost_move_task(struct rq *rq, struct task_struct *next, lockdep_assert_held(&rq->lock); lockdep_assert_held(&next->pi_lock); + WARN_ON_ONCE(rq->ghost.skip_latched_preemption); + + /* + * Cleared in invalidate_cached_tasks() via move_queued_task() + * and dequeue_task_ghost(). We cannot clear it here because + * move_queued_task() will release rq->lock (the rq returned + * by move_queued_task() is different than the one passed in). + */ + rq->ghost.skip_latched_preemption = true; + /* * 'next' was enqueued on a different CPU than where the agent * wants to run it now so migrate it to this runqueue before @@ -1594,7 +1699,8 @@ static void enclave_reap_tasks(struct work_struct *work) static const struct sched_param param = { .sched_priority = 0 }; spin_lock_irqsave(&e->lock, irq_fl); - WARN_ON_ONCE(!e->is_dying); + WARN_ON_ONCE(!enclave_is_reapable(e)); + /* * The moment we unlock, the task could exit. We can't lock the task * either, since the lock ordering is pi_lock->e_lock. Once we let go @@ -1710,11 +1816,12 @@ void ghost_destroy_enclave(struct ghost_enclave *e) int cpu; spin_lock_irqsave(&e->lock, flags); - if (e->is_dying) { + if (enclave_is_dying(e)) { spin_unlock_irqrestore(&e->lock, flags); return; } - e->is_dying = true; + /* Don't accept new agents into the enclave or changes to its cpuset */ + e->is_dying = ENCLAVE_IS_DYING; /* * At this point, no one can change the cpus or add new agents, since @@ -1785,15 +1892,30 @@ void ghost_destroy_enclave(struct ghost_enclave *e) put_task_struct(agent); spin_lock_irqsave(&e->lock, flags); } + spin_unlock_irqrestore(&e->lock, flags); + + synchronize_rcu(); /* Required after unpublishing a cpu */ + /* + * It is safe to reap all tasks in the enclave only _after_ + * synchronize_rcu returns: we have unpublished the enclave cpus + * and synchronize_rcu() guarantees that any older rcu read-side + * critical sections in find_task_by_gtid() have completed. + * + * Since 'e->lock' is dropped before synchronize_rcu we must + * prevent enclave_add_task() from sneaking in and scheduling + * the task reaper before synchronize_rcu returns. + * + * This is indicated by or'ing ENCLAVE_IS_REAPABLE into 'e->is_dying'. + */ + spin_lock_irqsave(&e->lock, flags); + e->is_dying |= ENCLAVE_IS_REAPABLE; spin_unlock_irqrestore(&e->lock, flags); kref_get(&e->kref); /* Reaper gets a kref */ if (!schedule_work(&e->task_reaper)) kref_put(&e->kref, enclave_release); - synchronize_rcu(); /* Required after unpublishing a cpu */ - /* * Removes the enclave and all of its files from ghostfs. There may * still be open FDs, each of which holds a reference. When the last FD @@ -1825,7 +1947,7 @@ int ghost_enclave_set_cpus(struct ghost_enclave *e, const struct cpumask *cpus) spin_lock_irqsave(&e->lock, flags); - if (e->is_dying) { + if (enclave_is_dying(e)) { ret = -EXDEV; goto out_e; } @@ -1930,7 +2052,7 @@ static void enclave_add_task(struct ghost_enclave *e, struct task_struct *p) if (p->ghost.agent) return; spin_lock_irqsave(&e->lock, flags); - if (e->is_dying) { + if (enclave_is_reapable(e)) { /* * The task entered as the enclave was dying, and likely will * miss the reaper. @@ -1939,6 +2061,7 @@ static void enclave_add_task(struct ghost_enclave *e, struct task_struct *p) } WARN_ON_ONCE(!list_empty(&p->ghost.task_list)); list_add_tail(&p->ghost.task_list, &e->task_list); + e->nr_tasks++; spin_unlock_irqrestore(&e->lock, flags); } @@ -1952,6 +2075,7 @@ static void __enclave_remove_task(struct ghost_enclave *e, return; WARN_ON_ONCE(list_empty(&p->ghost.task_list)); list_del_init(&p->ghost.task_list); + e->nr_tasks--; } static void enclave_add_sw_region(struct ghost_enclave *e, @@ -2125,22 +2249,22 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, * enclave's kref. However, it is possible for the enclave to die * before we get enqueued (see uses of `new_task` and ghost_task_new()). */ - if (e->is_dying) { + if (enclave_is_dying(e)) { error = -EXDEV; goto done; } - if (!q) { + if (q) { /* - * Associate task with the default queue. Agents typically pass - * in their own queue. + * It's possible for a client task to have no queue: it may have + * arrived before an agent set the default queue during an agent + * update. + * + * However, agent tasks should have provided some queue: either + * explicitly or the default queue. We handle this case in + * ghost_prep_agent(). */ - q = e->def_q; - } - if (!q) { - error = -ENXIO; - goto done; + queue_incref(q); } - queue_incref(q); p->ghost.dst_q = q; /* Allocate a status_word */ @@ -2271,7 +2395,7 @@ static int ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, unsigned long irq_fl; spin_lock_irqsave(&e->lock, irq_fl); - error = __ghost_prep_task(e, p, /*q=*/NULL, forked); + error = __ghost_prep_task(e, p, e->def_q, forked); spin_unlock_irqrestore(&e->lock, irq_fl); return error; @@ -2297,6 +2421,17 @@ static int ghost_prep_agent(struct ghost_enclave *e, struct task_struct *p, if (rq->ghost.agent != NULL) return -EBUSY; + /* + * Clean up old transactions. Once this cpu has an agent, an old + * transaction could succeed and turn into a latched task and start + * running. + * + * You may be wondering why we do this here in addition to when the old + * agent exited. It's possible for a global agent from an older agent + * process to write a transaction *after* that cpu's agent task exited. + */ + ghost_claim_and_kill_txn(rq->cpu, GHOST_TXN_NO_AGENT); + /* We hold the rq lock, which is already irqsaved. */ spin_lock(&e->lock); @@ -2304,6 +2439,18 @@ static int ghost_prep_agent(struct ghost_enclave *e, struct task_struct *p, ret = -ENODEV; goto out; } + if (!q) { + /* + * Agents typically pass in their own queue, but if not they get + * the def_q (typically for tests). But they must have *some* + * queue. + */ + if (!e->def_q) { + ret = -ENXIO; + goto out; + } + q = e->def_q; + } ret = __ghost_prep_task(e, p, q, /*forked=*/false); if (ret) @@ -2707,6 +2854,33 @@ static struct task_struct *find_task_by_gtid(gtid_t gtid) return current; p = find_task_by_pid_ns(pid, &init_pid_ns); + + /* + * It is possible for a task to schedule after losing its identity + * during exit (do_exit -> unhash_process -> detach_pid -> free_pid). + * + * Since the agent identifies tasks to schedule using a 'gtid' we + * must provide an alternate lookup path so the dying task can be + * scheduled and properly die. + */ + if (unlikely(!p)) { + ulong flags; + struct task_struct *t; + struct ghost_enclave *e; + + e = rcu_dereference(per_cpu(enclave, smp_processor_id())); + if (WARN_ON_ONCE(!e)) + return NULL; + + spin_lock_irqsave(&e->lock, flags); + list_for_each_entry(t, &e->task_list, ghost.task_list) { + if (t->gtid == gtid) { + p = t; + break; + } + } + spin_unlock_irqrestore(&e->lock, flags); + } if (!p) return NULL; @@ -2714,6 +2888,26 @@ static struct task_struct *find_task_by_gtid(gtid_t gtid) if (p->gtid != gtid) return NULL; + /* + * We rely on rcu to guarantee stability of 'p': + * + * If 'p' was obtained via find_task_by_pid_ns() then release_task() + * waits for an rcu grace period via call_rcu(delayed_put_task_struct). + * This includes the case where userspace moves a task out of ghost + * via sched_setscheduler(2). + * + * If 'p' was obtained via enclave->task_list then we need to account + * for the task dying or moving out of the ghost sched_class when its + * enclave is destroyed. + * + * A dying task ensures that the task_struct will remain stable for + * an rcu grace period via call_rcu(ghost_delayed_put_task_struct). + * + * A dying enclave will synchronize_rcu() in ghost_destroy_enclave() + * before moving its tasks out of ghost (we don't use call_rcu() here + * because the task may transition in and out of ghost any number of + * times independent of call_rcu() invocation). + */ return p; } @@ -3221,7 +3415,6 @@ static int _produce(struct ghost_queue *q, uint32_t barrier, int type, static inline int produce_for_task(struct task_struct *p, int type, void *payload, int payload_size) { - task_barrier_inc(task_rq(p), p); return _produce(p->ghost.dst_q, task_barrier_get(p), type, payload, payload_size); } @@ -3264,11 +3457,17 @@ static inline bool cpu_skip_message(struct rq *rq) static inline bool cpu_deliver_msg_tick(struct rq *rq) { struct ghost_msg_payload_cpu_tick payload; + struct ghost_enclave *e; if (cpu_skip_message(rq)) return false; - if (bpf_sched_ghost_skip_tick()) + rcu_read_lock(); + e = rcu_dereference(per_cpu(enclave, cpu_of(rq))); + if (!e || ghost_bpf_skip_tick(e, rq)) { + rcu_read_unlock(); return false; + } + rcu_read_unlock(); payload.cpu = cpu_of(rq); @@ -3321,9 +3520,16 @@ static bool ghost_in_switchto(struct rq *rq) return rq->ghost.switchto_count ? true : false; } -static inline bool task_skip_message(struct rq *rq, struct task_struct *p) +/* + * Returns 0 if we should produce a message for the task, < 0 otherwise. + * + * If the task is not an agent, this will task_barrier_inc(), even if we should + * not produce a message. + */ +static inline int __task_deliver_common(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); + /* * Ignore the agent's task_state changes. * @@ -3331,20 +3537,35 @@ static inline bool task_skip_message(struct rq *rq, struct task_struct *p) * it's not runnable. */ if (unlikely(is_agent(rq, p))) - return true; + return -1; - if (WARN_ON_ONCE(!p->ghost.dst_q)) - return true; + /* + * Increment the barrier even if we are not going to send a message due + * to a missing queue, since the barrier protects some parts of the SW's + * state. + * + * Normally, this would be unsafe, since the barrier technically + * protects the messages, and userspace could be confused because it + * sees the barrier, but cannot see the message. However, since p has + * no dst_q, it has not been associated yet, which means no scheduler + * is operating on it. This can happen if a task arrives before an + * agent sets the enclave's default queue. The agent will associate + * that task during task Discovery. + */ + task_barrier_inc(rq, p); - return false; + if (!p->ghost.dst_q) + return -1; + + return 0; } -static bool task_deliver_msg_task_new(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_new payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); payload.runnable = task_on_rq_queued(p); @@ -3352,76 +3573,74 @@ static bool task_deliver_msg_task_new(struct rq *rq, struct task_struct *p) if (_get_sw_info(p->ghost.enclave, p->ghost.status_word, &payload.sw_info)) { WARN(1, "New task PID %d didn't have a status word!", p->pid); - return false; + return; } - return !produce_for_task(p, MSG_TASK_NEW, &payload, sizeof(payload)); + produce_for_task(p, MSG_TASK_NEW, &payload, sizeof(payload)); } -static bool task_deliver_msg_yield(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_yield(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_yield payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); payload.from_switchto = ghost_in_switchto(rq); - return !produce_for_task(p, MSG_TASK_YIELD, &payload, sizeof(payload)); + produce_for_task(p, MSG_TASK_YIELD, &payload, sizeof(payload)); } -static bool task_deliver_msg_preempt(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_preempt payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); payload.from_switchto = ghost_in_switchto(rq); - return !produce_for_task(p, MSG_TASK_PREEMPT, &payload, - sizeof(payload)); + produce_for_task(p, MSG_TASK_PREEMPT, &payload, sizeof(payload)); } -static bool task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_blocked payload = {.gtid = gtid(p)}; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); payload.from_switchto = ghost_in_switchto(rq); - return !produce_for_task(p, MSG_TASK_BLOCKED, &payload, - sizeof(payload)); + produce_for_task(p, MSG_TASK_BLOCKED, &payload, sizeof(payload)); } -static bool task_deliver_msg_dead(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_dead(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_dead payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); - return !produce_for_task(p, MSG_TASK_DEAD, &payload, sizeof(payload)); + produce_for_task(p, MSG_TASK_DEAD, &payload, sizeof(payload)); } -static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_departed payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); payload.cpu = cpu_of(rq); @@ -3430,10 +3649,24 @@ static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p) else payload.from_switchto = false; - return !produce_for_task(p, MSG_TASK_DEPARTED, &payload, + produce_for_task(p, MSG_TASK_DEPARTED, &payload, sizeof(payload)); } +static void task_deliver_msg_affinity_changed(struct rq *rq, + struct task_struct *p) +{ + struct ghost_msg_payload_task_affinity_changed payload; + + if (__task_deliver_common(rq, p)) + return; + + payload.gtid = gtid(p); + + produce_for_task(p, MSG_TASK_AFFINITY_CHANGED, &payload, + sizeof(payload)); +} + static inline bool deferrable_wakeup(struct task_struct *p) { #ifdef notyet @@ -3448,37 +3681,38 @@ static inline bool deferrable_wakeup(struct task_struct *p) return p->sched_deferrable_wakeup; } -static bool task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_wakeup payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); payload.deferrable = deferrable_wakeup(p); - return !produce_for_task(p, MSG_TASK_WAKEUP, &payload, sizeof(payload)); + payload.last_ran_cpu = p->ghost.twi.last_ran_cpu; + payload.wake_up_cpu = p->ghost.twi.wake_up_cpu; + payload.waker_cpu = p->ghost.twi.waker_cpu; + produce_for_task(p, MSG_TASK_WAKEUP, &payload, sizeof(payload)); } -static bool task_deliver_msg_switchto(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_switchto(struct rq *rq, struct task_struct *p) { struct ghost_msg_payload_task_switchto payload; - if (task_skip_message(rq, p)) - return false; + if (__task_deliver_common(rq, p)) + return; payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); - return !produce_for_task(p, MSG_TASK_SWITCHTO, &payload, - sizeof(payload)); + produce_for_task(p, MSG_TASK_SWITCHTO, &payload, sizeof(payload)); } static void release_from_ghost(struct rq *rq, struct task_struct *p) { struct ghost_enclave *e = p->ghost.enclave; - bool wake_agent; ulong flags; VM_BUG_ON(!e); @@ -3505,9 +3739,9 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) */ ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_F_CANFREE); if (p->state == TASK_DEAD) - wake_agent = task_deliver_msg_dead(rq, p); + task_deliver_msg_dead(rq, p); else - wake_agent = task_deliver_msg_departed(rq, p); + task_deliver_msg_departed(rq, p); /* status_word is off-limits to the kernel */ p->ghost.status_word = NULL; @@ -3520,7 +3754,32 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) WRITE_ONCE(rq->ghost.agent_barrier, 0); p->ghost.agent = false; VM_BUG_ON(rq->ghost.blocked_in_run); - VM_BUG_ON(wake_agent); + + /* + * Unlatched any tasks that had been committed. Latched tasks + * will be selected in PNT after a new agent attaches, which + * violates the quiescence needed by in-place agent handoffs. + * Now that ghost.agent is NULL, any new transactions will fail. + * + * If the only concern is running a latched task, we could wait + * to clear latched_task until a new agent attaches, since PNT + * won't pick the latched_task unless there is an agent. + * However, it's possible that the agent process is trying to + * shrink its enclave: stop the agent, then remove the cpu, + * without killing the entire agent. So we might never attach + * an old agent. This is also why we use a preemption instead + * of invalidate_cached_tasks: preempt the committed task and + * let the agent (if there is one) handle it. + */ + if (rq->ghost.latched_task) + ghost_latched_task_preempted(rq); + /* + * Clean up any pending transactions. We need to do this here, + * in addition to in setsched, since the global agent may be + * spinning waiting for this transaction to return before it + * gracefully exits. + */ + ghost_claim_and_kill_txn(rq->cpu, GHOST_TXN_NO_AGENT); /* See ghost_destroy_enclave() */ if (rq->ghost.agent_remove_enclave_cpu) { @@ -3540,13 +3799,20 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) spin_unlock_irqrestore(&e->lock, flags); - if (wake_agent) - ghost_wake_agent_of(p); + ghost_wake_agent_of(p); /* Release reference to 'dst_q' */ - VM_BUG_ON(!p->ghost.dst_q); - queue_decref(p->ghost.dst_q); - p->ghost.dst_q = NULL; + if (p->ghost.dst_q) { + queue_decref(p->ghost.dst_q); + p->ghost.dst_q = NULL; + } +} + +static void ghost_delayed_put_task_struct(struct rcu_head *rhp) +{ + struct task_struct *tsk = container_of(rhp, struct task_struct, + ghost.rcu); + put_task_struct(tsk); } static void task_dead_ghost(struct task_struct *p) @@ -3561,6 +3827,23 @@ static void task_dead_ghost(struct task_struct *p) release_from_ghost(rq, p); head = splice_balance_callbacks(rq); task_rq_unlock(rq, p, &rf); + + /* + * 'p' is dead and the last reference to its task_struct may be + * dropped after returning from this function (see finish_task_switch). + * + * We need to drain all rcu-protected callers of find_task_by_gtid() + * before returning from this function. Typically this would happen + * via delayed_put_task_struct() called by release_task() but since + * we provide an alternate gtid lookup that bypasses find_task_by_pid() + * we cannot rely on that. + * + * Note that we cannot call synchronize_rcu() directly because this + * function is called in a non-preemptible context. + */ + get_task_struct(p); + call_rcu(&p->ghost.rcu, ghost_delayed_put_task_struct); + /* * 'rq_pin_lock' issues a warning when the there are pending callback * functions for the runqueue. The point of this warning is to ensure @@ -3606,7 +3889,7 @@ static void ghost_set_pnt_state(struct rq *rq, struct task_struct *p, * * If it was blocked_in_run, clear it and reschedule, which ensures it wakes up. */ -void ghost_wake_agent_on(int cpu) +static void ghost_wake_agent_on(int cpu) { struct rq *dst_rq; @@ -4391,6 +4674,12 @@ static inline struct ghost_txn *_ghost_get_txn_ptr(int cpu) static inline void _ghost_set_txn_state(struct ghost_txn *txn, enum ghost_txn_state state) { + /* + * Set the time with a relaxed store since we update the txn state below + * with a release store. Userspace syncs with the kernel on that release + * store, so the release store acts a barrier. + */ + txn->commit_time = ktime_get_ns(); smp_store_release(&txn->state, state); } @@ -4407,6 +4696,20 @@ static inline void ghost_poison_txn(int cpu) ghost_set_txn_state(cpu, GHOST_TXN_POISONED); } +static inline void ghost_claim_and_kill_txn(int cpu, enum ghost_txn_state err) +{ + /* claim_txn and set_txn_state must be called by the same cpu */ + preempt_disable(); + + rcu_read_lock(); + if (unlikely(ghost_claim_txn(cpu, -1))) + ghost_set_txn_state(cpu, err); + rcu_read_unlock(); + + /* We're often called from within the scheduler */ + preempt_enable_no_resched(); +} + static inline bool ghost_txn_succeeded(int state) { return state == GHOST_TXN_COMPLETE; @@ -5105,6 +5408,7 @@ void ghost_switchto(struct rq *rq, struct task_struct *prev, list_add_tail(&next->ghost.run_list, &rq->ghost.tasks); next->ghost.last_runnable_at = 0; /* we're on_cpu */ + ghost_sw_set_time(next->ghost.status_word, ktime_get_ns()); ghost_sw_set_flag(next->ghost.status_word, GHOST_SW_TASK_RUNNABLE | GHOST_SW_TASK_ONCPU); diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 5a4aecbef564..5b89b33c65fd 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -583,6 +583,7 @@ static int gf_status_show(struct seq_file *sf, void *v) struct ghost_enclave *e = seq_to_e(sf); unsigned long fl; bool is_active; + unsigned long nr_tasks; /* * Userspace uses this to find any active enclave, since they don't have @@ -595,9 +596,11 @@ static int gf_status_show(struct seq_file *sf, void *v) * is for the *enclave*, not the *agent*. */ is_active = e->agent_online; + nr_tasks = e->nr_tasks; spin_unlock_irqrestore(&e->lock, fl); seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); + seq_printf(sf, "nr_tasks %lu\n", nr_tasks); return 0; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3f9c630c715a..ed7ad1ea9bbd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -112,6 +112,7 @@ struct ghost_rq { bool agent_should_wake; bool must_resched; /* rq->curr must reschedule in PNT */ bool check_prev_preemption; /* see 'ghost_prepare_task_switch()' */ + bool skip_latched_preemption; int ghost_nr_running; int run_flags; /* flags passed to 'ghost_run()' */ @@ -180,6 +181,7 @@ struct ghost_enclave { struct ghost_queue *def_q; /* default queue */ struct list_head task_list; /* all non-agent tasks in the enclave */ + unsigned long nr_tasks; struct work_struct task_reaper; struct enclave_work ew; /* to defer work while holding locks */ struct work_struct enclave_actual_release;/* work for enclave_release */ @@ -193,9 +195,13 @@ struct ghost_enclave { struct work_struct enclave_destroyer; unsigned long id; - bool is_dying; + int is_dying; bool agent_online; /* userspace says agent can schedule. */ struct kernfs_node *enclave_dir; + +#ifdef CONFIG_BPF + struct bpf_prog *bpf_tick; +#endif }; /* In kernel/sched/ghostfs.c */ @@ -237,10 +243,16 @@ extern void ghost_sched_cleanup_fork(struct task_struct *p); extern void ghost_latched_task_preempted(struct rq *rq); extern void ghost_task_preempted(struct rq *rq, struct task_struct *prev); extern unsigned long ghost_cfs_added_load(struct rq *rq); -extern void ghost_wake_agent_on(int cpu); extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); -extern bool bpf_sched_ghost_skip_tick(void); +#ifdef CONFIG_BPF +extern bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq); +#else +static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) +{ + return false; +} +#endif extern void ghost_wait_for_rendezvous(struct rq *rq); extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); @@ -265,6 +277,17 @@ static inline void ghost_sw_clear_flag(struct ghost_status_word *sw, smp_store_release(&sw->flags, sw->flags & ~flag); } +static inline void ghost_sw_set_time(struct ghost_status_word *sw, + s64 time) { + /* + * Do a relaxed store since userspace syncs with the release store to + * `sw->flags` for setting the oncpu bit in `ghost_sw_set_flag`. We set + * the time in this function before setting the oncpu bit, so we use + * that release store as a barrier. + */ + WRITE_ONCE(sw->switch_time, time); +} + static inline int ghost_schedattr_to_enclave_fd(const struct sched_attr *attr) { return attr->sched_runtime; From 5c534a2cba39e03b4f1a80b25bf281ff871692e4 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Tue, 17 Aug 2021 22:22:12 -0700 Subject: [PATCH 006/165] Apply KCLs up through and including kcl/395855 --- kernel/sched/ghost.c | 115 +++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 49 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 72215e76be2c..b08ed43a02be 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -81,6 +81,8 @@ static void ghost_commit_pending_txn(enum txn_commit_at where); static inline int queue_decref(struct ghost_queue *q); static void release_from_ghost(struct rq *rq, struct task_struct *p); static void ghost_wake_agent_on(int cpu); +static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, + bool was_latched); struct rq *context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf); @@ -1009,7 +1011,7 @@ void ghost_latched_task_preempted(struct rq *rq) WARN_ON_ONCE(!rq->ghost.agent); if (task_has_ghost_policy(latched)) { - ghost_task_preempted(rq, latched); + _ghost_task_preempted(rq, latched, true); ghost_wake_agent_of(latched); } else { /* @@ -2293,31 +2295,22 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, return error; } -/* Only called from enclave_release. */ +/* + * Only called from enclave_release. Any users of this SWR had an enclave kref, + * so if we're here, they must all have stopped using their SW. This also + * includes anyone mmapping the ghostfs file. + * + * This does not mean that the SW's were fully freed. They may have been in the + * GHOST_SW_F_CANFREE state. Userspace may have failed to free them, due to a + * crash. + */ static int __sw_region_free(struct ghost_sw_region *swr) { struct ghost_sw_region_header *h = swr->header; list_del(&swr->list); - /* - * There cannot be any references to 'region' from either - * userspace or kernel so we don't need to take the lock. - */ - - if (h->available != h->capacity) { - /* - * XXX agent closed the file descriptor (voluntary or crashed) - * but there are message sources that point into it from the - * kernel. - * - * Until we understand this better just let it leak. - */ - WARN_ONCE(1, "%s: id/%u capacity/%u and avail/%d", - __func__, h->id, h->capacity, h->available); - } else { - vfree(h); - } + vfree(h); kfree(swr); /* XXX memcg uncharge */ @@ -3595,7 +3588,8 @@ static void task_deliver_msg_yield(struct rq *rq, struct task_struct *p) produce_for_task(p, MSG_TASK_YIELD, &payload, sizeof(payload)); } -static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, + bool from_switchto) { struct ghost_msg_payload_task_preempt payload; @@ -3605,7 +3599,7 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p) payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); - payload.from_switchto = ghost_in_switchto(rq); + payload.from_switchto = from_switchto; produce_for_task(p, MSG_TASK_PREEMPT, &payload, sizeof(payload)); } @@ -3869,13 +3863,16 @@ static void ghost_set_pnt_state(struct rq *rq, struct task_struct *p, lockdep_assert_held(&rq->lock); - if (rq->ghost.latched_task != p) { + if (rq->ghost.latched_task && rq->ghost.latched_task != p) { /* * We're overwriting a 'latched_task' that never got oncpu. - * If we ever require the kernel to send TASK_PREEMPTED for - * all preemptions, we'll need to do so here. + * From the agent's perspective, this is a preemption, even if + * it is one that the agent implicitly requested by scheduling p + * onto this cpu to replace latched_task. Contrast this to + * invalidate_cached_tasks(), which moves p from one cpu to + * another. */ - ; + ghost_latched_task_preempted(rq); } rq->ghost.latched_task = p; rq->ghost.must_resched = false; @@ -3949,8 +3946,11 @@ void ghost_wake_agent_of(struct task_struct *p) ghost_wake_agent_on(task_target_cpu(p)); } -void ghost_task_preempted(struct rq *rq, struct task_struct *p) +static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, + bool was_latched) { + bool from_switchto; + lockdep_assert_held(&rq->lock); VM_BUG_ON(task_rq(p) != rq); @@ -3964,8 +3964,14 @@ void ghost_task_preempted(struct rq *rq, struct task_struct *p) if (p == rq->curr) update_curr_ghost(rq); + /* + * If a latched task was preempted then by definition it was not + * part of any switchto chain. + */ + from_switchto = was_latched ? false : ghost_in_switchto(rq); + /* Produce MSG_TASK_PREEMPT into 'p->ghost.dst_q' */ - task_deliver_msg_preempt(rq, p); + task_deliver_msg_preempt(rq, p, from_switchto); /* * Wakeup agent on this CPU. @@ -3979,6 +3985,14 @@ void ghost_task_preempted(struct rq *rq, struct task_struct *p) schedule_agent(rq, false); } +void ghost_task_preempted(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(task_rq(p) != rq); + + _ghost_task_preempted(rq, p, false); +} + static void ghost_task_new(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); @@ -4096,20 +4110,19 @@ static inline bool same_process(struct task_struct *p, struct task_struct *q) return p->group_leader == q->group_leader; } -static inline bool run_flags_valid(int run_flags, int unsupported_flags) +static inline bool run_flags_valid(int run_flags, int valid_run_flags, + gtid_t gtid) { - const int valid_run_flags = RTLA_ON_PREEMPT | - RTLA_ON_BLOCKED | - RTLA_ON_YIELD | - RTLA_ON_IDLE | - NEED_L1D_FLUSH | - NEED_CPU_NOT_IDLE | - ALLOW_TASK_ONCPU; - if (run_flags & ~valid_run_flags) return false; - if (run_flags & unsupported_flags) + if ((run_flags & RTLA_ON_IDLE) && (gtid != GHOST_NULL_GTID)) + return false; + + if ((run_flags & NEED_CPU_NOT_IDLE) && (gtid != GHOST_IDLE_GTID)) + return false; + + if ((gtid == GHOST_AGENT_GTID) && (run_flags != 0)) return false; return true; @@ -4129,24 +4142,26 @@ SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, int error = 0; int this_cpu; + const int supported_flags = RTLA_ON_IDLE; + if (!capable(CAP_SYS_NICE)) return -EPERM; if (run_cpu < 0 || run_cpu >= nr_cpu_ids || !cpu_online(run_cpu)) return -EINVAL; + if (!run_flags_valid(run_flags, supported_flags, gtid)) + return -EINVAL; + preempt_disable(); this_cpu = raw_smp_processor_id(); switch (gtid) { case GHOST_NULL_GTID: /* yield on local cpu */ - if ((run_flags & ~(RTLA_ON_IDLE | NEED_CPU_NOT_IDLE)) || - (run_cpu != this_cpu)) + if (run_cpu != this_cpu) error = -EINVAL; break; case GHOST_AGENT_GTID: /* ping agent */ - if (run_flags) - error = -EINVAL; break; default: error = -EINVAL; @@ -4383,6 +4398,14 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, int run_flags, state = GHOST_TXN_COMPLETE; struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, run_cpu)); + const int supported_flags = RTLA_ON_PREEMPT | + RTLA_ON_BLOCKED | + RTLA_ON_YIELD | + RTLA_ON_IDLE | + NEED_L1D_FLUSH | + NEED_CPU_NOT_IDLE | + ALLOW_TASK_ONCPU; + VM_BUG_ON(preemptible()); VM_BUG_ON(commit_state == NULL); VM_BUG_ON(need_rendezvous == NULL); @@ -4407,13 +4430,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, gtid = READ_ONCE(txn->gtid); run_flags = READ_ONCE(txn->run_flags); - if (!run_flags_valid(run_flags, 0)) { - state = GHOST_TXN_INVALID_FLAGS; - goto done; - } - - if ((run_flags & NEED_CPU_NOT_IDLE) && - (gtid != GHOST_NULL_GTID && gtid != GHOST_IDLE_GTID)) { + if (!run_flags_valid(run_flags, supported_flags, gtid)) { state = GHOST_TXN_INVALID_FLAGS; goto done; } From 2b63ce82df37ace91e0b35b53e97f36f04245442 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 18 Aug 2021 11:08:23 -0700 Subject: [PATCH 007/165] Apply kcl/395979 Merge conflict resolutions: - kernel/sched/core.c: added out_return in pick_next_task Change-Id: I1371554d7cecbf371fdb811b937e7d878e0b6c8b --- include/uapi/linux/ghost.h | 5 ++++- kernel/sched/core.c | 18 ++++++++++++--- kernel/sched/ghost.c | 46 ++++++++++++++++++++++---------------- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index d824877dcf39..4709342da9eb 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 35 +#define GHOST_VERSION 36 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -355,6 +355,9 @@ enum ghost_base_ops { #define ALLOW_TASK_ONCPU (1 << 8) /* If task is already running on remote * cpu then let it keep running there. */ +#define ELIDE_PREEMPT (1 << 9) /* Do not send TASK_PREEMPT if we preempt + * a previous ghost task on this cpu + */ /* txn->commit_flags */ enum txn_commit_at { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4057f1083661..be5b3c4da4e6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4283,7 +4283,6 @@ static inline void ghost_prepare_task_switch(struct rq *rq, if (rq->ghost.check_prev_preemption) { rq->ghost.check_prev_preemption = false; - WARN_ON_ONCE(task_has_ghost_policy(next)); ghost_task_preempted(rq, prev); ghost_wake_agent_of(prev); } @@ -5144,7 +5143,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) p = pick_next_task_idle(rq); } - return p; + goto out_return; } restart: @@ -5153,11 +5152,24 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) for_each_class(class) { p = class->pick_next_task(rq); if (p) - return p; + goto out_return; } /* The idle class should always have a runnable task: */ BUG(); + +out_return: +#ifdef CONFIG_SCHED_CLASS_GHOST + /* + * pick_next_task opted to keep the same task running, but we left + * check_prev_preemption on! This will break switchto, which checks + * that field during context_switch() + */ + if (WARN_ON_ONCE(p == prev && rq->ghost.check_prev_preemption)) + rq->ghost.check_prev_preemption = false; +#endif + + return p; } /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b08ed43a02be..22acdda901b0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1067,11 +1067,15 @@ static struct task_struct *pick_next_ghost_agent(struct rq *rq) WARN_ON_ONCE(nr_running < rq->ghost.ghost_nr_running); preempted = (nr_running > rq->ghost.ghost_nr_running); - if (rq->ghost.check_prev_preemption) { - if (next || preempted) { + /* Preempted by local agent or another sched_class. */ + if (next || preempted) { + /* + * Even though 'preempted' may be set, the task may have blocked + * or yielded. check_prev_preemption tells us if this is an + * actual preemption: the task did not stop voluntarily. + */ + if (rq->ghost.check_prev_preemption) { /* - * Preempted by local agent or another sched_class. - * * Paranoia: force_offcpu guarantees that 'prev' does * not stay oncpu after producing TASK_PREEMPTED(prev). */ @@ -1079,21 +1083,17 @@ static struct task_struct *pick_next_ghost_agent(struct rq *rq) ghost_wake_agent_of(prev); force_offcpu(rq, false); rq->ghost.check_prev_preemption = false; - - /* did the preemption msg wake up the local agent? */ - if (!next) - next = pick_agent(rq); } - } - /* - * CPU is switching to a non-ghost task while a task is latched. - * - * Treat this like latched_task preemption because we don't know when - * the CPU will be available again so no point in keeping it latched. - */ - if (rq->ghost.latched_task && preempted) { - ghost_latched_task_preempted(rq); + /* + * CPU is switching to a non-ghost task while a task is latched. + * + * Treat this like latched_task preemption because we don't know + * when the CPU will be available again so no point in keeping + * it latched. + */ + if (rq->ghost.latched_task) + ghost_latched_task_preempted(rq); /* did the preemption msg wake up the local agent? */ if (!next) @@ -1126,7 +1126,12 @@ static struct task_struct *pick_next_task_ghost(struct rq *rq) * (e.g. pick_next_task_fair() does this with core tagging enabled). */ if (rq->ghost.switchto_count == 0) { - rq->ghost.check_prev_preemption = false; + /* + * This is the only time we clear check_prev_preemption without + * sending a TASK_PREEMPT. + */ + if (rq->ghost.run_flags & ELIDE_PREEMPT) + rq->ghost.check_prev_preemption = false; } else { WARN_ON_ONCE(rq->ghost.switchto_count > 0); @@ -1223,6 +1228,7 @@ static struct task_struct *pick_next_task_ghost(struct rq *rq) */ if (unlikely(prev != agent && !rq->ghost.must_resched)) { next = prev; + rq->ghost.check_prev_preemption = false; goto done; } } @@ -4404,7 +4410,9 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, RTLA_ON_IDLE | NEED_L1D_FLUSH | NEED_CPU_NOT_IDLE | - ALLOW_TASK_ONCPU; + ALLOW_TASK_ONCPU | + ELIDE_PREEMPT | + 0; VM_BUG_ON(preemptible()); VM_BUG_ON(commit_state == NULL); From 46891cd131fb6bd5aacd52de0f6e55277a496e9d Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 25 Jun 2021 12:49:03 -0400 Subject: [PATCH 008/165] sched/ghost: add ghost_wake_agent_on_check() In a future patch, BPF programs will be able to call this. The check is to ensure that the caller's cpu's enclave matches the target cpu's enclave. This ensures that BPF programs (and thus their agents) are affecting their own enclaves. Regarding get_cpu(), I noticed the old code was calling smp_processor_id(), but it was in a preemptible path (via sys_ghost). The WARN_ON caught it. It seemed simplest to have the caller get_cpu, instead of mucking with the various return paths in the __ helper. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I0ecf3902257280d86f737a7a7cbbcf201e392682 --- kernel/sched/ghost.c | 60 +++++++++++++++++++++++++++++++++++++++----- kernel/sched/sched.h | 1 + 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 22acdda901b0..17d3c3d96317 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -96,6 +96,26 @@ static void __enclave_remove_task(struct ghost_enclave *e, static int __sw_region_free(struct ghost_sw_region *region); static const struct file_operations queue_fops; +/* True if X and Y have the same enclave, including having no enclave. */ +static bool check_same_enclave(int cpu_x, int cpu_y) +{ + struct ghost_enclave *x, *y; + + if (WARN_ON_ONCE(cpu_x < 0 || cpu_y < 0 + || cpu_x >= nr_cpu_ids || cpu_y >= nr_cpu_ids)) + return false; + + if (cpu_x == cpu_y) + return true; + + rcu_read_lock(); + x = rcu_dereference(per_cpu(enclave, cpu_x)); + y = rcu_dereference(per_cpu(enclave, cpu_y)); + rcu_read_unlock(); + + return x == y; +} + /* enclave::is_dying */ #define ENCLAVE_IS_DYING (1U << 0) #define ENCLAVE_IS_REAPABLE (1U << 1) @@ -3891,13 +3911,21 @@ static void ghost_set_pnt_state(struct rq *rq, struct task_struct *p, * (it will return ESTALE). * * If it was blocked_in_run, clear it and reschedule, which ensures it wakes up. + * + * Returns 0 on success, -error on failure. */ -static void ghost_wake_agent_on(int cpu) +static int __ghost_wake_agent_on(int cpu, int this_cpu, + bool check_caller_enclave) { struct rq *dst_rq; - if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu)) - return; + if (cpu < 0) + return -EINVAL; + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -ERANGE; + + if (check_caller_enclave && !check_same_enclave(this_cpu, cpu)) + return -EXDEV; dst_rq = cpu_rq(cpu); @@ -3924,7 +3952,7 @@ static void ghost_wake_agent_on(int cpu) * mostly for paranoia. */ if (!READ_ONCE(dst_rq->ghost.blocked_in_run)) - return; + return 0; /* * We can't write blocked_in_run, since we don't hold the RQ lock. @@ -3933,18 +3961,38 @@ static void ghost_wake_agent_on(int cpu) * an optimization to reduce the number of rescheds. */ if (READ_ONCE(dst_rq->ghost.agent_should_wake)) - return; + return 0; WRITE_ONCE(dst_rq->ghost.agent_should_wake, true); /* Write must come before the IPI/resched */ smp_wmb(); - if (cpu == smp_processor_id()) { + if (cpu == this_cpu) { set_tsk_need_resched(dst_rq->curr); set_preempt_need_resched(); } else { resched_cpu_unlocked(cpu); } + return 0; +} + +static void ghost_wake_agent_on(int cpu) +{ + int this_cpu = get_cpu(); + + __ghost_wake_agent_on(cpu, this_cpu, /*check_caller_enclave=*/ false); + put_cpu(); +} + +int ghost_wake_agent_on_check(int cpu) +{ + int this_cpu = get_cpu(); + int ret; + + ret = __ghost_wake_agent_on(cpu, this_cpu, + /*check_caller_enclave=*/ true); + put_cpu(); + return ret; } void ghost_wake_agent_of(struct task_struct *p) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ed7ad1ea9bbd..6d4a253ceee5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -243,6 +243,7 @@ extern void ghost_sched_cleanup_fork(struct task_struct *p); extern void ghost_latched_task_preempted(struct rq *rq); extern void ghost_task_preempted(struct rq *rq, struct task_struct *prev); extern unsigned long ghost_cfs_added_load(struct rq *rq); +extern int ghost_wake_agent_on_check(int cpu); extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); #ifdef CONFIG_BPF From 62b13b968960b7cefe3af96b0e6caf07dc8f4b9b Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 29 Jun 2021 14:42:33 -0400 Subject: [PATCH 009/165] sched/ghost: add bpf_ghost_wake_agent() This is a BPF helper function that wakes an agent on a cpu. It is callable by any programs of BPF_PROG_TYPE_GHOST_SCHED. Right now, that's just the skip_tick hook. Soon, it will include PNT. The caller must belong to the same enclave as the destination. That's not a huge deal, but it will keep enclaves from mucking with one another. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I924956f41dbf8fe921ce000ae1bf9231e5df44fe --- include/uapi/linux/bpf.h | 48 ++++++++++++++++++++++++++++++++++++++ kernel/sched/bpf.c | 15 ++++++++++++ scripts/bpf_helpers_doc.py | 1 + 3 files changed, 64 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 650e93ef4b5a..7a35648abe6d 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3832,6 +3832,13 @@ union bpf_attr { * Return * A pointer to a struct socket on success or NULL if the file is * not a socket. + * + * long bpf_ghost_wake_agent(struct bpf_ghost_sched *ctx, u32 cpu) + * Description + * Wakes the ghost agent on *cpu* + * + * Return + * 0 on success, < 0 on error. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -3997,6 +4004,47 @@ union bpf_attr { FN(ktime_get_coarse_ns), \ FN(ima_inode_hash), \ FN(sock_from_file), \ + FN(placeholder_999), \ + FN(placeholder_001), \ + FN(placeholder_002), \ + FN(placeholder_003), \ + FN(placeholder_004), \ + FN(placeholder_005), \ + FN(placeholder_006), \ + FN(placeholder_007), \ + FN(placeholder_008), \ + FN(placeholder_009), \ + FN(placeholder_010), \ + FN(placeholder_011), \ + FN(placeholder_012), \ + FN(placeholder_013), \ + FN(placeholder_014), \ + FN(placeholder_015), \ + FN(placeholder_016), \ + FN(placeholder_017), \ + FN(placeholder_018), \ + FN(placeholder_019), \ + FN(placeholder_020), \ + FN(placeholder_021), \ + FN(placeholder_022), \ + FN(placeholder_023), \ + FN(placeholder_024), \ + FN(placeholder_025), \ + FN(placeholder_026), \ + FN(placeholder_027), \ + FN(placeholder_028), \ + FN(placeholder_029), \ + FN(placeholder_030), \ + FN(placeholder_031), \ + FN(placeholder_032), \ + FN(placeholder_033), \ + FN(placeholder_034), \ + FN(placeholder_035), \ + FN(placeholder_036), \ + FN(placeholder_037), \ + FN(placeholder_038), \ + FN(placeholder_039), \ + FN(ghost_wake_agent), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 9e8ae143bfde..d703ef02e96d 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -16,6 +16,19 @@ #ifdef CONFIG_SCHED_CLASS_GHOST +BPF_CALL_2(bpf_ghost_wake_agent, struct bpf_ghost_sched_kern *, ctx, u32, cpu) +{ + return ghost_wake_agent_on_check(cpu); +} + +static const struct bpf_func_proto bpf_ghost_wake_agent_proto = { + .func = bpf_ghost_wake_agent, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, +}; + bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { struct bpf_ghost_sched_kern ctx = {}; @@ -240,6 +253,8 @@ static const struct bpf_func_proto * ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { + case BPF_FUNC_ghost_wake_agent: + return &bpf_ghost_wake_agent_proto; default: return bpf_base_func_proto(func_id); } diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index 867ada23281c..d7f88bd35e1f 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -469,6 +469,7 @@ class PrinterHelpers(Printer): 'struct bpf_tcp_sock', 'struct bpf_tunnel_key', 'struct bpf_xfrm_state', + 'struct bpf_ghost_sched', 'struct linux_binprm', 'struct pt_regs', 'struct sk_reuseport_md', From 5ad5514d118448cc828d56c3ca29a4f6dd176cd8 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 25 Jun 2021 16:38:47 -0400 Subject: [PATCH 010/165] sched/ghost: add a BPF program to pick_next_task() This program runs whenever ghost has nothing to do: the agent is not runnable, there is no latched task, and there is no commit to do. The program returns 1 when it thinks PNT should retry its loop, such as if it woke the agent or latched a task. To prevent BPF from causing an infinite loop (consider a program that always returns 1), we only run it at most once per global pick_next_task(). Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: If409554e14df794fb75eba5beea0f596fb7b3522 --- include/uapi/linux/bpf.h | 1 + kernel/bpf/syscall.c | 2 ++ kernel/sched/bpf.c | 65 ++++++++++++++++++++++++++++++++++++++++ kernel/sched/core.c | 1 + kernel/sched/ghost.c | 12 +++++++- kernel/sched/sched.h | 13 ++++++++ 6 files changed, 93 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 7a35648abe6d..c9e3fcebea31 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -243,6 +243,7 @@ enum bpf_attach_type { BPF_SK_LOOKUP, BPF_XDP, BPF_GHOST_SCHED_SKIP_TICK = 50, + BPF_GHOST_SCHED_PNT = 51, __MAX_BPF_ATTACH_TYPE }; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index dddfe6f5c51c..c3df31de5c1c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2005,6 +2005,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, case BPF_PROG_TYPE_GHOST_SCHED: switch (expected_attach_type) { case BPF_GHOST_SCHED_SKIP_TICK: + case BPF_GHOST_SCHED_PNT: return 0; default: return -EINVAL; @@ -2960,6 +2961,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) case BPF_XDP: return BPF_PROG_TYPE_XDP; case BPF_GHOST_SCHED_SKIP_TICK: + case BPF_GHOST_SCHED_PNT: return BPF_PROG_TYPE_GHOST_SCHED; default: return BPF_PROG_TYPE_UNSPEC; diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index d703ef02e96d..bd7986f98cfe 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -44,6 +44,36 @@ bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) return BPF_PROG_RUN(prog, &ctx) != 1; } +/* Returns true if pick_next_task_ghost should retry its loop. */ +bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) +{ + struct bpf_ghost_sched_kern ctx = {}; + struct bpf_prog *prog; + int ret; + + lockdep_assert_held(&rq->lock); + + prog = rcu_dereference(e->bpf_pnt); + if (!prog) + return false; + + /* + * BPF programs attached here may call ghost_run_gtid(), which requires + * that we not hold any RQ locks. We are called from + * pick_next_task_ghost where it is safe to unlock the RQ. + */ + rq_unpin_lock(rq, rf); + raw_spin_unlock(&rq->lock); + + ret = BPF_PROG_RUN(prog, &ctx); + + raw_spin_lock(&rq->lock); + rq_repin_lock(rq, rf); + + /* prog returns 1 meaning "retry". */ + return ret == 1; +} + static int ghost_sched_tick_attach(struct ghost_enclave *e, struct bpf_prog *prog) { @@ -69,6 +99,31 @@ static void ghost_sched_tick_detach(struct ghost_enclave *e, spin_unlock_irqrestore(&e->lock, irq_fl); } +static int ghost_sched_pnt_attach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->bpf_pnt) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + rcu_assign_pointer(e->bpf_pnt, prog); + spin_unlock_irqrestore(&e->lock, irq_fl); + return 0; +} + +static void ghost_sched_pnt_detach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + rcu_replace_pointer(e->bpf_pnt, NULL, lockdep_is_held(&e->lock)); + spin_unlock_irqrestore(&e->lock, irq_fl); +} + struct bpf_ghost_sched_link { struct bpf_link link; struct ghost_enclave *e; @@ -89,6 +144,9 @@ static void bpf_ghost_sched_link_release(struct bpf_link *link) case BPF_GHOST_SCHED_SKIP_TICK: ghost_sched_tick_detach(e, link->prog); break; + case BPF_GHOST_SCHED_PNT: + ghost_sched_pnt_detach(e, link->prog); + break; default: WARN_ONCE(1, "Unexpected release for ea_type %d", sc_link->ea_type); @@ -129,6 +187,7 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, switch (ea_type) { case BPF_GHOST_SCHED_SKIP_TICK: + case BPF_GHOST_SCHED_PNT: break; default: return -EINVAL; @@ -168,6 +227,9 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, case BPF_GHOST_SCHED_SKIP_TICK: err = ghost_sched_tick_attach(e, prog); break; + case BPF_GHOST_SCHED_PNT: + err = ghost_sched_pnt_attach(e, prog); + break; default: pr_warn("bad sched bpf ea_type %d, should be unreachable", ea_type); @@ -190,6 +252,7 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, enum ghost_sched_bpf_attach_type { GHOST_SCHED_BPF_INVALID = -1, GHOST_SCHED_BPF_TICK = 0, + GHOST_SCHED_BPF_PNT, MAX_SCHED_BPF_ATTACH_TYPE }; @@ -199,6 +262,8 @@ to_ghost_sched_bpf_attach_type(enum bpf_attach_type attach_type) switch (attach_type) { case BPF_GHOST_SCHED_SKIP_TICK: return GHOST_SCHED_BPF_TICK; + case BPF_GHOST_SCHED_PNT: + return GHOST_SCHED_BPF_PNT; default: return GHOST_SCHED_BPF_INVALID; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index be5b3c4da4e6..44afebc173c8 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5116,6 +5116,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) /* a negative 'switchto_count' indicates end of the chain */ rq->ghost.switchto_count = -rq->ghost.switchto_count; WARN_ON_ONCE(rq->ghost.switchto_count > 0); + rq->ghost.pnt_bpf_once = true; #endif /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 17d3c3d96317..85bf743970a3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -765,6 +765,8 @@ static void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) static int balance_ghost(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) { + + bool bpf_ret; struct task_struct *agent = rq->ghost.agent; if (!agent || !agent->on_rq) @@ -786,11 +788,19 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, rq_repin_lock(rq, rf); } + if (!rq->ghost.latched_task && rq->ghost.pnt_bpf_once) { + rq->ghost.pnt_bpf_once = false; + /* If there is a BPF program, this will unlock the RQ */ + rq->ghost.in_pnt_bpf = true; + bpf_ret = ghost_bpf_pnt(agent->ghost.enclave, rq, rf); + rq->ghost.in_pnt_bpf = false; + } + /* * We have something to run in 'latched_task' or a higher priority * sched_class became runnable while the rq->lock was dropped. */ - return rq->ghost.latched_task || rq_adj_nr_running(rq); + return rq->ghost.latched_task || rq_adj_nr_running(rq) || bpf_ret; } static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6d4a253ceee5..eddd16fd97b4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -113,6 +113,8 @@ struct ghost_rq { bool must_resched; /* rq->curr must reschedule in PNT */ bool check_prev_preemption; /* see 'ghost_prepare_task_switch()' */ bool skip_latched_preemption; + bool pnt_bpf_once; /* BPF runs at most once in PNT */ + bool in_pnt_bpf; /* running BPF at PNT */ int ghost_nr_running; int run_flags; /* flags passed to 'ghost_run()' */ @@ -201,6 +203,7 @@ struct ghost_enclave { #ifdef CONFIG_BPF struct bpf_prog *bpf_tick; + struct bpf_prog *bpf_pnt; #endif }; @@ -246,14 +249,24 @@ extern unsigned long ghost_cfs_added_load(struct rq *rq); extern int ghost_wake_agent_on_check(int cpu); extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); + +struct rq_flags; #ifdef CONFIG_BPF extern bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq); +extern bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, + struct rq_flags *rf); #else static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { return false; } +static inline bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, + struct rq_flags *rf) +{ + return false; +} #endif + extern void ghost_wait_for_rendezvous(struct rq *rq); extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); From b70eb8a61e2172ba6f13d30bdf27531a7fcfd5cc Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 25 Jun 2021 16:42:17 -0400 Subject: [PATCH 011/165] sched/ghost: add bpf_ghost_run_gtid() This is a helper that bpf programs can call. It attempts to latch gtid to run on the calling cpu. It is similar to the old ghost_run, but it doesn't support "idle" or "agent". Unlike bpf_ghost_wake_agent(), this helper can only be called from the PNT attach point - we can extend it to other "trusted" attach points in the future. The helper calls ghost_run_gtid(), which grabs an RQ lock, so we need to be sure that the RQ lock is not currently held by whoever is calling the BPF program. (The RQ lock is held during the tick programs). One thing to note: this calls ghost_set_pnt_state(), which may replace a latched task. Our caller was run from PNT and held the RQ, but released it. In that time, another cpu could have committed the TXN on our RQ. We could have ghost_run_gtid() abort if it sees a latched task, thereby picking the TXN over the BPF programs answer. Though either way, some task will run and the other will be preempted. I opted not to "abort on latch", since I expect we may call this helper function from a BPF function on wakeup, and in those situations, the agent (via BPF) may want to preempt a latched task. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I9e45d0ee262a51f535475152c9e58745c577257c --- include/uapi/linux/bpf.h | 9 ++++ kernel/sched/bpf.c | 30 ++++++++++- kernel/sched/ghost.c | 108 +++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 4 ++ 4 files changed, 150 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c9e3fcebea31..804d9e2346ae 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3840,6 +3840,14 @@ union bpf_attr { * * Return * 0 on success, < 0 on error. + * + * long bpf_ghost_run_gtid(struct bpf_ghost_sched *ctx, s64 gtid, u32 + * task_barrier, s32 run_flags) + * Description + * Runs (latches) task **gtid** on this cpu. + * + * Return + * 0 on success, < 0 on error. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4046,6 +4054,7 @@ union bpf_attr { FN(placeholder_038), \ FN(placeholder_039), \ FN(ghost_wake_agent), \ + FN(ghost_run_gtid), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index bd7986f98cfe..2c73d16a34a3 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -29,6 +29,23 @@ static const struct bpf_func_proto bpf_ghost_wake_agent_proto = { .arg2_type = ARG_ANYTHING, }; +BPF_CALL_4(bpf_ghost_run_gtid, struct bpf_ghost_sched_kern *, ctx, s64, gtid, + u32, task_barrier, int, run_flags) +{ + return ghost_run_gtid_on(gtid, task_barrier, run_flags, + smp_processor_id()); +} + +static const struct bpf_func_proto bpf_ghost_run_gtid_proto = { + .func = bpf_ghost_run_gtid, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, + .arg4_type = ARG_ANYTHING, +}; + bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { struct bpf_ghost_sched_kern ctx = {}; @@ -53,9 +70,12 @@ bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) lockdep_assert_held(&rq->lock); + rcu_read_lock(); prog = rcu_dereference(e->bpf_pnt); - if (!prog) + if (!prog) { + rcu_read_unlock(); return false; + } /* * BPF programs attached here may call ghost_run_gtid(), which requires @@ -70,6 +90,7 @@ bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) raw_spin_lock(&rq->lock); rq_repin_lock(rq, rf); + rcu_read_unlock(); /* prog returns 1 meaning "retry". */ return ret == 1; } @@ -320,6 +341,13 @@ ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) switch (func_id) { case BPF_FUNC_ghost_wake_agent: return &bpf_ghost_wake_agent_proto; + case BPF_FUNC_ghost_run_gtid: + switch (prog->expected_attach_type) { + case BPF_GHOST_SCHED_PNT: + return &bpf_ghost_run_gtid_proto; + default: + return NULL; + } default: return bpf_base_func_proto(func_id); } diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 85bf743970a3..72c6f9f2fc00 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4327,6 +4327,114 @@ SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, return error; } +static int __ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, + int cpu, bool check_caller_enclave) +{ + struct rq_flags rf; + struct rq *rq; + int err = 0; + struct task_struct *next; + + const int supported_flags = RTLA_ON_PREEMPT | + RTLA_ON_BLOCKED | + RTLA_ON_YIELD | + NEED_L1D_FLUSH | + ALLOW_TASK_ONCPU | + ELIDE_PREEMPT | + 0; + + WARN_ON_ONCE(preemptible()); + + if (cpu < 0) + return -EINVAL; + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -ERANGE; + + if (!run_flags_valid(run_flags, supported_flags, gtid)) + return -EINVAL; + + if (check_caller_enclave && + !check_same_enclave(smp_processor_id(), cpu)) { + return -EXDEV; + } + + rcu_read_lock(); + next = find_task_by_gtid(gtid); + if (next == NULL || next->ghost.agent) { + rcu_read_unlock(); + return -ENOENT; + } + rq = task_rq_lock(next, &rf); + rcu_read_unlock(); + + err = validate_next_task(rq, next, task_barrier, /*state=*/ NULL); + if (err) { + task_rq_unlock(rq, next, &rf); + return err; + } + + if (task_running(rq, next)) { + task_rq_unlock(rq, next, &rf); + return -EBUSY; + } + + rq = ghost_move_task(rq, next, cpu, &rf); + + /* + * Must not latch a task if there is no agent, otherwise PNT will never + * see it. (Latched tasks are cleared when the agent exits). + */ + if (unlikely(!rq->ghost.agent)) { + task_rq_unlock(rq, next, &rf); + return -EINVAL; + } + + /* + * If the RQ is in the middle of PNT (where it briefly unlocks), + * ghost_can_schedule() is not accurate. rq->curr is still the + * task that is scheduling, and it may be from CFS. + * + * This does not mean that our thread is in PNT. It's possible that we + * are a wakeup BPF program and the PNT thread has unlocked the RQ and + * is running its own BPF program. Either way, PNT will check for a + * latched task or for a higher priority task when it relocks. + */ + if (!rq->ghost.in_pnt_bpf && !ghost_can_schedule(rq, gtid)) { + task_rq_unlock(rq, next, &rf); + return -ENOSPC; + } + + ghost_set_pnt_state(rq, next, run_flags); + task_rq_unlock(rq, next, &rf); + + return 0; +} + +/* + * Attempts to run gtid on cpu. Returns 0 or -error. + * + * Called from BPF helpers. The programs that can call those helpers are + * explicitly allowed in ghost_sched_func_proto. + */ +int ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, int cpu) +{ + return __ghost_run_gtid_on(gtid, task_barrier, run_flags, cpu, + /*check_caller_enclave=*/ false); +} + +/* + * Attempts to run gtid on cpu. Returns 0 or -error. + * + * Like ghost_run_gtid_on, but checks that the calling CPU and the target cpu + * are in the same enclave. + */ +int ghost_run_gtid_on_check(gtid_t gtid, u32 task_barrier, int run_flags, + int cpu) +{ + return __ghost_run_gtid_on(gtid, task_barrier, run_flags, cpu, + /*check_caller_enclave=*/ true); +} + static inline bool _ghost_txn_ready(int cpu, int *commit_flags) { struct ghost_txn *txn; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index eddd16fd97b4..ee605aab67da 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -249,6 +249,10 @@ extern unsigned long ghost_cfs_added_load(struct rq *rq); extern int ghost_wake_agent_on_check(int cpu); extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); +extern int ghost_run_gtid_on(s64 gtid, u32 task_barrier, int run_flags, + int cpu); +extern int ghost_run_gtid_on_check(s64 gtid, u32 task_barrier, int run_flags, + int cpu); struct rq_flags; #ifdef CONFIG_BPF From 8c313a434b3fad1f994b6790bc0edd8f70b95a9d Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 14 Jul 2021 23:05:42 -0700 Subject: [PATCH 012/165] sched: avoid reordering MSG_TASK_NEW and MSG_TASK_AFFINITY_CHANGED. It is possible to reorder TASK_NEW and TASK_AFFINITY_CHANGED msgs when an oncpu task executing sched_setaffinity() is switched into ghost. This violates the assumption that TASK_NEW is the first msg produced and confuses the agent. The kernel defers producing a TASK_NEW msg when a running task switches into ghost until the task schedules (the ghost set_curr_task handler forces the issue by setting NEED_RESCHED on the task). The expectation is that the task will schedule immediately and TASK_NEW is produced via ghost_prepare_task_switch(). In some cases however this assumption is broken. For e.g. when an oncpu task is moved to the ghost sched_class while it is in the kernel doing sched_setaffinity(). Initial conditions: task p is running on cpu-x in the cfs sched_class. cpu-x cpu-y T0 sched_setscheduler(p, ghost) task_rq_lock(p) T1 sched_setaffinity(p, new_mask) spinning on task_rq_lock(p) held by cpu-y. T2 p->sched_class = ghost_sched_class T3 p->ghost.new_task = true via switched_to_ghost(). MSG_TASK_NEW deferred until 'p' gets offcpu. T4 set_tsk_need_resched(curr) via set_curr_task_ghost() to get 'p' offcpu. T5 task_rq_unlock(p) before returning from sched_setscheduler(). T6 ... acquire task_rq_lock(p) p->allowed_cpus = new_mask T7 produce TASK_AFFINITY_CHANGED msg via set_cpus_allowed_ghost() while the TASK_NEW msg is still deferred. Tested: //third_party/ghost/api_test (cl/384364382) Effort: sched/ghost Change-Id: I539718257123a115d2fcbace0f34b23263d2c5fa --- kernel/sched/ghost.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 72c6f9f2fc00..09310124aa53 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3688,6 +3688,15 @@ static void task_deliver_msg_affinity_changed(struct rq *rq, { struct ghost_msg_payload_task_affinity_changed payload; + /* + * A running task can be switched into ghost while it is executing + * sched_setaffinity. In this case the TASK_NEW msg is held pending + * until the task schedules and producing the TASK_AFFINITY_CHANGED + * msg is useless at best since the agent has no idea about this task. + */ + if (unlikely(p->ghost.new_task)) + return; + if (__task_deliver_common(rq, p)) return; From 7d8ba102ee829ddfa06c67356c6f0ccdf60ffdee Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 14 Jul 2021 23:47:40 -0700 Subject: [PATCH 013/165] sched: avoid reordering MSG_TASK_NEW and MSG_TASK_DEPARTED It is possible to reorder TASK_NEW and TASK_DEPARTED msgs when an oncpu task executing sched_setscheduler(cfs) is switched into ghost. This violates the assumption that TASK_NEW is the first msg produced and confuses the agent. The kernel defers producing a TASK_NEW msg when a running task switches into ghost until the task schedules (the ghost set_curr_task handler forces the issue by setting NEED_RESCHED on the task). The expectation is that the task will schedule immediately and TASK_NEW is produced via ghost_prepare_task_switch(). In some cases however this assumption is broken. For e.g. when an oncpu task is moved to the ghost sched_class while it is in the kernel doing sched_setscheduler(cfs). Initial conditions: task p is running on cpu-x in the cfs sched_class. cpu-x cpu-y T0 sched_setscheduler(p, ghost) task_rq_lock(p) T1 sched_setscheduler(p, cfs) spinning on task_rq_lock(p) held by cpu-y. T2 p->sched_class = ghost_sched_class T3 p->ghost.new_task = true via switched_to_ghost(). MSG_TASK_NEW deferred until 'p' gets offcpu. T4 set_tsk_need_resched(curr) via set_curr_task_ghost() to get 'p' offcpu. T5 task_rq_unlock(p) before returning from sched_setscheduler(). T6 ... acquire task_rq_lock(p) p->sched_class = cfs_sched_class T7 produce TASK_DEPARTED msg via switched_from_ghost() while the TASK_NEW msg is still deferred. Tested: //third_party/ghost/api_test (cl/384364382) Effort: sched/ghost Change-Id: I15b44a04ddab399509ccfa67503ce4813bc5c5e2 --- kernel/sched/ghost.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 09310124aa53..b48b6701411d 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -549,6 +549,26 @@ static void switched_to_ghost(struct rq *rq, struct task_struct *p) static void switched_from_ghost(struct rq *rq, struct task_struct *p) { + /* + * A running task can be switched into ghost while it is executing + * sched_setscheduler(cfs). Make sure TASK_NEW is produced before + * TASK_DEPARTED in this case. + * + * Note that unlike TASK_AFFINITY_CHANGED (which we just forget in + * a similar situation) we must produce TASK_DEPARTED so the task's + * status_word is freed by the agent. + * + * Also note that we must call ghost_task_new() here before calling + * release_from_ghost() since the former sets things up for the + * latter to tear down (e.g. adding task to enclave->task_list). + */ + if (unlikely(p->ghost.new_task)) { + WARN_ON_ONCE(!task_current(rq, p)); + p->ghost.new_task = false; + ghost_task_new(rq, p); + ghost_wake_agent_of(p); + } + release_from_ghost(rq, p); /* @@ -1391,8 +1411,8 @@ static void task_woken_ghost(struct rq *rq, struct task_struct *p) ghost_sw_set_flag(sw, GHOST_SW_TASK_RUNNABLE); if (unlikely(p->ghost.new_task)) { - ghost_task_new(rq, p); p->ghost.new_task = false; + ghost_task_new(rq, p); goto done; } task_deliver_msg_wakeup(rq, p); @@ -3760,6 +3780,7 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) lockdep_assert_held(&p->pi_lock); WARN_ON_ONCE(is_cached_task(rq, p)); + WARN_ON_ONCE(p->ghost.new_task); spin_lock_irqsave(&e->lock, flags); From 9a1e35766301c4d062c0a566e61fea3aea34bf9a Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 20 Jul 2021 11:10:59 -0700 Subject: [PATCH 014/165] sched: simplify latched_task preemption during context_switch. Before "a9a7f79: check need_resched with rq->lock held before doing switchto." it was possible for transaction commit to race with a task entering context_switch due to switchto. After a9a7f79 the race is no longer possible so we can simplify the 'latched_task' preemption logic in ghost_prepare_task_switch(). Tested: - kokonut test //sched:ghost_smoketest Indus http://sponge2/1632130f-df7c-4aa0-9141-31f810d9ad30 (one known failure tracked in b/192287338) Arcadia http://sponge2/e4045c4a-4c0e-4b72-ada0-4a9d4fec15bf - agent_muppet && switchto_test Change-Id: I7a9879cf50e624de9092acea18c5ea4600adbc3e Effort: sched/ghost --- kernel/sched/core.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 44afebc173c8..3022602233db 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4300,15 +4300,29 @@ static inline void ghost_prepare_task_switch(struct rq *rq, } /* - * CPU in a ghost switchto chain or switching to a non-ghost task - * while a task is latched. + * CPU is switching to a non-ghost task while a task is latched. * * Treat this like latched_task preemption because we don't know when * the CPU will be available again so no point in keeping it latched. */ - if (rq->ghost.latched_task && - (!task_has_ghost_policy(next) || rq->ghost.switchto_count > 0)) { + if (rq->ghost.latched_task) { + /* + * If 'next' was returned from pick_next_task_ghost() then + * 'latched_task' must have been cleared. Conversely if + * there is 'latched_task' then 'next' could not have + * been returned from pick_next_task_ghost(). + */ + WARN_ON_ONCE(task_has_ghost_policy(next) && + !is_agent(rq, next)); ghost_latched_task_preempted(rq); + + /* + * XXX the WARN above is susceptible to a false-negative + * when pick_next_task_ghost returns the idle task. This + * is not the common case but it highlights that what we + * really need to check is the sched_class that produced + * 'next'. + */ } /* From ffb8a28c7472151eda04e7d6b588638b8825d448 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 20 Jul 2021 11:21:36 -0700 Subject: [PATCH 015/165] sched: WARN if preempt msg is produced while a switchto chain is active. This WARN proved helpful while debugging an issue where a TASK_PREEMPT was being produced for a task that was in an active switchto chain. Note that we should be producing exactly two messages for a switchto chain: - TASK_SWITCHTO when the switchto chain begins. - One of TASK_PREEMPT/BLOCKED/YIELD/DEPARTED when the chain is broken. Producing the TASK_PREEMPT while the chain was still in progress violated this contract and caused agent to CHECK fail. Tested: Ran all unit tests in virtme to verify that the warning is not seen. Change-Id: Ib187a13dff95ce63cede2e065214d9e9c2d4e5dc Effort: sched/ghost --- kernel/sched/ghost.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b48b6701411d..c964001120f0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3652,6 +3652,20 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, if (__task_deliver_common(rq, p)) return; + /* + * It doesn't make sense to produce a TASK_PREEMPT while a switchto + * chain is active. + * + * Stated differently TASK_PREEMPT is only expected when: + * 1. the task is not part of an active switchto chain: + * - a task that got oncpu via __schedule(). + * - a latched_task. + * 2. the task was in an active switchto chain that is now broken: + * - preempted by a higher priority sched_class. + * - preempted by the agent doing a transaction commit. + */ + WARN_ON_ONCE(from_switchto && rq->ghost.switchto_count > 0); + payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); From 6ad4f7410c2ecde9da7aa9799ee7cc4c90a1bdde Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 20 Jul 2021 14:14:38 -0700 Subject: [PATCH 016/165] sched: fix null-pointer deref when delivering TASK_AFFINITY_CHANGED. This is easily reproduced by running //third_party/ghost/api_test --gunit_filter=ApiTest.SchedAffinityRace The panic was due to deferencing 'task->ghost.status_word' in task_barrier_inc() when delivering MSG_TASK_AFFINITY_CHANGED. Initial condition is task 'p' is executing on CPU2 in ghost: CPU1 CPU2 T0 ghost task 'p' oncpu in do_exit() but hasn't lost its identity. T1 sched_setaffinity(p) is able to find 'p' and take a ref on its task_struct. T2 'p' schedules for the last time and task_dead_ghost(p) releases resources in 'p->ghost' like the status_word. T3 set_cpus_allowed_ptr_common() takes task_rq_lock(p) and calls set_cpus_allowed_ghost() which panics the kernel when trying to deliver the AFFINITY_CHANGED msg. Tested: //third_party/ghost/api_test --gunit_filter=ApiTest.SchedAffinityRace in a loop for 10000 times. Change-Id: I04d3e731992f7552b9f7a71754547f881704f1ae Effort: sched/ghost --- kernel/sched/ghost.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index c964001120f0..587ee1b6b717 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1387,6 +1387,13 @@ static void set_cpus_allowed_ghost(struct task_struct *p, locked = true; } + /* + * N.B. sched_setaffinity() can race with exit() such that the task + * is already dead and contents of 'p->ghost' are no longer valid. + * + * Task msg delivery handles this properly but be careful when + * accessing 'p->ghost' directly in this function. + */ task_deliver_msg_affinity_changed(rq, p); if (locked) @@ -3579,6 +3586,16 @@ static inline int __task_deliver_common(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); + /* + * Some operations (e.g. sched_setaffinity()) can race with task death + * in which case the 'status_word' is off limits and will cause an oops + * in task_barrier_inc() below. + */ + if (unlikely(!p->ghost.status_word)) { + WARN_ON_ONCE(p->state != TASK_DEAD); + return -1; + } + /* * Ignore the agent's task_state changes. * From ed828b95941853bf21dcdc41da3ce2c9845900e8 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 21 Jul 2021 09:13:39 -0700 Subject: [PATCH 017/165] sched: WARN if a deferred TASK_NEW is detected during msg delivery. Suggested by: brho@ Tested: run all unit tests in virtme and verify warning is not produced Change-Id: I2a4ae2daaefb422d7c6d95f1d4c8cf0b8ee4a1fd Effort: sched/ghost --- kernel/sched/ghost.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 587ee1b6b717..ae4b3b61d56b 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3605,6 +3605,14 @@ static inline int __task_deliver_common(struct rq *rq, struct task_struct *p) if (unlikely(is_agent(rq, p))) return -1; + /* + * There should not be any deferred TASK_NEW at this point so WARN + * and proceed. The agent will also flag that it received a msg from + * an unknown task which is useful in case kernel log is unavailable. + * (for e.g. see b/193059731). + */ + WARN_ON_ONCE(p->ghost.new_task); + /* * Increment the barrier even if we are not going to send a message due * to a missing queue, since the barrier protects some parts of the SW's From e8e03923cd8fd563c66653069e5641d148274a4c Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 22 Jul 2021 15:54:42 -0700 Subject: [PATCH 018/165] sched: only call update_curr_ghost if a task is accumulating cputime. Only a task that is running can accumulate cputime: specifically a task that is runnable but not running could not have accumulated cputime since the last time it got off the cpu. While debugging cl/386301191 it became evident that dequeue_task_ghost is routinely called for runnable-but-not-running tasks when migrating a task via ghost_move_task() during txn commit. In this case the call to update_curr_ghost() serves no purpose and is just overhead. Fix this by calling update_curr_ghost() from dequeue_task_ghost() only when the task being dequeued was running. Change-Id: Ic3421f250a8ffc4ccf16b54309c1589aaa9cacd7 Effort: sched/ghost --- kernel/sched/ghost.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index ae4b3b61d56b..67884ab57134 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -587,7 +587,18 @@ static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) const bool sleeping = flags & DEQUEUE_SLEEP; struct ghost_status_word *sw = p->ghost.status_word; - update_curr_ghost(rq); + /* + * A task is accumulating cputime only when it is oncpu. Thus it is + * useless to call update_curr_ghost for a task that is 'on_rq' but + * is not running (in this case we'll just update the cputime of + * whatever task happens to be oncpu). + * + * Ordinarily we wouldn't care but we routinely dequeue_task_ghost() + * when migrating a task via ghost_move_task() during txn commit so + * we call update_curr_ghost() only if 'p' is actually running. + */ + if (task_current(rq, p)) + update_curr_ghost(rq); BUG_ON(rq->ghost.ghost_nr_running <= 0); rq->ghost.ghost_nr_running--; From 348e6331663639f4dfcd7e55a2c5706d02a670da Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Wed, 7 Jul 2021 18:05:20 -0700 Subject: [PATCH 019/165] sched: Allow RTLA_ON_IDLE to be used when committing a txn that targets a remote CPU Currently, RTLA_ON_IDLE may only be used when an agent is yielding itself. However, there are scenarios where a global agent is scheduling remote CPUs and it wants the satellite agents on those remote CPUs to wake up when the remote CPUs go idle. Thus, this KCL relaxes this constraint on RTLA_ON_IDLE so that the global agent may use that flag when committing transactions that target remote CPUs. Tested: Coming soon Effort: sched/ghost Change-Id: I05f23be82faf189addfbad69cec14a33b6c8713c --- kernel/sched/ghost.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 67884ab57134..37f4682cacff 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4254,13 +4254,21 @@ static inline bool same_process(struct task_struct *p, struct task_struct *q) return p->group_leader == q->group_leader; } +/* + * Checks that the run flags are valid for a ghOSt txn or a ghost_run syscall. + */ static inline bool run_flags_valid(int run_flags, int valid_run_flags, gtid_t gtid) { if (run_flags & ~valid_run_flags) return false; - if ((run_flags & RTLA_ON_IDLE) && (gtid != GHOST_NULL_GTID)) + /* + * RTLA_ON_IDLE can be combined with GHOST_NULL_GTID (which is equal to + * 0), but should not be combined with any other special GTIDs. + */ + BUILD_BUG_ON(GHOST_NULL_GTID != 0); + if ((run_flags & RTLA_ON_IDLE) && gtid < 0) return false; if ((run_flags & NEED_CPU_NOT_IDLE) && (gtid != GHOST_IDLE_GTID)) From c0b5276c057f2aa2c3520cae6587b9442caf1d41 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 28 Jul 2021 13:31:39 -0400 Subject: [PATCH 020/165] sched/ghost: add run_flags to trace_sched_ghost_latched() This helps schedghostidle determine if the latcher was BPF or not. Tested: schedghostidle with BPF Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ibd18726d057eeb9396afbd27688477855106d14d --- include/trace/events/sched.h | 10 +++++++--- kernel/sched/ghost.c | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 683d8fdfa78d..121e4a5b785c 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -687,21 +687,25 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TRACE_EVENT(sched_ghost_latched, TP_PROTO(struct task_struct *old, - struct task_struct *new), + struct task_struct *new, + int run_flags), - TP_ARGS(old, new), + TP_ARGS(old, new, run_flags), TP_STRUCT__entry( __field(pid_t, old_pid) __field(pid_t, new_pid) + __field(int, run_flags) ), TP_fast_assign( __entry->old_pid = old ? old->pid : 0; __entry->new_pid = new ? new->pid : 0; + __entry->run_flags = run_flags; ), - TP_printk("old_pid=%d, new_pid=%d", __entry->old_pid, __entry->new_pid) + TP_printk("old_pid=%d, new_pid=%d, run_flags=0x%x", __entry->old_pid, + __entry->new_pid, __entry->run_flags) ); /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 37f4682cacff..879111fe1ced 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3975,7 +3975,7 @@ static void task_dead_ghost(struct task_struct *p) static void ghost_set_pnt_state(struct rq *rq, struct task_struct *p, int run_flags) { - trace_sched_ghost_latched(rq->ghost.latched_task, p); + trace_sched_ghost_latched(rq->ghost.latched_task, p, run_flags); lockdep_assert_held(&rq->lock); From 3ffe3fb25f86cfe8c235670c3d4cfcf0193042b4 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 20 Jul 2021 15:33:52 -0400 Subject: [PATCH 021/165] sched/ghost: add MSG_TASK_LATCHED If the agent is using BPF-PNT, knowing that a task got latched simplifies the agent's state machine and accounting. Instead of inferring success by getting a preempt or block message, the agent can request a "latched" message. The main benefit to the agent is that it can easily determine the state of a given Cpu. e.g. TaskLatched handler sets cs->current. Once cs->current is set, that lets the rest of the scheduler interact with that Cpu as if it was latched by a normal transaction. For instance, it can tell if a cpu is available or not. Additionally, the agent knows when the task was latched, which helps with its accounting, and we do not need to have yet-another case in all of the message handles (e.g. "from_bpf", just like "from_switchto"). The agent could check with its BPF maps to determine the outcome of a BPF-PNT, however this is outside the stream of messages for a task, which makes the agent's job difficult. (Specifically, if the agent is looking at a BPF map for the outcome of a successful transaction, it needs to be careful: it may already have handled a TaskDeparted for that task. The agent could track which BPF slot every task used, but that does proscribe specific ways that the agent must use BPF, versus a "fire and forget" model for running tasks.) It helps the agent's cpu accounting for us to always send TASK_LATCHED for a cpu after the previous task has left the cpu (e.g. preempt). However, if the BPF-Latched task is preempted before it got on cpu, we might send the message for TaskLatched before the previous task's "got off cpu" message. To help the agent handle this scenario, we tell the agent this was a latched_preempt. Also, that WARN_ON_ONCE we had can get hit if you have a latched task when the agent exits. To be submitted in conjunction with cl/387829782. Tested: edf_test Change-Id: I95892b9985e88081884c2e75ef996880f07064a5 Signed-off-by: Barret Rhoden --- include/uapi/linux/ghost.h | 11 +++++- kernel/sched/core.c | 3 ++ kernel/sched/ghost.c | 70 +++++++++++++++++++++++++++++++++----- kernel/sched/sched.h | 1 + 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 4709342da9eb..9b9ed41b6862 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 36 +#define GHOST_VERSION 37 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -174,6 +174,7 @@ enum { MSG_TASK_DEPARTED, MSG_TASK_SWITCHTO, MSG_TASK_AFFINITY_CHANGED, + MSG_TASK_LATCHED, /* cpu messages */ MSG_CPU_TICK = _MSG_CPU_FIRST, @@ -249,6 +250,13 @@ struct ghost_msg_payload_task_switchto { int cpu; }; +struct ghost_msg_payload_task_latched { + uint64_t gtid; + uint64_t commit_time; + int cpu; + char latched_preempt; +}; + struct ghost_msg_payload_cpu_not_idle { int cpu; uint64_t next_gtid; @@ -358,6 +366,7 @@ enum ghost_base_ops { #define ELIDE_PREEMPT (1 << 9) /* Do not send TASK_PREEMPT if we preempt * a previous ghost task on this cpu */ +#define SEND_TASK_LATCHED (1 << 10) /* Send TASK_LATCHED at commit time */ /* txn->commit_flags */ enum txn_commit_at { diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 3022602233db..b56d47a0912d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4325,6 +4325,9 @@ static inline void ghost_prepare_task_switch(struct rq *rq, */ } + if (task_has_ghost_policy(next) && !is_agent(rq, next)) + ghost_task_got_oncpu(rq, next); + /* * The last task in the chain scheduled (blocked/yielded/preempted). */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 879111fe1ced..bd14e5528131 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -70,6 +70,8 @@ static void task_deliver_msg_affinity_changed(struct rq *rq, struct task_struct *p); static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p); static void task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p); +static void task_deliver_msg_latched(struct rq *rq, struct task_struct *p, + bool latched_preempt); static bool cpu_deliver_msg_tick(struct rq *rq); static int task_target_cpu(struct task_struct *p); static int agent_target_cpu(struct rq *rq); @@ -1072,6 +1074,15 @@ void ghost_latched_task_preempted(struct rq *rq) WARN_ON_ONCE(!rq->ghost.agent); if (task_has_ghost_policy(latched)) { + /* + * Normally, TASK_LATCHED is not sent until we context switch to + * the task. The agent is expecting this message before + * TASK_PREEMPT. + */ + if (rq->ghost.run_flags & SEND_TASK_LATCHED) { + task_deliver_msg_latched(rq, latched, true); + rq->ghost.run_flags &= ~SEND_TASK_LATCHED; + } _ghost_task_preempted(rq, latched, true); ghost_wake_agent_of(latched); } else { @@ -3712,11 +3723,12 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, static void task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_blocked payload = {.gtid = gtid(p)}; + struct ghost_msg_payload_task_blocked payload; if (__task_deliver_common(rq, p)) return; + payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); payload.from_switchto = ghost_in_switchto(rq); @@ -3776,6 +3788,22 @@ static void task_deliver_msg_affinity_changed(struct rq *rq, sizeof(payload)); } +static void task_deliver_msg_latched(struct rq *rq, struct task_struct *p, + bool latched_preempt) +{ + struct ghost_msg_payload_task_latched payload; + + if (__task_deliver_common(rq, p)) + return; + + payload.gtid = gtid(p); + payload.commit_time = ktime_get_ns(); + payload.cpu = cpu_of(rq); + payload.latched_preempt = latched_preempt; + + produce_for_task(p, MSG_TASK_LATCHED, &payload, sizeof(payload)); +} + static inline bool deferrable_wakeup(struct task_struct *p) { #ifdef notyet @@ -3857,14 +3885,6 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) p->ghost.status_word = NULL; if (unlikely(is_agent(rq, p))) { - /* We don't allow agents to setsched away from ghost */ - WARN_ON_ONCE(p->state != TASK_DEAD); - rq->ghost.agent = NULL; - rq->ghost.run_flags = 0; - WRITE_ONCE(rq->ghost.agent_barrier, 0); - p->ghost.agent = false; - VM_BUG_ON(rq->ghost.blocked_in_run); - /* * Unlatched any tasks that had been committed. Latched tasks * will be selected in PNT after a new agent attaches, which @@ -3883,6 +3903,19 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) */ if (rq->ghost.latched_task) ghost_latched_task_preempted(rq); + /* We don't allow agents to setsched away from ghost */ + WARN_ON_ONCE(p->state != TASK_DEAD); + rq->ghost.agent = NULL; + /* + * Clear run_flags after dealing with latched_task: the + * run_flags pertain to the latched_task, especially for + * SEND_TASK_LATCHED. + */ + rq->ghost.run_flags = 0; + WRITE_ONCE(rq->ghost.agent_barrier, 0); + p->ghost.agent = false; + VM_BUG_ON(rq->ghost.blocked_in_run); + /* * Clean up any pending transactions. We need to do this here, * in addition to in setsched, since the global agent may be @@ -4137,6 +4170,23 @@ void ghost_task_preempted(struct rq *rq, struct task_struct *p) _ghost_task_preempted(rq, p, false); } +void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_held(&rq->lock); + VM_BUG_ON(task_rq(p) != rq); + + /* + * We must defer sending TASK_LATCHED until any prev ghost tasks got off + * cpu. Otherwise the agent will have a hard time reconciling the + * current cpu state. + */ + if (rq->ghost.run_flags & SEND_TASK_LATCHED) { + task_deliver_msg_latched(rq, p, false); + /* Do not send the message more than once per commit. */ + rq->ghost.run_flags &= ~SEND_TASK_LATCHED; + } +} + static void ghost_task_new(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); @@ -4429,6 +4479,7 @@ static int __ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, NEED_L1D_FLUSH | ALLOW_TASK_ONCPU | ELIDE_PREEMPT | + SEND_TASK_LATCHED | 0; WARN_ON_ONCE(preemptible()); @@ -4666,6 +4717,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, NEED_CPU_NOT_IDLE | ALLOW_TASK_ONCPU | ELIDE_PREEMPT | + SEND_TASK_LATCHED | 0; VM_BUG_ON(preemptible()); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ee605aab67da..bede66d530d7 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -245,6 +245,7 @@ extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); extern void ghost_latched_task_preempted(struct rq *rq); extern void ghost_task_preempted(struct rq *rq, struct task_struct *prev); +extern void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p); extern unsigned long ghost_cfs_added_load(struct rq *rq); extern int ghost_wake_agent_on_check(int cpu); extern void ghost_wake_agent_of(struct task_struct *p); From 63bb34c7f7b36f42dbfaec41c1ef1b29973dede2 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 29 Jul 2021 10:58:51 -0400 Subject: [PATCH 022/165] sched/ghost: add was_latched to TASK_PREEMPT This field tells the agent that the task never actually got on cpu and was preempted in the latched state. Despite our attempts to maintain ordering of messages, it is possible to send latched preemption messages before the previous task got off cpu. I ran into this with BPF-PNT, but I have a hard time recreating the scenario. I think it was something like this: 01: task0 on_cpu in switchto 02: spurious PNT call, perhaps CFS briefly woke and migrated a task 03: since t0 was in switchto, we set must_resched, so we have to pick some other task to run. 04: there is no latched task, so BPF runs 05: BPF latches a task: task1. the RQ is unlocked briefly in here. 06: the agent wants to run task2. it issues a txn while the RQ is still unlocked. 07: task2 gets latched. to do so, we must unlatch task1. this sends TASK_LATCHED and TASK_PREEMPT for task1. 08: commit complete, unlock the RQ 09: we're still in PNT, it grabs the RQ lock and continues 10: task2 is latched and selected to run 11: context switch from task0 to task2, producing TASK_PREEMPT (with from_switchto) for task0. The order of messages was: - Latched task1 - Preempt task1 - Preempt task0 That's a little weird, since userspace thought that task0 was on cpu, yet it receives Latched for t1. We could attempt to send the preempt for t0 when we latch t1, however it turns out we easily send latched-preempts out of order. Consider an agent issuing several transactions for the same cpu: 01: task0 on_cpu in a non-preemptible region 02: agent commits/latches task1. sets need_resched. 03: before we run PNT, the agent sees the txn from t1 completed, and issues another. 04: agent latches task2. when doing so, it preempts task1. (latched_task_preemption), send TASK_PREEMPT for t1. 05: PNT runs, ghost_produce_prev_msgs sets check_preempt_prev 06: PNT picks task2, since it was latched 07: context switch from t0 to t2, send TASK_PREEMPT for t0. The order of messages was: - Preempt task1 - Preempt task0 Even though task0 was on cpu. The agent can handle this, since it knows the transactions completed, and it can adjust cs->current and cs->next. But from looking at the messages, you can't easily construct the cpu's state. The agent had to use external information: success of the transactions. If the agent requested SEND_TASK_LATCHED, the order of messages would be: - Latched task0 (from before step 01) - Latched task1 - Preempt task1 - Preempt task0 Madness! We can't send the messages for task0 when we latch task1, since we don't know yet if task0 will block or yield or be preempted, so we'll have to live with this. To be submitted in conjunction with cl/387829782. Tested: agent + switchto_test and simple_exp Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I9f2a62eff021eafdaf50763b36e5883a5cae397b --- include/uapi/linux/ghost.h | 3 ++- kernel/sched/ghost.c | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 9b9ed41b6862..1c884e27566b 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 37 +#define GHOST_VERSION 38 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -195,6 +195,7 @@ struct ghost_msg_payload_task_preempt { uint64_t runtime; /* cumulative runtime in ns */ int cpu; char from_switchto; + char was_latched; }; struct ghost_msg_payload_task_yield { diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index bd14e5528131..94e91a6f08d0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3692,7 +3692,7 @@ static void task_deliver_msg_yield(struct rq *rq, struct task_struct *p) } static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, - bool from_switchto) + bool from_switchto, bool was_latched) { struct ghost_msg_payload_task_preempt payload; @@ -3717,6 +3717,7 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); payload.from_switchto = from_switchto; + payload.was_latched = was_latched; produce_for_task(p, MSG_TASK_PREEMPT, &payload, sizeof(payload)); } @@ -4148,7 +4149,7 @@ static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, from_switchto = was_latched ? false : ghost_in_switchto(rq); /* Produce MSG_TASK_PREEMPT into 'p->ghost.dst_q' */ - task_deliver_msg_preempt(rq, p, from_switchto); + task_deliver_msg_preempt(rq, p, from_switchto, was_latched); /* * Wakeup agent on this CPU. From 36bdde1b456faa75eeaae00ddd2e1a8f8d59462e Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 29 Jul 2021 12:46:39 -0400 Subject: [PATCH 023/165] sched/ghost: add cpu_seqnum to messages with a cpu parameter In a couple places, we've relied on the fact that we had a single message channel in the global agent, forcing an ordering of messages for a given cpu, such that we handle the messages in the order they were sent. That is no longer true, as we move to having multiple channels with tasks sharded among the channels. Messages for a task are always sent in order to a particular channel, but tasks running on the same cpu might not use the same channel. To allow the agent to manage cpu state when receiving out-of-order messages pertaining to a cpu, we now send the cpu_seqnum. The cpu_seqnum is a per-cpu "history" counter included with all messages that contain cpu state. This state depends on the message. For TASK_LATCHED, it is that the task is latched on that cpu. A TASK_PREEMPT for task0 on cpuA that means "cpuA has no ghost task running" for that instant when we sent the preempt. That state is true when we send the message. But right after the message was sent, we could have a ghost task on cpu: perhaps by a completed transaction, which does not involve a message. The agent can use the cpu_seqnum to discard old cpu state information. This requires that all cpu state be determined by the current message. It'd be a pain to recreate state by handling *all* messages. Addtionally, we'll eventually need to handle losing messages, so we don't want to rely on reconstructing state from out-of-order messages, since some messages may never arrive. To be submitted in conjunction with cl/387829782. Tested: edf_test, agent + switchto_test. No changes in userspace yet. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I088cf6de2e7ade84f29cd53b05e1854dd01190a4 --- include/uapi/linux/ghost.h | 8 +++++++- kernel/sched/ghost.c | 6 ++++++ kernel/sched/sched.h | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 1c884e27566b..3a8f449c9184 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 38 +#define GHOST_VERSION 39 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -193,6 +193,7 @@ struct ghost_msg_payload_task_new { struct ghost_msg_payload_task_preempt { uint64_t gtid; uint64_t runtime; /* cumulative runtime in ns */ + uint64_t cpu_seqnum; /* cpu sequence number */ int cpu; char from_switchto; char was_latched; @@ -201,6 +202,7 @@ struct ghost_msg_payload_task_preempt { struct ghost_msg_payload_task_yield { uint64_t gtid; uint64_t runtime; /* cumulative runtime in ns */ + uint64_t cpu_seqnum; int cpu; char from_switchto; }; @@ -208,6 +210,7 @@ struct ghost_msg_payload_task_yield { struct ghost_msg_payload_task_blocked { uint64_t gtid; uint64_t runtime; /* cumulative runtime in ns */ + uint64_t cpu_seqnum; int cpu; char from_switchto; }; @@ -218,6 +221,7 @@ struct ghost_msg_payload_task_dead { struct ghost_msg_payload_task_departed { uint64_t gtid; + uint64_t cpu_seqnum; int cpu; char from_switchto; }; @@ -248,12 +252,14 @@ struct ghost_msg_payload_task_wakeup { struct ghost_msg_payload_task_switchto { uint64_t gtid; uint64_t runtime; /* cumulative runtime in ns */ + uint64_t cpu_seqnum; int cpu; }; struct ghost_msg_payload_task_latched { uint64_t gtid; uint64_t commit_time; + uint64_t cpu_seqnum; int cpu; char latched_preempt; }; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 94e91a6f08d0..97ec086ec1c7 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3686,6 +3686,7 @@ static void task_deliver_msg_yield(struct rq *rq, struct task_struct *p) payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); + payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; payload.from_switchto = ghost_in_switchto(rq); produce_for_task(p, MSG_TASK_YIELD, &payload, sizeof(payload)); @@ -3716,6 +3717,7 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); + payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; payload.from_switchto = from_switchto; payload.was_latched = was_latched; @@ -3732,6 +3734,7 @@ static void task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); + payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; payload.from_switchto = ghost_in_switchto(rq); produce_for_task(p, MSG_TASK_BLOCKED, &payload, sizeof(payload)); @@ -3757,6 +3760,7 @@ static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) payload.gtid = gtid(p); payload.cpu = cpu_of(rq); + payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; if (task_current(rq, p) && ghost_in_switchto(rq)) payload.from_switchto = true; else @@ -3800,6 +3804,7 @@ static void task_deliver_msg_latched(struct rq *rq, struct task_struct *p, payload.gtid = gtid(p); payload.commit_time = ktime_get_ns(); payload.cpu = cpu_of(rq); + payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; payload.latched_preempt = latched_preempt; produce_for_task(p, MSG_TASK_LATCHED, &payload, sizeof(payload)); @@ -3844,6 +3849,7 @@ static void task_deliver_msg_switchto(struct rq *rq, struct task_struct *p) payload.gtid = gtid(p); payload.runtime = p->se.sum_exec_runtime; payload.cpu = cpu_of(rq); + payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; produce_for_task(p, MSG_TASK_SWITCHTO, &payload, sizeof(payload)); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bede66d530d7..bd882768453c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -117,6 +117,7 @@ struct ghost_rq { bool in_pnt_bpf; /* running BPF at PNT */ int ghost_nr_running; int run_flags; /* flags passed to 'ghost_run()' */ + uint64_t cpu_seqnum; /* history for msgs about this cpu */ /* For deferring work to the balance_callback */ struct list_head enclave_work; /* work to do */ From af6f9eb332c9555caac25c0e855d5da6b1361f8b Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 30 Jul 2021 10:59:36 -0400 Subject: [PATCH 024/165] sched/ghost: add cpu_seqnum to struct ghost_txn cpu_seqnum tells the user which moment in time we were at when we committed the transaction. Agents which use Async transactions (i.e. don't spin until the kernel commits it) treat the completion of the transaction as an implicit TASK_LATCHED message. By adding cpu_seqnum to the transaction, we correctly order this implicit message with the real cpu messages. It is safe to increment the cpu_seqnum independently of the message stream: unlike our current task_barrier, the seqnum does not have a 1:1 relationship with a message. If we wanted to, we could increment that counter at arbitrary points. When a global agent handles out-of-order messages and there were pending transaction, it is difficult to determine what to set cs->current to. The agent only updates cpu state when it gets a message with a newer cpu_seqnum. However, if there is a pending transaction (CpuState cs->next != NULL) it must SyncCpuState at some point. This function checks to see if a transaction is complete, sets cs->current = cs->next, and sets cs->next->state = ON_CPU. The agent wants to sync before handling messages for cs->next; e.g. if we get TaskBlocked for next, we want to set its state to ON_CPU first. (Arguably, we could ignore this). Since we may handle messages out of order, and only the most recent message should mess with CpuState, we need to Sync in any message, even messages for newer tasks on that cpu. Consider this: Messages and cpu_seqnum are in parens. 01: task0 is already running 02: agent decides to preempt task0 with task1 03: task0 blocks (B0) 04: cpu has no latched task, runs BPF 05: BPF latches task2 06: task2 runs (L1 - sent when t2 is on_cpu) 07: agent commits txn to run task1 08: task2 is preempted (P3, on ctx_switch, since it was on_cpu) 09: task1 blocks (B4) 10: BPF latches and runs task3 (L5) There are other similar scenarios, such as if task2 is latch-preempted, if you change 6,7,8 -> 7,6,8 (t2 doesn't run, but we get a TaskLatched). Due to out-of-order messages, the only constraint on messages is that the agent receives L1 before P3, since they both belong to the same task. The agent will use the cpu_seqnum to know which is the most recent version of history. Additionally, when the agent checks messages, it may be at any time after step 7. Let's say all steps completed. Say we receive L1 first. When we handle it, cs->next is set. If messages were in order, we could wait until we get B4 to sync state, since B4 pertains to task1 and is the message that must happen after the commit. And we need to set cs->current at some point: if we don't, we may have a task on_cpu but current is not set. But since messages are not in order, if we don't Sync now (when we get L1), we might not have another opportunity to Sync, because B4 could be older in history. In the case here, B4 is *newer*, so we will be able to muck with cs->current. But when we're at L1, we don't know that yet. We could actually be handling L5 - there's no good way of knowing. All we know is that we're more recent than the previous cs->cpu_seqnum. The root of the issue is that when we SyncCpuState, we are inferring a TASK_LATCHED message: we know a commit succeeded, so we set cs->current. If we asked for a TASK_LATCHED with the commit (which is what BPF does), then we would be able to just set cs->current in the TaskLatched() handler, and *not* do it in SyncCpuState (i.e. have Sync only reap the commit, but not muck with *cpu* state). But since we don't want to get TASK_LATCHED messages all the time, we infer that the latching happened. The solution is to treat the completion of the transaction as a change in the cpu's history: increment and report the cpu_seqnum. That allows userspace to both reap the transaction as well as (optionally) change its CpuState: if the transaction represents a newer version of history. To be submitted in conjunction with cl/387829782. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I0d7988ba7a272e96841974d7886e13e975888886 --- include/uapi/linux/ghost.h | 3 ++- kernel/sched/ghost.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 3a8f449c9184..64f980b3def8 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 39 +#define GHOST_VERSION 40 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -454,6 +454,7 @@ struct ghost_txn { uint8_t unused; int64_t gtid; int64_t commit_time; /* the time that the txn commit succeeded/failed */ + uint64_t cpu_seqnum; /* * Context-dependent fields. */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 97ec086ec1c7..6c58e51a9508 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4986,6 +4986,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, if (*need_rendezvous) smp_store_release(&rq->ghost.rendezvous, rendezvous); + txn->cpu_seqnum = ++rq->ghost.cpu_seqnum; unlock_rq: rq_unlock_irqrestore(rq, &rf); done: From 4fa952641f28db91decc53e6db70d7eee7548385 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 30 Jul 2021 12:35:13 -0400 Subject: [PATCH 025/165] sched/ghost: add was_current to TASK_DEPARTED TASK_DEPARTED has a cpu parameter, but if the task was not actually running, then the agent can't infer anything about the cpu. In particular, it does not know if a switchto chain ended (even if from_switchto is not set) or if there is another ghost task current. In the new world of out-of-order messages, the main thing about from_switchto is for the *task*, not for the cpu. It's the kernel's way of saying "you probably weren't expecting this task to run, since you thought it was blocked, but it woke up, ran, and then blocked/preempted/departed". Since from_switchto is pertinent only for the task, we can't use cpu state to help interpret it. (Also, was_current will always be true for every message other than departed, where it is occasionally false). But from_switchto no longer conveys *useful* information about the cpu itself. It does tell us at that point in cpu_seqnum history that we left switchto. But when we handle a message with from_switchto=true in the agent, we might already have handled a *newer* message (one with from_switchto=false) and we left ST in response to that newer message (it's an "implicitly left ST, discussed below). Alternatively, we might have not received the original TaskSwitchto message yet. Keep in mind this rule: the agent can only use the most recent cpu_seqnum to adjust cs->current or cs->in_switchto. So if we receive a message that is newer than any others, then we can adjust in_switchto. For almost every message, we'll leave the ST chain: basically any message that means "X is or was running on this cpu" (latched, blocked, preempted, departed(was_current=true)". I think of these as implicitly leaving ST. The important part is that whether or not we leave ST is independent of whether the payload's from_switchto was set. We leave ST as soon as we get a message that implied the ST chain ended. Later on, we might receive an older message with from_switchto set: but we already set cs->in_switchto=false. We might also receive an older message TaskSwitchto. But it's old, so we ignore it: we already left. Here's another scenario that shows why we can't use from_switchto to adjust cs->in_switchto: ... ST1(task0), B2(task1,from_st=true), L3(task2), ST4(task2), ... those are the messages in the order they were sent: we enter ST, then block with from_st=true. Then BPF latches a new task, then that task STs. We could receive those messages out of order: (L3 and ST4 are from the same task, so they are in order). possible order of handling: ST1, L3, ST4, B2(from_switchto=true). When we handle B2, we can't touch cs->in_switchto, since 2 < 4. Otherwise, we'll falsely think we're no longer in a ST chain. The ST chain ended with L3. All the agent knew was that we were in an ST chain, then suddenly we latched something. The agent knows it'll eventually get some message with from_switchto set, but due to the above scenario, it can't wait until it gets B2 to adjust in_switchto, since we might be back in a ST chain. Also: I was slightly worried that the was_current check wouldn't handle the case where the departed task was latched, but was_current=false. However, in that case, the kernel will send a preemption before the departed(was_current = false). So it's OK. Also also: Neel pointed out that the agent will be able to check task->on_cpu() to determine if it was current or not. This flag makes it a little easier on the agent, but is essentially a double-check. Since the task's messages are delivered in order, the agent would know a task latched (either via MSG_TASK_LATCHED or by completing a TXN) before it handled MSG_TASK_DEPARTED. To be submitted in conjunction with cl/387829782. Tested: agent + switchto_test and simple_exp Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I398a6f2e4b315d78864d8a85e74f7c57f780f415 --- include/uapi/linux/ghost.h | 3 ++- kernel/sched/ghost.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 64f980b3def8..c77fe203d34c 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 40 +#define GHOST_VERSION 41 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -224,6 +224,7 @@ struct ghost_msg_payload_task_departed { uint64_t cpu_seqnum; int cpu; char from_switchto; + char was_current; }; struct ghost_msg_payload_task_affinity_changed { diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 6c58e51a9508..f9c1496a8b57 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3765,6 +3765,7 @@ static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) payload.from_switchto = true; else payload.from_switchto = false; + payload.was_current = task_current(rq, p); produce_for_task(p, MSG_TASK_DEPARTED, &payload, sizeof(payload)); From 5afb4981320dd749ee69e55aee9d13f30ccbf8bb Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Sat, 23 Oct 2021 13:55:42 -0700 Subject: [PATCH 026/165] sched/ghost: Add ghOSt paper and talk links to the README Change-Id: I67d6f61b521238f774c710d1232b5978b12d7eb3 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2d69d3e33fe4..7088c7e4151e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ performant framework for delegation of thread scheduling policy to userspace processes that enables policy optimization, non-disruptive upgrades, and fault isolation. +[SOSP '21 Paper](https://dl.acm.org/doi/10.1145/3477132.3483542)\ +[SOSP '21 Talk](https://youtu.be/j4ABe4dsbIY) + You must compile and install this kernel in order to use the [ghOSt userspace component](https://www.github.com/google/ghost-userspace). From bb7bcaaba839ad1cfd17d812fcb0043c911b5fc3 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 3 Aug 2021 14:08:08 -0700 Subject: [PATCH 027/165] sched: actually treat 'txn->commit_flags' as flags. Prior to this change 'txn->commit_flags' was used to communicate singular, mutually exclusive values: - greedy commit (commit_flags = 0) - inline commit (commit_flags = COMMIT_AT_TXN_COMMIT) - PNT commit (commit_flags = COMMIT_AT_SCHEDULE) This made it impossible to use 'commit_flags' to communicate any other information to the kernel. For example ALLOW_TASK_ONCPU is communicated via 'run_flags' even though 'commit_flags' would be a better choice (ALLOW_TASK_ONCPU is only relevant at commit time as opposed to when the task is running). Fix this by interpreting 'commit_flags' to contain individual flag values as opposed to a singular enumeration value. Note that there is no change in ABI. Tested: all unit tests pass in virtme Effort: sched/ghost Change-Id: I0fa869fce9ca30870c29f6637e92fd9edcf28fae --- include/uapi/linux/ghost.h | 14 +++++--------- kernel/sched/ghost.c | 37 ++++++++++++++++++++++++------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index c77fe203d34c..4bc2e80e3fe4 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -377,15 +377,11 @@ enum ghost_base_ops { #define SEND_TASK_LATCHED (1 << 10) /* Send TASK_LATCHED at commit time */ /* txn->commit_flags */ -enum txn_commit_at { - /* - * commit_flags = 0 indicates a greedy commit (i.e. agent doesn't - * care where the commit happens). The kernel tries to apply the - * commit at the earliest opportunity (e.g. return-to-user). - */ - COMMIT_AT_SCHEDULE = 1, /* commit when oncpu task schedules */ - COMMIT_AT_TXN_COMMIT, /* commit in GHOST_COMMIT_TXN op */ -}; +#define COMMIT_AT_SCHEDULE (1 << 0) /* commit when oncpu task schedules */ +#define COMMIT_AT_TXN_COMMIT (1 << 1) /* commit in GHOST_COMMIT_TXN op */ + +/* Union of all COMMIT_AT_XYZ flags */ +#define COMMIT_AT_FLAGS (COMMIT_AT_SCHEDULE | COMMIT_AT_TXN_COMMIT) /* special 'gtid' encodings that can be passed to ghost_run() */ #define GHOST_NULL_GTID (0) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index f9c1496a8b57..fddfccdfbc95 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -76,10 +76,10 @@ static bool cpu_deliver_msg_tick(struct rq *rq); static int task_target_cpu(struct task_struct *p); static int agent_target_cpu(struct rq *rq); static inline bool ghost_txn_ready(int cpu); -static bool _ghost_commit_pending_txn(int cpu, enum txn_commit_at where); +static bool _ghost_commit_pending_txn(int cpu, int where); static inline void ghost_claim_and_kill_txn(int cpu, enum ghost_txn_state err); static void ghost_commit_all_greedy_txns(void); -static void ghost_commit_pending_txn(enum txn_commit_at where); +static void ghost_commit_pending_txn(int where); static inline int queue_decref(struct ghost_queue *q); static void release_from_ghost(struct rq *rq, struct task_struct *p); static void ghost_wake_agent_on(int cpu); @@ -4607,11 +4607,16 @@ static inline bool ghost_txn_ready(int cpu) return _ghost_txn_ready(cpu, NULL); } +static inline bool _ghost_txn_greedy(int commit_flags) +{ + return (commit_flags & COMMIT_AT_FLAGS) == 0; +} + static inline bool ghost_txn_greedy(int cpu) { - int commit_flags; + int flags; - return _ghost_txn_ready(cpu, &commit_flags) && commit_flags == 0; + return _ghost_txn_ready(cpu, &flags) && _ghost_txn_greedy(flags); } /* @@ -4624,22 +4629,28 @@ static inline bool ghost_txn_greedy(int cpu) * is ignored (indicated by where = -1). * * Returns 'true' if txn was claimed and 'false' otherwise. - * - * N.B. 'where' is deliberately an 'int' as opposed to 'enum txn_commit_at' - * otherwise the compiler optimizes away the 'where >= 0' conditional below - * since it "knows" all possible enum values. */ static inline bool ghost_claim_txn(int cpu, int where) { struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, cpu)); + int commit_flags; VM_BUG_ON(preemptible()); VM_BUG_ON(cpu < 0 || cpu >= nr_cpu_ids); + /* + * Ensure that COMMIT_AT_XYZ flag is never a negative value so + * that a commit explicitly requested by the agent can be safely + * indicated by where == -1. + */ + BUILD_BUG_ON(COMMIT_AT_FLAGS < 0); + if (!txn || smp_load_acquire(&txn->state) != GHOST_TXN_READY) return false; - if (where >= 0 && txn->commit_flags != 0 && txn->commit_flags != where) + commit_flags = READ_ONCE(txn->commit_flags); + commit_flags &= COMMIT_AT_FLAGS; + if (where >= 0 && commit_flags != 0 && commit_flags != where) return false; /* @@ -5103,14 +5114,14 @@ static inline void ghost_send_reschedule(struct cpumask *mask) } #endif -static bool _ghost_commit_pending_txn(int cpu, enum txn_commit_at where) +static bool _ghost_commit_pending_txn(int cpu, int where) { if (unlikely(ghost_claim_txn(cpu, where))) return ghost_commit_txn(cpu, false, NULL); return false; } -static void ghost_commit_pending_txn(enum txn_commit_at where) +static void ghost_commit_pending_txn(int where) { int cpu = get_cpu(); @@ -5440,8 +5451,8 @@ static int gsys_ghost_commit_txn(ulong __user *user_mask_ptr, if (!_ghost_txn_ready(cpu, &commit_flags)) continue; - greedy_commit = (commit_flags == 0); - inline_commit = (commit_flags == COMMIT_AT_TXN_COMMIT); + greedy_commit = _ghost_txn_greedy(commit_flags); + inline_commit = (commit_flags & COMMIT_AT_TXN_COMMIT); if (cpu != this_cpu && !inline_commit) { /* From a9f0b7d0198f01f8ca757f30f0dcc017747b192f Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 4 Aug 2021 10:28:40 -0700 Subject: [PATCH 028/165] sched: move ALLOW_TASK_ONCPU to commit_flags The effect of ALLOW_TASK_ONCPU is limited to when a transaction is committed so it goes into txn->commit_flags. Submitted in conjunction with cl/388747781 Tested: verified all unit tests in virtme. Effort: sched/ghost Change-Id: I5a8fb2e9c964d61527252f355b010730f419aaf4 --- include/uapi/linux/ghost.h | 8 +++--- kernel/sched/ghost.c | 52 +++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 4bc2e80e3fe4..1db6be658bcd 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 41 +#define GHOST_VERSION 42 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -368,9 +368,6 @@ enum ghost_base_ops { #define NEED_CPU_NOT_IDLE (1 << 7) /* Notify agent when a non-idle task is * scheduled on the cpu. */ -#define ALLOW_TASK_ONCPU (1 << 8) /* If task is already running on remote - * cpu then let it keep running there. - */ #define ELIDE_PREEMPT (1 << 9) /* Do not send TASK_PREEMPT if we preempt * a previous ghost task on this cpu */ @@ -379,6 +376,9 @@ enum ghost_base_ops { /* txn->commit_flags */ #define COMMIT_AT_SCHEDULE (1 << 0) /* commit when oncpu task schedules */ #define COMMIT_AT_TXN_COMMIT (1 << 1) /* commit in GHOST_COMMIT_TXN op */ +#define ALLOW_TASK_ONCPU (1 << 2) /* If task is running on a remote cpu + * then let continue running there. + */ /* Union of all COMMIT_AT_XYZ flags */ #define COMMIT_AT_FLAGS (COMMIT_AT_SCHEDULE | COMMIT_AT_TXN_COMMIT) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index fddfccdfbc95..cabb9c2096c8 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4338,6 +4338,19 @@ static inline bool run_flags_valid(int run_flags, int valid_run_flags, return true; } +static inline bool commit_flags_valid(int commit_flags, int valid_commit_flags) +{ + if (commit_flags & ~valid_commit_flags) + return false; + + /* Exactly one or none of the COMMIT_AT_XYZ flags must be set */ + BUILD_BUG_ON(sizeof_field(struct ghost_txn, commit_flags) != 1); + if (hweight8(commit_flags & COMMIT_AT_FLAGS) > 1) + return false; + + return true; +} + /* * ghOSt API to yield local cpu or ping a remote cpu. * @@ -4485,7 +4498,6 @@ static int __ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, RTLA_ON_BLOCKED | RTLA_ON_YIELD | NEED_L1D_FLUSH | - ALLOW_TASK_ONCPU | ELIDE_PREEMPT | SEND_TASK_LATCHED | 0; @@ -4725,19 +4737,23 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, struct rq_flags rf; struct task_struct *next; bool local_run, resched = false; - int run_flags, state = GHOST_TXN_COMPLETE; + int commit_flags, run_flags, state = GHOST_TXN_COMPLETE; struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, run_cpu)); - const int supported_flags = RTLA_ON_PREEMPT | - RTLA_ON_BLOCKED | - RTLA_ON_YIELD | - RTLA_ON_IDLE | - NEED_L1D_FLUSH | - NEED_CPU_NOT_IDLE | - ALLOW_TASK_ONCPU | - ELIDE_PREEMPT | - SEND_TASK_LATCHED | - 0; + const int supported_run_flags = RTLA_ON_PREEMPT | + RTLA_ON_BLOCKED | + RTLA_ON_YIELD | + RTLA_ON_IDLE | + NEED_L1D_FLUSH | + ELIDE_PREEMPT | + NEED_CPU_NOT_IDLE | + SEND_TASK_LATCHED | + 0; + + const int supported_commit_flags = COMMIT_AT_SCHEDULE | + COMMIT_AT_TXN_COMMIT | + ALLOW_TASK_ONCPU | + 0; VM_BUG_ON(preemptible()); VM_BUG_ON(commit_state == NULL); @@ -4763,7 +4779,13 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, gtid = READ_ONCE(txn->gtid); run_flags = READ_ONCE(txn->run_flags); - if (!run_flags_valid(run_flags, supported_flags, gtid)) { + if (!run_flags_valid(run_flags, supported_run_flags, gtid)) { + state = GHOST_TXN_INVALID_FLAGS; + goto done; + } + + commit_flags = READ_ONCE(txn->commit_flags); + if (!commit_flags_valid(commit_flags, supported_commit_flags)) { state = GHOST_TXN_INVALID_FLAGS; goto done; } @@ -4791,7 +4813,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, goto done; } - if (!(run_flags & ALLOW_TASK_ONCPU) || + if (!(commit_flags & ALLOW_TASK_ONCPU) || (task_cpu(next) != run_cpu)) { if (validate_next_offcpu(rq, next, &state)) { task_rq_unlock(rq, next, &rf); @@ -4831,7 +4853,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, if (gtid == GHOST_IDLE_GTID) next = rq->idle; - if (!(run_flags & ALLOW_TASK_ONCPU)) { + if (!(commit_flags & ALLOW_TASK_ONCPU)) { if (validate_next_offcpu(rq, next, &state)) goto unlock_rq; } From c220ff5ec805445432e585f45ce7602182c62b05 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 5 Aug 2021 10:16:21 -0700 Subject: [PATCH 029/165] sched: don't advertise a departing task as runnable. Advertising a departing task as runnable invites a race where the agent can try to schedule the task before it handles the TASK_DEPARTED msg. In this case the commit fails with a GHOST_TXN_INVALID_TARGET error which is a fatal error in the agent (b/195081642 has more details). Forcing 'runnable=false' in this situation leads to a couple of discrepancies: - 'task_new->runnable' is not coherent with the GHOST_SW_TASK_RUNNABLE flag in the task's status_word (but ultimately fine since this is indistinguishable from a blocked task that woke up while the agent was handling the TASK_NEW). - 'task_new->runnable' is not coherent with 'departed->was_current' but this should be okay since 'was_runnable' deals with task state whereas 'was_current' deals with cpu state. Tested: - 10000 iterations of api_test --gunit_filter=ApiTest.DepartedRace - kokonut test //sched:ghost_smoketest http://sponge2/a5f65f0e-68e4-4152-94c8-e1d6628bc42c Effort: sched/ghost Google-Bug-Id: 195081642 Change-Id: I89ebea15961bd70c2899f2c324d3e15be6fc49d3 --- kernel/sched/ghost.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index cabb9c2096c8..8a2d20b66678 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -63,6 +63,8 @@ int sysctl_ghost_switchto_disable; int __read_mostly sysctl_ghost_commit_at_tick; static void ghost_task_new(struct rq *rq, struct task_struct *p); +static void _ghost_task_new(struct rq *rq, struct task_struct *p, + bool runnable); static void ghost_task_yield(struct rq *rq, struct task_struct *p); static void ghost_task_blocked(struct rq *rq, struct task_struct *p); static void task_dead_ghost(struct task_struct *p); @@ -567,7 +569,14 @@ static void switched_from_ghost(struct rq *rq, struct task_struct *p) if (unlikely(p->ghost.new_task)) { WARN_ON_ONCE(!task_current(rq, p)); p->ghost.new_task = false; - ghost_task_new(rq, p); + /* + * Task is departing from ghost so don't advertise it as + * runnable otherwise the agent could try to schedule it + * before it sees TASK_DEPARTED (in this case the commit + * fails with GHOST_TXN_INVALID_TARGET which is treated as + * a fatal error by the agent). + */ + _ghost_task_new(rq, p, /*runnable=*/false); ghost_wake_agent_of(p); } @@ -3656,7 +3665,8 @@ static inline int __task_deliver_common(struct rq *rq, struct task_struct *p) return 0; } -static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p) +static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p, + bool runnable) { struct ghost_msg_payload_task_new payload; @@ -3664,7 +3674,7 @@ static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p) return; payload.gtid = gtid(p); - payload.runnable = task_on_rq_queued(p); + payload.runnable = runnable; payload.runtime = p->se.sum_exec_runtime; if (_get_sw_info(p->ghost.enclave, p->ghost.status_word, &payload.sw_info)) { @@ -4195,7 +4205,7 @@ void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p) } } -static void ghost_task_new(struct rq *rq, struct task_struct *p) +static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable) { lockdep_assert_held(&rq->lock); VM_BUG_ON(task_rq(p) != rq); @@ -4206,7 +4216,12 @@ static void ghost_task_new(struct rq *rq, struct task_struct *p) if (p == rq->curr) update_curr_ghost(rq); - task_deliver_msg_task_new(rq, p); + task_deliver_msg_task_new(rq, p, runnable); +} + +static void ghost_task_new(struct rq *rq, struct task_struct *p) +{ + _ghost_task_new(rq, p, task_on_rq_queued(p)); } static void ghost_task_yield(struct rq *rq, struct task_struct *p) From b91de8ce3502aba75c4fe12f9cb1c57d633d05d9 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Thu, 29 Jul 2021 23:02:32 -0700 Subject: [PATCH 030/165] sched: convert ghost_switchto_disable sysctl to an enclave tunable Replace the ghost_switchto_disable sysctl with a "switchto_disabled" enclave tunable. This tunable has the same meaning as the sysctl and is accessible via ghostfs (similar to runnable_timeout). The associated test is modified in cl/387863829. The presubmit failure will go away once gtests is updated to that CL. Tested: //prodkernel/tests/switchto_tunable_test (cl/387863829) in virtme Effort: sched/ghost Google-Bug-Id: 195752832 Change-Id: I71c672d0bd07d0e06694b623610d2d05b8b02292 --- include/linux/sched/sysctl.h | 1 - kernel/sched/ghost.c | 1 - kernel/sched/ghostfs.c | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ kernel/sysctl.c | 7 ------- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 05bd0481985f..261799db0e3c 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -34,7 +34,6 @@ extern unsigned int sysctl_sched_child_runs_first; #ifdef CONFIG_SCHED_CLASS_GHOST extern unsigned long sysctl_ghost_cfs_load_added; extern int sysctl_ghost_wake_on_waker_cpu; -extern int sysctl_ghost_switchto_disable; extern int sysctl_ghost_commit_at_tick; #endif diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8a2d20b66678..48b488a16d78 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -59,7 +59,6 @@ static DEFINE_PER_CPU(int64_t, sync_group_cookie); /* The load contribution that CFS sees for a running ghOSt task */ unsigned long sysctl_ghost_cfs_load_added = 1024; int __read_mostly sysctl_ghost_wake_on_waker_cpu; -int sysctl_ghost_switchto_disable; int __read_mostly sysctl_ghost_commit_at_tick; static void ghost_task_new(struct rq *rq, struct task_struct *p); diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 5b89b33c65fd..43925bb47264 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -578,6 +578,37 @@ static struct kernfs_ops gf_ops_e_runnable_timeout = { .write = gf_runnable_timeout_write, }; +static int gf_switchto_disabled_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->switchto_disabled)); + return 0; +} + +static ssize_t gf_switchto_disabled_write(struct kernfs_open_file *of, + char *buf, size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->switchto_disabled, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_switchto_disabled = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_switchto_disabled_show, + .write = gf_switchto_disabled_write, +}; + static int gf_status_show(struct seq_file *sf, void *v) { struct ghost_enclave *e = seq_to_e(sf); @@ -646,6 +677,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_runnable_timeout, }, + { + .name = "switchto_disabled", + .mode = 0664, + .ops = &gf_ops_e_switchto_disabled, + }, { .name = "status", .mode = 0444, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bd882768453c..ee95d5118639 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -197,6 +197,8 @@ struct ghost_enclave { ktime_t max_unscheduled; struct work_struct enclave_destroyer; + bool switchto_disabled; + unsigned long id; int is_dying; bool agent_online; /* userspace says agent can schedule. */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 745b75099967..2cdb9354beef 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3156,13 +3156,6 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, - { - .procname = "ghost_switchto_disable", - .data = &sysctl_ghost_switchto_disable, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, { .procname = "ghost_commit_at_tick", .data = &sysctl_ghost_commit_at_tick, From 1711dde77499774fb8f0afcf037859214c33bdf8 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Fri, 6 Aug 2021 10:51:26 -0700 Subject: [PATCH 031/165] sched: ALLOW_TASK_ONCPU is now in commit_flags Fixes: d7837b65a90a ("sched: move ALLOW_TASK_ONCPU to commit_flags") Tested: api_test on a debug kernel build. Effort: sched/ghost Change-Id: I9b7349711db2f281f41df109b92f896f17de57de --- kernel/sched/ghost.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 48b488a16d78..fd0c4f8f95d3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4940,7 +4940,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, if (next && task_running(rq, next)) { /* 'next' is already oncpu */ - VM_BUG_ON(!(run_flags & ALLOW_TASK_ONCPU)); + VM_BUG_ON(!(commit_flags & ALLOW_TASK_ONCPU)); resched = false; /* From 37e27b12d859727688212c24007b704189b67e75 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Fri, 6 Aug 2021 09:25:03 -0700 Subject: [PATCH 032/165] sched: free queue memory when last reference is dropped. Prior to this change queue memory was leaked if there were references on the queue when it was released by the filesystem. This was a reasonable tradeoff during early development but it is not acceptable in a world that supports agent upgrade and restart (we will leak queue memory on agent death). Fix this by freeing queue memory when the last reference is dropped. This is a bit harder than it sounds because we can drop references in awkward contexts (e.g. while holding rq lock or when preemption is disabled). To circumvent this we defer processing via an rcu_callback. The rcu callback doesn't hold scheduler locks but is still an atomic context (it may be invoked in softirq context) so we must further defer to a normal process context via schedule_work. Tested: - all unit tests pass in virtme with debug build. - whitebox tested to verify number of queues freed equals the number of queues allocated. Effort: sched/ghost Google-Bug-Id: 192491600 Change-Id: I46d76f761cf2db4ef69dc773d521f0b0cd859eb1 --- kernel/sched/ghost.c | 84 ++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index fd0c4f8f95d3..cf7a83bc95bf 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -81,7 +81,6 @@ static bool _ghost_commit_pending_txn(int cpu, int where); static inline void ghost_claim_and_kill_txn(int cpu, enum ghost_txn_state err); static void ghost_commit_all_greedy_txns(void); static void ghost_commit_pending_txn(int where); -static inline int queue_decref(struct ghost_queue *q); static void release_from_ghost(struct rq *rq, struct task_struct *p); static void ghost_wake_agent_on(int cpu); static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, @@ -1658,22 +1657,61 @@ struct ghost_queue { ulong mapsize; /* size of the vmalloc'ed region */ struct queue_notifier *notifier; /* rcu-protected agent wakeup info */ -#ifdef CONFIG_DEBUG_GHOST - int fd; /* Stash away fd for debug. */ -#endif + + struct rcu_head rcu; /* deferred free glue */ + struct work_struct free_work; }; +/* + * Free the memory resources associated with the ghost_queue (must be called + * in sleepable process context). + */ +static void __queue_free_work(struct work_struct *work) +{ + struct ghost_queue *q = container_of(work, struct ghost_queue, + free_work); + vfree(q->addr); + kfree(q->notifier); + kfree(q); +} + +void _queue_free_rcu_callback(struct rcu_head *rhp) +{ + struct ghost_queue *q = container_of(rhp, struct ghost_queue, rcu); + + /* + * Further defer work to a preemptible process context: the rcu + * callback may be called from a softirq context and cannot block. + */ + schedule_work(&q->free_work); +} + static inline int queue_decref(struct ghost_queue *q) { ulong flags; int error = 0; + bool canfree = false; spin_lock_irqsave(&q->lock, flags); - if (WARN_ON_ONCE(!q->refs)) + if (WARN_ON_ONCE(!q->refs)) { error = -EINVAL; - else - q->refs--; + } else { + if (--q->refs == 0) + canfree = true; + } spin_unlock_irqrestore(&q->lock, flags); + + if (canfree) { + /* + * We may be called from awkward contexts that hold scheduler + * locks or that are non-preemptible and this runs afoul of + * sleepable locks taken during vfree(q->addr). + * + * Defer freeing of queue memory to an rcu callback (this has + * nothing to do with rcu and we use it solely for convenience). + */ + call_rcu(&q->rcu, _queue_free_rcu_callback); + } return error; } @@ -2475,7 +2513,7 @@ struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, } static int ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, - bool forked) + bool forked) { int error; unsigned long irq_fl; @@ -2791,32 +2829,11 @@ static int queue_release(struct inode *inode, struct file *file) { struct ghost_queue *q = file->private_data; struct ghost_enclave *e = q->enclave; - ulong flags, refs; enclave_maybe_del_default_queue(e, q); q->enclave = NULL; kref_put(&e->kref, enclave_release); - - spin_lock_irqsave(&q->lock, flags); - refs = q->refs; - spin_unlock_irqrestore(&q->lock, flags); - - if (refs) { - /* - * XXX we could flag that this queue is not reachable - * from userspace anymore and can be freed when the - * last reference is dropped (e.g. when all referencing - * tasks die or change association). - * - * Just leak memory for now. - */ - WARN_ONCE(1, "%s: leaking queue with %lu refs", __func__, refs); - } else { - vfree(q->addr); - kfree(q->notifier); - kfree(q); - /* XXX memcg uncharge */ - } + queue_decref(q); /* drop inode reference */ return 0; } @@ -2905,13 +2922,12 @@ static int ghost_create_queue(int elems, int node, int flags, error = fd; goto err_getfd; } -#ifdef CONFIG_DEBUG_GHOST - q->fd = fd; -#endif + kref_get(&e->kref); q->enclave = e; + queue_incref(q); /* inode gets its own reference */ + INIT_WORK(&q->free_work, __queue_free_work); - /* XXX memcg charge */ ghost_fdput_enclave(e, &f_enc); return fd; From 4bc38bc7d20de206c54646418480d88ee00ddb1c Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Mon, 9 Aug 2021 20:15:40 -0400 Subject: [PATCH 033/165] sched/ghost: use krefs with struct ghost_queue We're using the classic "last reference out, free the object" pattern, which is a perfect match for a kref. The one time we were using the return value from decref was just to panick the kernel (for debug builds, at least). Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I87b97cb631307ba68ae60d787ddbcaf66aa01f0d --- kernel/sched/ghost.c | 52 ++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index cf7a83bc95bf..32cb3f335ce6 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1639,7 +1639,7 @@ struct ghost_queue { * See go/ghost-queue-change for more details. */ spinlock_t lock; - ulong refs; + struct kref kref; struct ghost_enclave *enclave; @@ -1686,42 +1686,29 @@ void _queue_free_rcu_callback(struct rcu_head *rhp) schedule_work(&q->free_work); } -static inline int queue_decref(struct ghost_queue *q) +static void __queue_kref_release(struct kref *k) { - ulong flags; - int error = 0; - bool canfree = false; + struct ghost_queue *q = container_of(k, struct ghost_queue, kref); - spin_lock_irqsave(&q->lock, flags); - if (WARN_ON_ONCE(!q->refs)) { - error = -EINVAL; - } else { - if (--q->refs == 0) - canfree = true; - } - spin_unlock_irqrestore(&q->lock, flags); + /* + * We may be called from awkward contexts that hold scheduler + * locks or that are non-preemptible and this runs afoul of + * sleepable locks taken during vfree(q->addr). + * + * Defer freeing of queue memory to an rcu callback (this has + * nothing to do with rcu and we use it solely for convenience). + */ + call_rcu(&q->rcu, _queue_free_rcu_callback); +} - if (canfree) { - /* - * We may be called from awkward contexts that hold scheduler - * locks or that are non-preemptible and this runs afoul of - * sleepable locks taken during vfree(q->addr). - * - * Defer freeing of queue memory to an rcu callback (this has - * nothing to do with rcu and we use it solely for convenience). - */ - call_rcu(&q->rcu, _queue_free_rcu_callback); - } - return error; +static inline void queue_decref(struct ghost_queue *q) +{ + kref_put(&q->kref, __queue_kref_release); } static inline void queue_incref(struct ghost_queue *q) { - ulong flags; - - spin_lock_irqsave(&q->lock, flags); - q->refs++; - spin_unlock_irqrestore(&q->lock, flags); + kref_get(&q->kref); } /* Hold e->lock and the cpu_rsvp lock */ @@ -2643,8 +2630,7 @@ void ghost_sched_cleanup_fork(struct task_struct *p) q = p->ghost.dst_q; if (q != NULL) { p->ghost.dst_q = NULL; - error = queue_decref(q); - VM_BUG_ON(error); + queue_decref(q); } sw = p->ghost.status_word; @@ -2895,6 +2881,7 @@ static int ghost_create_queue(int elems, int node, int flags, } spin_lock_init(&q->lock); + kref_init(&q->kref); /* sets to 1; inode gets its own reference */ q->addr = vmalloc_user_node_flags(size, node, GFP_KERNEL); if (!q->addr) { error = -ENOMEM; @@ -2925,7 +2912,6 @@ static int ghost_create_queue(int elems, int node, int flags, kref_get(&e->kref); q->enclave = e; - queue_incref(q); /* inode gets its own reference */ INIT_WORK(&q->free_work, __queue_free_work); ghost_fdput_enclave(e, &f_enc); From 45a2884912ec80edb7af13de1ceab1f09c1d29a0 Mon Sep 17 00:00:00 2001 From: Josh Don Date: Tue, 3 Aug 2021 13:54:09 -0700 Subject: [PATCH 034/165] sched: refactor ghost_commit_txn exit path Consolidate the exit labels into a single "out" label that can handle unlocking the rq. This will be useful for the next patch in the series. Tested: builds Effort: sched/ghost Change-Id: I2e679065464a4bbf3ca3c71b8e496e82524bea7c --- kernel/sched/ghost.c | 45 ++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 32cb3f335ce6..9ccce919991b 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4749,7 +4749,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, int *commit_state, bool *need_rendezvous) { gtid_t gtid; - struct rq *rq; + struct rq *rq = NULL; struct rq_flags rf; struct task_struct *next; bool local_run, resched = false; @@ -4780,35 +4780,35 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, if (!txn || txn->state != raw_smp_processor_id()) { state = GHOST_TXN_INVALID_CPU; - goto done; + goto out; } if (txn->version != GHOST_TXN_VERSION) { state = GHOST_TXN_UNSUPPORTED_VERSION; - goto done; + goto out; } if (txn->cpu != run_cpu) { state = GHOST_TXN_INVALID_CPU; - goto done; + goto out; } gtid = READ_ONCE(txn->gtid); run_flags = READ_ONCE(txn->run_flags); if (!run_flags_valid(run_flags, supported_run_flags, gtid)) { state = GHOST_TXN_INVALID_FLAGS; - goto done; + goto out; } commit_flags = READ_ONCE(txn->commit_flags); if (!commit_flags_valid(commit_flags, supported_commit_flags)) { state = GHOST_TXN_INVALID_FLAGS; - goto done; + goto out; } if (!cpu_online(run_cpu)) { state = GHOST_TXN_CPU_OFFLINE; - goto done; + goto out; } if (likely(gtid > 0)) { @@ -4818,7 +4818,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, rcu_read_unlock(); state = next ? GHOST_TXN_INVALID_TARGET : GHOST_TXN_TARGET_NOT_FOUND; - goto done; + goto out; } rq = task_rq_lock(next, &rf); @@ -4826,14 +4826,16 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, if (validate_next_task(rq, next, txn->task_barrier, &state)) { task_rq_unlock(rq, next, &rf); - goto done; + rq = NULL; + goto out; } if (!(commit_flags & ALLOW_TASK_ONCPU) || (task_cpu(next) != run_cpu)) { if (validate_next_offcpu(rq, next, &state)) { task_rq_unlock(rq, next, &rf); - goto done; + rq = NULL; + goto out; } } else if (task_running(rq, next) && rq->ghost.must_resched && @@ -4851,7 +4853,8 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, */ state = GHOST_TXN_TARGET_STALE; task_rq_unlock(rq, next, &rf); - goto done; + rq = NULL; + goto out; } rq = ghost_move_task(rq, next, run_cpu, &rf); @@ -4860,7 +4863,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, } else { if (gtid < GHOST_IDLE_GTID) { state = GHOST_TXN_INVALID_TARGET; - goto done; + goto out; } next = NULL; rq = cpu_rq(run_cpu); @@ -4871,7 +4874,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, if (!(commit_flags & ALLOW_TASK_ONCPU)) { if (validate_next_offcpu(rq, next, &state)) - goto unlock_rq; + goto out; } } @@ -4881,12 +4884,12 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, */ if (unlikely(!rq->ghost.agent)) { state = GHOST_TXN_NO_AGENT; - goto unlock_rq; + goto out; } if (unlikely(!txn_commit_allowed(rq, gtid, sync))) { state = GHOST_TXN_NOT_PERMITTED; - goto unlock_rq; + goto out; } if (next && !ghost_can_schedule(rq, gtid)) { @@ -4899,7 +4902,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, * anywhere else. */ state = GHOST_TXN_CPU_UNAVAIL; - goto unlock_rq; + goto out; } local_run = blocking_run(rq, sync, gtid); @@ -4917,7 +4920,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, READ_ONCE(txn->agent_barrier)) { rq->ghost.blocked_in_run = false; state = GHOST_TXN_AGENT_STALE; - goto unlock_rq; + goto out; } } else { /* @@ -5037,9 +5040,11 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, smp_store_release(&rq->ghost.rendezvous, rendezvous); txn->cpu_seqnum = ++rq->ghost.cpu_seqnum; -unlock_rq: - rq_unlock_irqrestore(rq, &rf); -done: +out: + if (rq) { + lockdep_assert_held(&rq->lock); + rq_unlock_irqrestore(rq, &rf); + } *commit_state = state; return resched; } From 7f5671e146fa0acf581bb588f6a90333aa84b4e4 Mon Sep 17 00:00:00 2001 From: Josh Don Date: Thu, 29 Jul 2021 09:56:45 -0700 Subject: [PATCH 035/165] sched: add more control over agent barrier increment Adds two new flags, in order to offer more control over the increment of the agent barrier as a part of transaction commit. ghost_run():ELIDE_AGENT_BARRIER_INC Skips the agent_barrier_inc that normally occurs when we latch a task as part of transaction commit. ghost_run():INC_AGENT_BARRIER_ON_FAILURE Increment the agent barrier if transaction commit fails. Submitted along with kcl/402931 and cl/387903701. Tested: See cl/387903701 Effort: sched/ghost Change-Id: Ifc93b50c52571d0d2f35784ba19ee7cff21c33af --- include/uapi/linux/ghost.h | 9 +++++- kernel/sched/ghost.c | 59 +++++++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 1db6be658bcd..bc224f6b25f9 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 42 +#define GHOST_VERSION 43 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -379,6 +379,13 @@ enum ghost_base_ops { #define ALLOW_TASK_ONCPU (1 << 2) /* If task is running on a remote cpu * then let continue running there. */ +#define ELIDE_AGENT_BARRIER_INC (1 << 3) /* Do not increment the agent + * barrier (ie. on successfully + * latching the task). + */ +#define INC_AGENT_BARRIER_ON_FAILURE (1 << 4) /* Increment agent_barrier + * on transaction failure. + */ /* Union of all COMMIT_AT_XYZ flags */ #define COMMIT_AT_FLAGS (COMMIT_AT_SCHEDULE | COMMIT_AT_TXN_COMMIT) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 9ccce919991b..f67772fa9b17 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4364,6 +4364,11 @@ static inline bool commit_flags_valid(int commit_flags, int valid_commit_flags) if (hweight8(commit_flags & COMMIT_AT_FLAGS) > 1) return false; + /* Cannot specify both elide barrier inc and inc on failure. */ + if ((commit_flags & ELIDE_AGENT_BARRIER_INC) && + (commit_flags & INC_AGENT_BARRIER_ON_FAILURE)) + return false; + return true; } @@ -4741,6 +4746,11 @@ static inline bool blocking_run(struct rq *rq, bool sync, gtid_t gtid) return _local_commit(rq, sync) && gtid != GHOST_AGENT_GTID; } +static inline bool ghost_txn_succeeded(int state) +{ + return state == GHOST_TXN_COMPLETE; +} + /* * Caller is responsible for claiming txn (before calling this function) * and finalizing it (after this function returns). @@ -4753,7 +4763,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, struct rq_flags rf; struct task_struct *next; bool local_run, resched = false; - int commit_flags, run_flags, state = GHOST_TXN_COMPLETE; + int commit_flags = 0, run_flags, state = GHOST_TXN_COMPLETE; struct ghost_txn *txn = rcu_dereference(per_cpu(ghost_txn, run_cpu)); const int supported_run_flags = RTLA_ON_PREEMPT | @@ -4766,9 +4776,11 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, SEND_TASK_LATCHED | 0; - const int supported_commit_flags = COMMIT_AT_SCHEDULE | - COMMIT_AT_TXN_COMMIT | - ALLOW_TASK_ONCPU | + const int supported_commit_flags = COMMIT_AT_SCHEDULE | + COMMIT_AT_TXN_COMMIT | + ALLOW_TASK_ONCPU | + ELIDE_AGENT_BARRIER_INC | + INC_AGENT_BARRIER_ON_FAILURE | 0; VM_BUG_ON(preemptible()); @@ -4987,7 +4999,8 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, */ /* "serialize" with remote-agent doing a local run */ - agent_barrier_inc(rq); + if (!(commit_flags & ELIDE_AGENT_BARRIER_INC)) + agent_barrier_inc(rq); /* * Update latched task unless this is a ping in which case @@ -5043,8 +5056,24 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, out: if (rq) { lockdep_assert_held(&rq->lock); - rq_unlock_irqrestore(rq, &rf); + if (WARN_ON_ONCE(rq != cpu_rq(run_cpu))) { + rq_unlock_irqrestore(rq, &rf); + rq = NULL; + } + } + + if ((commit_flags & INC_AGENT_BARRIER_ON_FAILURE) && + !ghost_txn_succeeded(state)) { + if (!rq) { + rq = cpu_rq(run_cpu); + rq_lock_irqsave(rq, &rf); + } + agent_barrier_inc(rq); } + + if (rq) + rq_unlock_irqrestore(rq, &rf); + *commit_state = state; return resched; } @@ -5104,11 +5133,6 @@ static inline void ghost_claim_and_kill_txn(int cpu, enum ghost_txn_state err) preempt_enable_no_resched(); } -static inline bool ghost_txn_succeeded(int state) -{ - return state == GHOST_TXN_COMPLETE; -} - static bool ghost_commit_txn(int run_cpu, bool sync, int *commit_state) { int state; @@ -5344,11 +5368,24 @@ static int gsys_ghost_sync_group(ulong __user *user_mask_ptr, */ for_each_cpu_wrap(cpu, cpumask, this_cpu + 1) { bool resched, need_rendezvous; + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + struct ghost_txn *txn; + int commit_flags; + /* * No point in committing this txn if we know a prior txn * failed to commit. */ if (failed) { + txn = rcu_dereference(per_cpu(ghost_txn, cpu)); + commit_flags = READ_ONCE(txn->commit_flags); + + if (commit_flags & INC_AGENT_BARRIER_ON_FAILURE) { + rq_lock_irqsave(rq, &rf); + agent_barrier_inc(rq); + rq_unlock_irqrestore(rq, &rf); + } ghost_poison_txn(cpu); __cpumask_clear_cpu(cpu, cpumask); continue; From 56142b24f0ecb20e45383b54aa493cf357ff7a6f Mon Sep 17 00:00:00 2001 From: Josh Don Date: Fri, 30 Jul 2021 16:02:30 -0700 Subject: [PATCH 036/165] sched: add flag to avoid agent preempting a latched task Adds the run_flag ghost_run():DEFER_LATCHED_PREEMPTION_BY_AGENT. This flag prevents the agent from preempting a latched task in pick_next_ghost_agent(). The purpose of this is to avoid a spurious preemption, which could induce undesired side-effects, such as agent barrier increment. To avoid unbounded scheduling blackout for the latched task, we have the agent preempt any latched task in its sched tick handler. Submitted along with kcl/402930 and cl/387903701. Tested: See cl/387903701 Effort: sched/ghost Change-Id: I949f65179b83a1b5c15a5db1e222cb10e1300693 --- include/uapi/linux/ghost.h | 9 ++++++++- kernel/sched/ghost.c | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index bc224f6b25f9..34f0d6aacfaa 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 43 +#define GHOST_VERSION 44 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -372,6 +372,13 @@ enum ghost_base_ops { * a previous ghost task on this cpu */ #define SEND_TASK_LATCHED (1 << 10) /* Send TASK_LATCHED at commit time */ +/* After the task is latched, don't immediately preempt it if the cpu local + * agent is picked to run; wait at least until the next sched tick hits + * (assuming the agent is still running). This provides a good tradeoff between + * avoiding spurious preemption and preventing an unbounded blackout for the + * latched task while the agent is runnable. + */ +#define DEFER_LATCHED_PREEMPTION_BY_AGENT (1 << 11) /* txn->commit_flags */ #define COMMIT_AT_SCHEDULE (1 << 0) /* commit when oncpu task schedules */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index f67772fa9b17..c4ade97c9152 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -796,6 +796,32 @@ static void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) if (unlikely(!agent)) return; + if (agent == p) { + /* + * We're currently running a ghost agent while a task is + * latched on the cpu. Normally, pick_next_ghost_agent() + * would have preempted the latched task, except in the + * case that DEFER_LATCHED_PREEMPTION_BY_AGENT is set. We need + * to prevent the latched task from experiencing an unbounded + * scheduling blackout if the agent runs for a while, so we + * preempt here in the tick handler. + * + * We could also hit this case without preemption deferment + * if a task was latched while the remote agent was running, + * and the tick hits before the remote agent reschedules. + * In that case, it is fine to preempt here instead of in + * the pick path. + * + * Note that the preemption edge in the pick path is not a new + * edge introduced by DEFER_LATCHED_PREEMPTION_BY_AGENT. Ie. + * previously when an agent is running with a ghost task + * latched, any trip through schedule() would trigger a latched + * preemption via pick_next_ghost_agent(). + */ + if (rq->ghost.latched_task) + ghost_latched_task_preempted(rq); + } + __update_curr_ghost(rq, false); if (cpu_deliver_msg_tick(rq)) @@ -1170,9 +1196,17 @@ static struct task_struct *pick_next_ghost_agent(struct rq *rq) * Treat this like latched_task preemption because we don't know * when the CPU will be available again so no point in keeping * it latched. + * + * If the DEFER_LATCHED_PREEMPTION_BY_AGENT run flag is set, we + * will defer preemption by the agent to the next tick. This + * helps to avoid spurious preemption due to the agent being + * runnable, while still bounding the time a latched task could + * face a scheduling blackout. */ - if (rq->ghost.latched_task) + if (rq->ghost.latched_task && (preempted || + !(rq->ghost.run_flags & DEFER_LATCHED_PREEMPTION_BY_AGENT))) { ghost_latched_task_preempted(rq); + } /* did the preemption msg wake up the local agent? */ if (!next) @@ -4774,6 +4808,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, ELIDE_PREEMPT | NEED_CPU_NOT_IDLE | SEND_TASK_LATCHED | + DEFER_LATCHED_PREEMPTION_BY_AGENT | 0; const int supported_commit_flags = COMMIT_AT_SCHEDULE | From 72a9cd0cde5de15e774ddd14f633faa2843e97cd Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 11 Aug 2021 11:56:42 -0400 Subject: [PATCH 037/165] sched/ghost: allow the agent to request DO_NOT_PREEMPT Add the flag DO_NOT_PREEMPT to commits and BPF latches such that the commit/latch fails if there is a ghost task currently running or latched. When BPF latches a task on a cpu, the agent does not immediately know about the change in state. The agent's notion of an "available cpu" may be stale - it will update it when it receives a TASK_LATCHED message. An agent can set DO_NOT_PREEMPT when it thinks it is scheduling onto an idle cpu. One alternative solution: have transactions (and BPF latches) take the cpu_seqnum as a parameter. In essence, to schedule task X on cpu A, you'd need to have up-to-date seqnum/barriers for both X and A. I'm less inclined to do that, since one of these days we'll support losing messages, and we might need to check the SW for an up-to-date cpu_seqnum. To be submitted in conjunction with cl/390146968. Tested: SLL, with DO_NOT_PREEMPT set for the "available_cpus_" assignment Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I4229cabb00cf49a2ad86db868f1ac5f4a84bbb8a --- include/uapi/linux/ghost.h | 3 ++- kernel/sched/ghost.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 34f0d6aacfaa..1d3001139345 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 44 +#define GHOST_VERSION 45 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -379,6 +379,7 @@ enum ghost_base_ops { * latched task while the agent is runnable. */ #define DEFER_LATCHED_PREEMPTION_BY_AGENT (1 << 11) +#define DO_NOT_PREEMPT (1 << 12) /* Do not preempt running tasks */ /* txn->commit_flags */ #define COMMIT_AT_SCHEDULE (1 << 0) /* commit when oncpu task schedules */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index c4ade97c9152..5d37250d3eda 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4555,6 +4555,7 @@ static int __ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, NEED_L1D_FLUSH | ELIDE_PREEMPT | SEND_TASK_LATCHED | + DO_NOT_PREEMPT | 0; WARN_ON_ONCE(preemptible()); @@ -4603,6 +4604,12 @@ static int __ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, return -EINVAL; } + if ((run_flags & DO_NOT_PREEMPT) && + (task_has_ghost_policy(rq->curr) || rq->ghost.latched_task)) { + task_rq_unlock(rq, next, &rf); + return -ESTALE; + } + /* * If the RQ is in the middle of PNT (where it briefly unlocks), * ghost_can_schedule() is not accurate. rq->curr is still the @@ -4809,6 +4816,7 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, NEED_CPU_NOT_IDLE | SEND_TASK_LATCHED | DEFER_LATCHED_PREEMPTION_BY_AGENT | + DO_NOT_PREEMPT | 0; const int supported_commit_flags = COMMIT_AT_SCHEDULE | @@ -4939,6 +4947,12 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, goto out; } + if (next && (run_flags & DO_NOT_PREEMPT) && + (task_has_ghost_policy(rq->curr) || rq->ghost.latched_task)) { + state = GHOST_TXN_AGENT_STALE; + goto out; + } + if (next && !ghost_can_schedule(rq, gtid)) { /* * Transaction cannot be committed if CPU is not available From fecb2614aed6da1ed98b056231b0e4d17dde282b Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 13 Aug 2021 15:53:45 -0400 Subject: [PATCH 038/165] sched/ghost: do not use BPF-PNT's return value to mean RETRY_TASK Since we're checking rq->latched_task anyways once we relock, using the bpf-pnt return value to tell us to retry wasn't doing much. Suggested-by: neelnatu@google.com Tested: SLL with a PNT-success returning 0 Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I6976c4fdf353ac2e4ef808693521df3bc0e84eab --- kernel/sched/bpf.c | 10 +++------- kernel/sched/ghost.c | 5 ++--- kernel/sched/sched.h | 5 ++--- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 2c73d16a34a3..f1e3581ed46c 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -61,12 +61,10 @@ bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) return BPF_PROG_RUN(prog, &ctx) != 1; } -/* Returns true if pick_next_task_ghost should retry its loop. */ -bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) +void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) { struct bpf_ghost_sched_kern ctx = {}; struct bpf_prog *prog; - int ret; lockdep_assert_held(&rq->lock); @@ -74,7 +72,7 @@ bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) prog = rcu_dereference(e->bpf_pnt); if (!prog) { rcu_read_unlock(); - return false; + return; } /* @@ -85,14 +83,12 @@ bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) rq_unpin_lock(rq, rf); raw_spin_unlock(&rq->lock); - ret = BPF_PROG_RUN(prog, &ctx); + BPF_PROG_RUN(prog, &ctx); raw_spin_lock(&rq->lock); rq_repin_lock(rq, rf); rcu_read_unlock(); - /* prog returns 1 meaning "retry". */ - return ret == 1; } static int ghost_sched_tick_attach(struct ghost_enclave *e, diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 5d37250d3eda..dd23a84cf1f0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -832,7 +832,6 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) { - bool bpf_ret; struct task_struct *agent = rq->ghost.agent; if (!agent || !agent->on_rq) @@ -858,7 +857,7 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, rq->ghost.pnt_bpf_once = false; /* If there is a BPF program, this will unlock the RQ */ rq->ghost.in_pnt_bpf = true; - bpf_ret = ghost_bpf_pnt(agent->ghost.enclave, rq, rf); + ghost_bpf_pnt(agent->ghost.enclave, rq, rf); rq->ghost.in_pnt_bpf = false; } @@ -866,7 +865,7 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, * We have something to run in 'latched_task' or a higher priority * sched_class became runnable while the rq->lock was dropped. */ - return rq->ghost.latched_task || rq_adj_nr_running(rq) || bpf_ret; + return rq->ghost.latched_task || rq_adj_nr_running(rq); } static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ee95d5118639..4fc4b688988c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -261,17 +261,16 @@ extern int ghost_run_gtid_on_check(s64 gtid, u32 task_barrier, int run_flags, struct rq_flags; #ifdef CONFIG_BPF extern bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq); -extern bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, +extern void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf); #else static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { return false; } -static inline bool ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, +static inline void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) { - return false; } #endif From 7db4d3c8cf4ba61c16fbfb4ab6e480a0c0391617 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 25 Aug 2021 12:38:21 -0700 Subject: [PATCH 039/165] sched: convert ghost_wake_on_waker_cpu sysctl to an enclave tunable Replace the ghost_wake_on_waker_cpu sysctl with a "wake_on_waker_cpu" enclave tunable. This tunable has the same meaning as the sysctl and is accessible via ghostfs. Tested: //prodkernel/tests/waker_cpu_test (cl/392957836) in virtme Effort: sched/ghost Change-Id: I7213aaebfd3e9034689a1c93de6f2593b63de496 --- include/linux/sched/sysctl.h | 1 - kernel/sched/ghost.c | 5 ++--- kernel/sched/ghostfs.c | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 1 + kernel/sysctl.c | 7 ------- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 261799db0e3c..211562863bd8 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -33,7 +33,6 @@ extern unsigned int sysctl_sched_child_runs_first; #ifdef CONFIG_SCHED_CLASS_GHOST extern unsigned long sysctl_ghost_cfs_load_added; -extern int sysctl_ghost_wake_on_waker_cpu; extern int sysctl_ghost_commit_at_tick; #endif diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index dd23a84cf1f0..e4adce50b576 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -58,7 +58,6 @@ static DEFINE_PER_CPU(int64_t, sync_group_cookie); /* The load contribution that CFS sees for a running ghOSt task */ unsigned long sysctl_ghost_cfs_load_added = 1024; -int __read_mostly sysctl_ghost_wake_on_waker_cpu; int __read_mostly sysctl_ghost_commit_at_tick; static void ghost_task_new(struct rq *rq, struct task_struct *p); @@ -894,10 +893,10 @@ static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) * pay the cost of task_rq_lock() across sockets which is * exactly what ttwu_queue_remote() aims to minimize. * - * This can be configured using 'sysctl kernel.ghost_wake_on_waker_cpu' + * This can be configured using 'p->ghost.enclave->wake_on_waker_cpu' * and the default is to keep the task on the same CPU it last ran on. */ - if (sysctl_ghost_wake_on_waker_cpu) + if (READ_ONCE(p->ghost.enclave->wake_on_waker_cpu)) p->ghost.twi.wake_up_cpu = waker_cpu; else p->ghost.twi.wake_up_cpu = task_cpu(p); diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 43925bb47264..c207310a1d10 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -609,6 +609,37 @@ static struct kernfs_ops gf_ops_e_switchto_disabled = { .write = gf_switchto_disabled_write, }; +static int gf_wake_on_waker_cpu_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->wake_on_waker_cpu)); + return 0; +} + +static ssize_t gf_wake_on_waker_cpu_write(struct kernfs_open_file *of, + char *buf, size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->wake_on_waker_cpu, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_wake_on_waker_cpu = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_wake_on_waker_cpu_show, + .write = gf_wake_on_waker_cpu_write, +}; + static int gf_status_show(struct seq_file *sf, void *v) { struct ghost_enclave *e = seq_to_e(sf); @@ -682,6 +713,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_switchto_disabled, }, + { + .name = "wake_on_waker_cpu", + .mode = 0664, + .ops = &gf_ops_e_wake_on_waker_cpu, + }, { .name = "status", .mode = 0444, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4fc4b688988c..6c8d87c41aea 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -198,6 +198,7 @@ struct ghost_enclave { struct work_struct enclave_destroyer; bool switchto_disabled; + bool wake_on_waker_cpu; unsigned long id; int is_dying; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2cdb9354beef..e8cfeb4b7c84 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3149,13 +3149,6 @@ static struct ctl_table vm_table[] = { .proc_handler = proc_doulongvec_minmax, .extra1 = SYSCTL_ZERO, }, - { - .procname = "ghost_wake_on_waker_cpu", - .data = &sysctl_ghost_wake_on_waker_cpu, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, { .procname = "ghost_commit_at_tick", .data = &sysctl_ghost_commit_at_tick, From ce73c729eb47359f27c2f1e8c3e9a95cd525dbb3 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 13 Aug 2021 16:26:32 -0400 Subject: [PATCH 040/165] sched/ghost: allow BPF-PNT to tell the kernel to not idle a cpu BPF-PNT fires when PNT has no latched task, and if BPF does not latch a task, the cpu will go idle. If the agent subsequently pushes a task into a BPF ring/runqueue, the cpu will not wake from idle to pull the task. The agent could Ping the cpu, waking the local agent. That is overkill, and the agent may take a while to determine that a cpu was idle when it could have pulled a task from a BPF ring. BPF-PNT programs now can return BPF_GHOST_PNT_DONT_IDLE (1), telling the kernel to not idle the cpu: saving the agent the overhead of Pinging the cpu. Technically, the idle task is selected, but it will immediately return from do_idle() and reenter PNT. Longer range, we'll want more fine-grained idle control, such as which C-state to go into. For now, the agent can control whether we spin calling PNT, or idle like usual. The agent can change this at runtime, per-cpu, similar to how the bpf timer tick suppression works. Tested: SLL (with the magic value 2 instead of 1) Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Iad278ee289771f1eef3515e94b3bebef827c23ca --- kernel/bpf/verifier.c | 1 + kernel/sched/bpf.c | 15 ++++++++++++++- kernel/sched/ghost.c | 16 ++++++++++++++++ kernel/sched/idle.c | 5 +++++ kernel/sched/sched.h | 2 ++ 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f9ec4be8de2d..7bcb4b369d38 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7982,6 +7982,7 @@ static int check_return_code(struct bpf_verifier_env *env) range = tnum_range(SK_DROP, SK_PASS); break; case BPF_PROG_TYPE_GHOST_SCHED: + range = tnum_range(0, INT_MAX); break; case BPF_PROG_TYPE_EXT: /* freplace program can return anything as its return value diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index f1e3581ed46c..725a5ab4086b 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -61,10 +61,13 @@ bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) return BPF_PROG_RUN(prog, &ctx) != 1; } +#define BPF_GHOST_PNT_DONT_IDLE 1 + void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) { struct bpf_ghost_sched_kern ctx = {}; struct bpf_prog *prog; + int ret; lockdep_assert_held(&rq->lock); @@ -83,12 +86,22 @@ void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) rq_unpin_lock(rq, rf); raw_spin_unlock(&rq->lock); - BPF_PROG_RUN(prog, &ctx); + ret = BPF_PROG_RUN(prog, &ctx); raw_spin_lock(&rq->lock); rq_repin_lock(rq, rf); rcu_read_unlock(); + + if (ret == BPF_GHOST_PNT_DONT_IDLE) { + /* + * The next time this rq selects the idle task, it will bail out + * of do_idle() quickly. Since we unlocked the RQ lock, it's + * possible that this rq will pick something other than the idle + * task, which is fine. + */ + rq->ghost.dont_idle_once = true; + } } static int ghost_sched_tick_attach(struct ghost_enclave *e, diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index e4adce50b576..c88e60bb0d0d 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -5906,6 +5906,22 @@ void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next) ghost_wake_agent_on(agent_target_cpu(rq)); } +void ghost_cpu_idle(void) +{ + struct rq *rq = this_rq(); + struct rq_flags rf; + + WARN_ON_ONCE(current != rq->idle); + + rq_lock_irq(rq, &rf); + if (rq->ghost.dont_idle_once) { + set_tsk_need_resched(current); + set_preempt_need_resched(); + rq->ghost.dont_idle_once = false; + } + rq_unlock_irq(rq, &rf); +} + unsigned long ghost_cfs_added_load(struct rq *rq) { int ghost_nr_running = rq->ghost.ghost_nr_running; diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 305727ea0677..00243b55f344 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -273,6 +273,11 @@ static void do_idle(void) __current_set_polling(); tick_nohz_idle_enter(); +#ifdef CONFIG_SCHED_CLASS_GHOST + /* Sets need_resched if rq->ghost.dont_idle_once is set */ + ghost_cpu_idle(); +#endif + while (!need_resched()) { rmb(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6c8d87c41aea..76d803c75fe3 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -115,6 +115,7 @@ struct ghost_rq { bool skip_latched_preemption; bool pnt_bpf_once; /* BPF runs at most once in PNT */ bool in_pnt_bpf; /* running BPF at PNT */ + bool dont_idle_once; /* Don't idle next time rq->idle runs */ int ghost_nr_running; int run_flags; /* flags passed to 'ghost_run()' */ uint64_t cpu_seqnum; /* history for msgs about this cpu */ @@ -258,6 +259,7 @@ extern int ghost_run_gtid_on(s64 gtid, u32 task_barrier, int run_flags, int cpu); extern int ghost_run_gtid_on_check(s64 gtid, u32 task_barrier, int run_flags, int cpu); +extern void ghost_cpu_idle(void); struct rq_flags; #ifdef CONFIG_BPF From 092f5778dcdbe6e7f7e67804ca9b793bcbeeb08c Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 25 Aug 2021 14:00:11 -0400 Subject: [PATCH 041/165] sched/ghost: ensure BPF helper IDs are correct The BPF helper IDs are a stable ABI. We want the same value for the helpers for both prodkernel and the OSS ghost kernel. Our current approach is to count lines and make sure we have the right "placeholders" in uapi/linux/bpf.h. Enforce with a build bug that our helpers start at the correct number. Suggested-by: neelnatu@google.com Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I8982f907aa7d611bdc8d75732159538ca3ac5bd9 --- include/uapi/linux/bpf.h | 1 + kernel/sched/bpf.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 804d9e2346ae..3c156f39ae1c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4013,6 +4013,7 @@ union bpf_attr { FN(ktime_get_coarse_ns), \ FN(ima_inode_hash), \ FN(sock_from_file), \ + FN(placeholder_998), \ FN(placeholder_999), \ FN(placeholder_001), \ FN(placeholder_002), \ diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 725a5ab4086b..bf367310995d 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -18,6 +18,12 @@ BPF_CALL_2(bpf_ghost_wake_agent, struct bpf_ghost_sched_kern *, ctx, u32, cpu) { + /* + * Each BPF helper has a corresponding integer in uapi/linux/bpf.h, + * similar to syscall numbers. Catch any ABI inconsistencies between + * prodkernel and open source. + */ + BUILD_BUG_ON(BPF_FUNC_ghost_wake_agent != 204); return ghost_wake_agent_on_check(cpu); } From 5bdf45ce88bc63c11e81a1cd4b402e0bf0dc618b Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Tue, 31 Aug 2021 12:34:54 -0700 Subject: [PATCH 042/165] sched: convert ghost_commit_at_tick sysctl to an enclave tunable Replace the ghost_commit_at_tick sysctl with a "commit_at_tick" enclave tunable. This tunable has the same meaning as the sysctl and is accessible via ghostfs. 'perf record -a -e cycles:k sleep 10' was ran on jrvj16 after starting agent_muppet with the following flags: --firstcpu 0 --ncpus 256 --globalcpu 17 --select_prio_rq_prob 1.0 --runnable_timeout=0 --bpf_pnt=false --bpf_only=false --bpf_never_idle=false After removing the sysctl and temporarily adding noinline to ghost_commit_all_greedy_txns(): Samples: 5M of event 'cycles:k' Event count (approx.): 826451770332 Overhead Command Shared Object Symbol ........ ............... ................. ................................ 15.92% MustangServer_L [kernel.kallsyms] [k] ghost_commit_all_greedy_txns 3.00% swapper [kernel.kallsyms] [k] mwait_idle_with_hints 2.23% MustangServer_L [kernel.kallsyms] [k] __update_curr_ghost 1.46% khugepaged [kernel.kallsyms] [k] smp_should_send_ipi 1.46% MustangServer_L [kernel.kallsyms] [k] copy_user_generic_string 0.97% khugepaged [kernel.kallsyms] [k] smp_cfm_core_cond After adding it back as a tunable: jrvj16:/sys/fs/ghost/enclave_1# cat commit_at_tick 0 Samples: 4M of event 'cycles:k' Event count (approx.): 527279986835 Overhead Command Shared Object Symbol ........ ............... ................. ............................ 4.04% swapper [kernel.kallsyms] [k] mwait_idle_with_hints 3.71% MustangServer_L [kernel.kallsyms] [k] __update_curr_ghost 2.10% MustangServer_L [kernel.kallsyms] [k] copy_user_generic_string 1.35% swapper [kernel.kallsyms] [k] update_rq_clock_task 1.31% swapper [kernel.kallsyms] [k] acpi_idle_enter 1.18% MustangServer_L [kernel.kallsyms] [k] __resctrl_sched_in 1.15% swapper [kernel.kallsyms] [k] __resctrl_sched_in 1.14% swapper [kernel.kallsyms] [k] pick_next_task_ghost 0.99% swapper [kernel.kallsyms] [k] prepare_task_switch jrvj16:/sys/fs/ghost/enclave_1# echo 1 > commit_at_tick jrvj16:/sys/fs/ghost/enclave_1# cat commit_at_tick 1 Samples: 5M of event 'cycles:k' Event count (approx.): 742319320301 Overhead Command Shared Object Symbol ........ ............... ................. ............................ 16.77% MustangServer_L [kernel.kallsyms] [k] ghost_tick 3.02% swapper [kernel.kallsyms] [k] mwait_idle_with_hints 2.70% MustangServer_L [kernel.kallsyms] [k] __update_curr_ghost 1.41% MustangServer_L [kernel.kallsyms] [k] copy_user_generic_string 1.03% swapper [kernel.kallsyms] [k] __resctrl_sched_in 0.91% MustangServer_L [kernel.kallsyms] [k] __x2apic_send_IPI_mask Tested: SLL on Arcadia as shown above Effort: sched/ghost Change-Id: I761915a0209ca00c663f38b664c8dd81702dcec8 --- include/linux/sched/sysctl.h | 1 - kernel/sched/ghost.c | 8 ++------ kernel/sched/ghostfs.c | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 1 + kernel/sysctl.c | 7 ------- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 211562863bd8..9490bda6e07f 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -33,7 +33,6 @@ extern unsigned int sysctl_sched_child_runs_first; #ifdef CONFIG_SCHED_CLASS_GHOST extern unsigned long sysctl_ghost_cfs_load_added; -extern int sysctl_ghost_commit_at_tick; #endif enum sched_tunable_scaling { diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index c88e60bb0d0d..d67756ad28f8 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -58,7 +58,6 @@ static DEFINE_PER_CPU(int64_t, sync_group_cookie); /* The load contribution that CFS sees for a running ghOSt task */ unsigned long sysctl_ghost_cfs_load_added = 1024; -int __read_mostly sysctl_ghost_commit_at_tick; static void ghost_task_new(struct rq *rq, struct task_struct *p); static void _ghost_task_new(struct rq *rq, struct task_struct *p, @@ -757,14 +756,11 @@ void ghost_tick(struct rq *rq) { struct ghost_enclave *e; - if (sysctl_ghost_commit_at_tick) - ghost_commit_all_greedy_txns(); - - ghost_commit_all_greedy_txns(); - rcu_read_lock(); e = rcu_dereference(*this_cpu_ptr(&enclave)); if (e) { + if (READ_ONCE(e->commit_at_tick)) + ghost_commit_all_greedy_txns(); if (!check_runnable_timeout(e, rq)) { kref_get(&e->kref); if (!schedule_work(&e->enclave_destroyer)) { diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index c207310a1d10..a094b087debf 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -640,6 +640,37 @@ static struct kernfs_ops gf_ops_e_wake_on_waker_cpu = { .write = gf_wake_on_waker_cpu_write, }; +static int gf_commit_at_tick_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->commit_at_tick)); + return 0; +} + +static ssize_t gf_commit_at_tick_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->commit_at_tick, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_commit_at_tick = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_commit_at_tick_show, + .write = gf_commit_at_tick_write, +}; + static int gf_status_show(struct seq_file *sf, void *v) { struct ghost_enclave *e = seq_to_e(sf); @@ -718,6 +749,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_wake_on_waker_cpu, }, + { + .name = "commit_at_tick", + .mode = 0664, + .ops = &gf_ops_e_commit_at_tick, + }, { .name = "status", .mode = 0444, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 76d803c75fe3..50fc56fe7736 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -200,6 +200,7 @@ struct ghost_enclave { bool switchto_disabled; bool wake_on_waker_cpu; + bool commit_at_tick; unsigned long id; int is_dying; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index e8cfeb4b7c84..c20869705955 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3149,13 +3149,6 @@ static struct ctl_table vm_table[] = { .proc_handler = proc_doulongvec_minmax, .extra1 = SYSCTL_ZERO, }, - { - .procname = "ghost_commit_at_tick", - .data = &sysctl_ghost_commit_at_tick, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, #endif { } }; From 02d52da08dcea62d97fcd4b1ed9b5787ccc6732d Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Mon, 13 Sep 2021 12:29:29 -0400 Subject: [PATCH 043/165] sched/ghost: restrict ioctl to writable ctl FDs The agent is the only one who should be allowed to open ctl writable. Ghost clients often have a readonly ctl FD. By restricting ctl to writable FDs only, we ensure only the agent can issue ioctl commands, using the filesystem's permissions instead of relying on capabilities. Tested: edf_test, capabilities_test, api_test, ioctl_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I817942d85f15ecf13fbcdd9e71b711e1e22e4643 --- kernel/sched/ghostfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index a094b087debf..52974535d4d4 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -484,6 +484,8 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, { struct ghost_enclave *e = of_to_e(of); + if (!(of->file->f_mode & FMODE_WRITE)) + return -EACCES; switch (cmd) { case GHOST_IOC_NULL: return 0; From 0674489375826e7159078985b3a8efe6ad588822 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 12 Sep 2021 18:04:42 -0700 Subject: [PATCH 044/165] sched: initialize status_word with correct gtid Prior to this change the 'gtid' field in the ghost_status_word was incorrect for a cloned task (it was incorrectly initialized to the parent's gtid). This happens because gtid allocation must happen after 'p->pid' is initialized which in turn happens after sched_fork->ghost_sched_fork->ghost_prep_task. Rather than changing how pid allocation is done in copy_process() we split status_word allocation and initialization. Thus a forked task will _allocate_ the status_word as before (via sched_fork()) but initialize it after gtid allocation. Tested: $ cat > /tmp/t.sh << EOF iter=0 while [ 1 ]; do sleep 1 iter=$((iter+1)) echo $iter done EOF $ # create enclave $ mount -t ghost ghost /sys/fs/ghost $ echo "create 1" > /sys/fs/ghost/ctl $ echo "0-5" > /sys/fs/ghost/enclave_1/cpulist $ fifo_agent --enclave=/sys/fs/ghost/enclave_1 & $ FIFO_AGENT_PID=$! $ /tmp/t.sh & $ T_SH_PID=$! $ echo $T_SH_PID | pushtosched 18 $ kill -INT $FIFO_AGENT_PID $ fifo_agent --enclave=/sys/fs/ghost/enclave_1 & Prior to this change we would reliably see the following error on agent restart: root@(none):/usr/local/google/home/neelnatu/linux/9xx# Initializing... ---- this was the pid of the /tmp/t.sh process Associating with status_word in region 0, index 6 Trying to associate with gtid 4501400604117010(2047), flags 0x1, runtime 51119671, barrier 82 Change-Id: Id2dbd99e7980cb7a793ad32c66a90b224da259dd --- include/uapi/linux/ghost.h | 3 ++- kernel/fork.c | 2 ++ kernel/sched/ghost.c | 48 +++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 3 ++- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 1d3001139345..cb99934834e6 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 45 +#define GHOST_VERSION 46 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -122,6 +122,7 @@ struct ghost_status_word { #define GHOST_SW_F_INUSE (1U << 0) /* status_word in use */ #define GHOST_SW_F_CANFREE (1U << 1) /* status_word can be freed */ +#define GHOST_SW_F_ALLOCATED (1U << 2) /* status_word is allocated */ /* agent-specific status_word.flags */ #define GHOST_SW_CPU_AVAIL (1U << 8) /* CPU available hint */ diff --git a/kernel/fork.c b/kernel/fork.c index a5842287d897..06a5bab2dfa2 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2319,6 +2319,8 @@ static __latent_entropy struct task_struct *copy_process( #ifdef CONFIG_SCHED_CLASS_GHOST p->gtid = ghost_alloc_gtid(p); + if (ghost_class(p->sched_class)) + ghost_initialize_status_word(p); #endif return p; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index d67756ad28f8..de5b4b6222f8 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -2252,6 +2252,11 @@ static inline bool status_word_canfree(struct ghost_status_word *sw) return sw->flags & GHOST_SW_F_CANFREE; } +static inline bool status_word_allocated(struct ghost_status_word *sw) +{ + return sw->flags & GHOST_SW_F_ALLOCATED; +} + static int free_status_word_locked(struct ghost_enclave *e, struct ghost_status_word *sw) { @@ -2266,7 +2271,7 @@ static int free_status_word_locked(struct ghost_enclave *e, first = first_status_word(header); limit = &first[header->capacity]; if (sw >= first && sw < limit) { - if (!status_word_inuse(sw)) { + if (!status_word_allocated(sw)) { /* * Trying to free an already free status word? */ @@ -2302,7 +2307,7 @@ static struct ghost_status_word *lookup_status_word_locked( } static struct ghost_status_word * -alloc_status_word_locked(struct ghost_enclave *e, uint64_t gtid) +alloc_status_word_locked(struct ghost_enclave *e) { struct ghost_sw_region *region; struct ghost_sw_region_header *header = NULL; @@ -2333,11 +2338,8 @@ alloc_status_word_locked(struct ghost_enclave *e, uint64_t gtid) /* * If this slot is free then claim it and adjust 'available'. */ - if (!status_word_inuse(found)) { - found->flags = GHOST_SW_F_INUSE; - found->barrier = 0; - found->gtid = gtid; - found->runtime = 0; + if (!status_word_allocated(found)) { + found->flags = GHOST_SW_F_ALLOCATED; header->available--; break; } @@ -2347,6 +2349,27 @@ alloc_status_word_locked(struct ghost_enclave *e, uint64_t gtid) return found; } +void ghost_initialize_status_word(struct task_struct *p) +{ + struct ghost_status_word *sw = p->ghost.status_word; + + if (WARN_ON_ONCE(!p->ghost.status_word)) + return; + + if (WARN_ON_ONCE(sw->flags != GHOST_SW_F_ALLOCATED)) + return; + + sw->gtid = gtid(p); + sw->barrier = 0; + sw->runtime = 0; + + /* + * Order is important to ensure that agent observes a fully formed + * status_word when discovering tasks from the status_word region. + */ + ghost_sw_set_flag(sw, GHOST_SW_F_INUSE); +} + int64_t ghost_alloc_gtid(struct task_struct *p) { gtid_t gtid; @@ -2416,16 +2439,25 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, p->ghost.dst_q = q; /* Allocate a status_word */ - sw = alloc_status_word_locked(e, gtid(p)); + sw = alloc_status_word_locked(e); if (sw == NULL) { error = -ENOMEM; goto done; } + kref_get(&e->kref); p->ghost.enclave = e; p->ghost.status_word = sw; p->ghost.new_task = forked; + if (!forked) { + ghost_initialize_status_word(p); + } else { + /* + * Caller will initialize status_word after allocating gtid + * (see copy_process). + */ + } done: if (error) { if (p->ghost.dst_q) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 50fc56fe7736..2486e0b97e42 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -281,7 +281,8 @@ static inline void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, extern void ghost_wait_for_rendezvous(struct rq *rq); extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); -int64_t ghost_alloc_gtid(struct task_struct *p); +extern int64_t ghost_alloc_gtid(struct task_struct *p); +extern void ghost_initialize_status_word(struct task_struct *p); extern void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); From e92deb297f6a597f6cbdff903d74bc73fd11069e Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 12 Sep 2021 21:59:50 -0700 Subject: [PATCH 045/165] sched: allow queue association for a task currently without a 'dst_q' If a task enters ghost in the middle of an agent restart then it is possible that it won't be assigned a 'dst_q' (this could happen if the enclave has no default queue at this point). Prior to this change ghost_associate_queue() assumed that if 'dst_q' was NULL then the task must be dead or departed. The syscall failed with an errno of ENOENT. This in turn triggered a CHECK fail in the agent during task discovery: ghost.cc:121(2137) CHECK FAILED: can_free() != 0 [0 == 0] This comes about when a task enters ghost during agent restart when 'enclave->def_q = NULL'. During task discovery the agent tries to associate the task with the now-valid default queue but this fails with ENOENT as discussed above. In response the agent tries to free the status_word and triggers the CHECK fail (the task is not dead or departed so GHOST_SW_F_CANFREE is not set). Tested: cat > /tmp/t.sh << EOF iter=0 while [ 1 ]; do sleep 1 iter=$((iter+1)) echo $iter done EOF mount -t ghost ghost /sys/fs/ghost echo "create 1" > /sys/fs/ghost/ctl echo "0-5" > /sys/fs/ghost/enclave_1/cpulist fifo_agent --enclave=/sys/fs/ghost/enclave_1 & FIFO_AGENT_PID=$! /tmp/t.sh & T_SH_PID=$! echo $T_SH_PID | pushtosched 18 kill -INT $FIFO_AGENT_PID echo $T_SH_PID | pushtosched 0 echo $T_SH_PID | pushtosched 18 $ fifo_agent --enclave=/sys/fs/ghost/enclave_1 & Prior to this change the agent would CHECK fail on restart (logs include private debugging changes): root@(none):/usr/local/google/home/neelnatu/linux/9xx# Initializing... ---- /tmp/t.sh status_word from previous incarnation. (observe that CANFREE is correctly set) sw(0,36): gtid 0xff4000000080d(2042), flags 0x7 Associating with status_word in region 0, index 36 Trying to associate with gtid 4490405487839245(2042), flags 0x7, runtime 32270540, barrier 16 AssociateTask failed: errno 2 ---- /bin/sleep status_word sw(0,42): gtid 0x10a60000000866(2131), flags 0x20005 Associating with status_word in region 0, index 42 Trying to associate with gtid 4686118557583462(2131), flags 0x20005, runtime 1941667, barrier 107 AssociateTask succeeded: status 0 ---- /tmp/t.sh status_word from current generation (observe that CANFREE is not set) sw(0,43): gtid 0xff4000000080d(2042), flags 0x5 Associating with status_word in region 0, index 43 Trying to associate with gtid 4490405487839245(2042), flags 0x5, runtime 32270540, barrier 1 AssociateTask failed: errno 2 third_party/ghost/lib/ghost.cc:121(2137) CHECK FAILED: can_free() != 0 [0 == 0] errno: 2 [No such file or directory] PID 2136 Backtrace: [0] 0x555650517aed : ghost::Exit() [1] 0x555650510772 : ghost::StatusWord::Free() [2] 0x5556504ff3a3 : ghost::Task::~Task() [3] 0x5556504e3d4e : ghost::FifoTask::~FifoTask() [4] 0x5556504e2542 : ghost::SingleThreadMallocTaskAllocator<>::FreeTaskImpl() [5] 0x5556504e2412 : ghost::ThreadSafeMallocTaskAllocator<>::FreeTask() [6] 0x5556504da43f : ghost::BasicDispatchScheduler<>::DiscoverTasks()::{lambda()#1}::operator()() [7] 0x5556504d9dee : std::__u::__function::__policy_invoker<>::__call_impl<>() [8] 0x5556504f226f : ghost::StatusWordTable::ForEachTaskStatusWord() [9] 0x5556504e88dd : ghost::LocalEnclave::ForEachTaskStatusWord() [10] 0x5556504d92a2 : ghost::BasicDispatchScheduler<>::DiscoverTasks() [11] 0x5556504e78ec : ghost::Enclave::Ready() [12] 0x5556504d886e : ghost::FullFifoAgent<>::FullFifoAgent() [13] 0x5556504d861b : std::__u::make_unique<>() [14] 0x5556504d7d8e : ghost::AgentProcess<>::AgentProcess() [15] 0x5556504d70d1 : main [16] 0x7fb1104bbbbd : __libc_start_main [17] 0x5556504d6129 : _start Effort: sched/ghost Change-Id: I0bd3e42a8f9397d66babea95d68a2b53a16d8d1c --- kernel/sched/ghost.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index de5b4b6222f8..587d14efc43f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3213,7 +3213,7 @@ static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, oldq = p->ghost.dst_q; sw = p->ghost.status_word; - if (unlikely(!oldq || !sw)) { + if (unlikely(!sw)) { /* Task is dead or switched to another sched_class */ WARN_ON_ONCE(p->state != TASK_DEAD && ghost_class(p->sched_class)); @@ -3250,7 +3250,8 @@ static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, queue_incref(newq); p->ghost.dst_q = newq; - queue_decref(oldq); + if (oldq) + queue_decref(oldq); done: if (p) From e8e65a47e646dc6e13875b51194a4295e253968d Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Mon, 13 Sep 2021 17:46:04 -0400 Subject: [PATCH 046/165] sched/ghost: remove unused bpf prog attach/detach PROG_ATTACH/DETACH are unused. I added them a while back when I thought we'd attach programs to tasks instead of enclaves, and I wanted the programs to live beyond the live of an agent. It's unlikely we'll do that. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ic0bd6a164bb54d848e956a38c782093716546f4d --- include/linux/sched.h | 9 ++--- kernel/bpf/syscall.c | 5 --- kernel/sched/bpf.c | 80 ------------------------------------------- 3 files changed, 2 insertions(+), 92 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 7d05ee963aa8..1667c2b45339 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1460,13 +1460,8 @@ struct bpf_ghost_sched_kern { struct bpf_prog; union bpf_attr; -enum bpf_prog_type; -int ghost_sched_bpf_prog_attach(const union bpf_attr *attr, - struct bpf_prog *prog); -int ghost_sched_bpf_prog_detach(const union bpf_attr *attr, - enum bpf_prog_type ptype); -int ghost_sched_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog); +extern int ghost_sched_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog); static inline struct pid *task_pid(struct task_struct *task) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c3df31de5c1c..154330189c39 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3018,9 +3018,6 @@ static int bpf_prog_attach(const union bpf_attr *attr) case BPF_PROG_TYPE_SOCK_OPS: ret = cgroup_bpf_prog_attach(attr, ptype, prog); break; - case BPF_PROG_TYPE_GHOST_SCHED: - ret = ghost_sched_bpf_prog_attach(attr, prog); - break; default: ret = -EINVAL; } @@ -3057,8 +3054,6 @@ static int bpf_prog_detach(const union bpf_attr *attr) case BPF_PROG_TYPE_CGROUP_SYSCTL: case BPF_PROG_TYPE_SOCK_OPS: return cgroup_bpf_prog_detach(attr, ptype); - case BPF_PROG_TYPE_GHOST_SCHED: - return ghost_sched_bpf_prog_detach(attr, ptype); default: return -EINVAL; } diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index bf367310995d..4af357f473b3 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -282,74 +282,6 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, return bpf_link_settle(&link_primer); } -/* netns does this to have a packed array of progs[type]. might do this for the - * task type only, or maybe for all ghost types. - */ -enum ghost_sched_bpf_attach_type { - GHOST_SCHED_BPF_INVALID = -1, - GHOST_SCHED_BPF_TICK = 0, - GHOST_SCHED_BPF_PNT, - MAX_SCHED_BPF_ATTACH_TYPE -}; - -static inline enum ghost_sched_bpf_attach_type -to_ghost_sched_bpf_attach_type(enum bpf_attach_type attach_type) -{ - switch (attach_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - return GHOST_SCHED_BPF_TICK; - case BPF_GHOST_SCHED_PNT: - return GHOST_SCHED_BPF_PNT; - default: - return GHOST_SCHED_BPF_INVALID; - } -} - -int ghost_sched_bpf_prog_attach(const union bpf_attr *attr, - struct bpf_prog *prog) -{ - enum ghost_sched_bpf_attach_type type; - - if (attr->target_fd || attr->attach_flags || attr->replace_bpf_fd) - return -EINVAL; - type = to_ghost_sched_bpf_attach_type(attr->attach_type); - if (type < 0) - return -EINVAL; - - /* no task attachable types yet */ - - return -1; -} - -int ghost_sched_bpf_prog_detach(const union bpf_attr *attr, - enum bpf_prog_type ptype) -{ - struct bpf_prog *prog; - enum ghost_sched_bpf_attach_type type; - - if (attr->attach_flags) - return -EINVAL; - - type = to_ghost_sched_bpf_attach_type(attr->attach_type); - if (type < 0) - return -EINVAL; - - prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); - if (IS_ERR(prog)) - return PTR_ERR(prog); - - if (prog->expected_attach_type != attr->attach_type) { - bpf_prog_put(prog); - return -EINVAL; - } - - /* no task attachable types yet */ - - bpf_prog_put(prog); - - return -1; -} - static const struct bpf_func_proto * ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -427,16 +359,4 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, return -EINVAL; } -int ghost_sched_bpf_prog_attach(const union bpf_attr *attr, - struct bpf_prog *prog) -{ - return -EINVAL; -} - -int ghost_sched_bpf_prog_detach(const union bpf_attr *attr, - enum bpf_prog_type ptype) -{ - return -EINVAL; -} - #endif /* !CONFIG_SCHED_CLASS_GHOST */ From ef0f710aa9b367d5a23ec888d8d49862b1eb77c4 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 14 Sep 2021 11:10:26 -0400 Subject: [PATCH 047/165] sched/ghost: remove ctx from bpf helpers We don't need to pass the context to the helpers. It's unused and unnecessary. Passing the context, e.g. bpf_ghost_sched, makes it harder to call the ghost bpf helpers from other program types, since not all prog types have the same context. Most don't. An upcoming commit adds another BPF_PROG_TYPE for ghost which will be able to call some of these helpers. Note that whether or not a particular helper is callable by a particular prog_type+attach_point is managed by the verifier op get_func_proto(), which has nothing to do with the context. This is an ABI change for anyone using a bpf helper. To be submitted with cl/396599402. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Iec790881b69f62840b1c1d39f56f19ca4f24ef67 --- include/linux/sched.h | 3 +-- include/uapi/linux/bpf.h | 5 ++--- include/uapi/linux/ghost.h | 2 +- kernel/sched/bpf.c | 11 ++++------- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 1667c2b45339..f7aac6ae4df1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1455,8 +1455,7 @@ struct task_struct { */ }; -struct bpf_ghost_sched_kern { -}; +#define bpf_ghost_sched_kern bpf_ghost_sched struct bpf_prog; union bpf_attr; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 3c156f39ae1c..c87293e2704f 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3834,15 +3834,14 @@ union bpf_attr { * A pointer to a struct socket on success or NULL if the file is * not a socket. * - * long bpf_ghost_wake_agent(struct bpf_ghost_sched *ctx, u32 cpu) + * long bpf_ghost_wake_agent(u32 cpu) * Description * Wakes the ghost agent on *cpu* * * Return * 0 on success, < 0 on error. * - * long bpf_ghost_run_gtid(struct bpf_ghost_sched *ctx, s64 gtid, u32 - * task_barrier, s32 run_flags) + * long bpf_ghost_run_gtid(s64 gtid, u32 task_barrier, s32 run_flags) * Description * Runs (latches) task **gtid** on this cpu. * diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index cb99934834e6..642416c6c025 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -28,7 +28,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 46 +#define GHOST_VERSION 47 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 4af357f473b3..b638c8cbae57 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -16,7 +16,7 @@ #ifdef CONFIG_SCHED_CLASS_GHOST -BPF_CALL_2(bpf_ghost_wake_agent, struct bpf_ghost_sched_kern *, ctx, u32, cpu) +BPF_CALL_1(bpf_ghost_wake_agent, u32, cpu) { /* * Each BPF helper has a corresponding integer in uapi/linux/bpf.h, @@ -31,12 +31,10 @@ static const struct bpf_func_proto bpf_ghost_wake_agent_proto = { .func = bpf_ghost_wake_agent, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_ANYTHING, + .arg1_type = ARG_ANYTHING, }; -BPF_CALL_4(bpf_ghost_run_gtid, struct bpf_ghost_sched_kern *, ctx, s64, gtid, - u32, task_barrier, int, run_flags) +BPF_CALL_3(bpf_ghost_run_gtid, s64, gtid, u32, task_barrier, int, run_flags) { return ghost_run_gtid_on(gtid, task_barrier, run_flags, smp_processor_id()); @@ -46,10 +44,9 @@ static const struct bpf_func_proto bpf_ghost_run_gtid_proto = { .func = bpf_ghost_run_gtid, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_CTX, + .arg1_type = ARG_ANYTHING, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, - .arg4_type = ARG_ANYTHING, }; bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) From 4a3e587fe0d35c628766eec13a6bbacc46ea0270 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 14 Sep 2021 11:09:04 -0700 Subject: [PATCH 048/165] sched: handle tasks moving into ghost while rq->lock is dropped in PNT. An oncpu task may switch into ghost (e.g. via a third party calling sched_setscheduler(2)) while the rq->lock is dropped in one of the pick_next_task handlers (e.g. during CFS idle load balancing). It is not possible to distinguish this in switched_to_ghost() so we resolve 'prev->ghost.new_task' here. Failing to do this has dire consequences if 'prev' is runnable since it will languish in the kernel forever (in contrast to a blocked task there is no trigger forthcoming to produce a TASK_NEW in the future). Tested: white box tested with //prodkernel/ghost:switchto_test which is occasionally able to repro the setsched(GHOST) during pick next task. Effort: sched/ghost Change-Id: I3396a383ee230c27412341f4ca6b5bad49d84f47 --- kernel/sched/core.c | 26 ++++++++++++++++++++++++++ kernel/sched/ghost.c | 3 +-- kernel/sched/sched.h | 1 + 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b56d47a0912d..87566bac9daf 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4270,6 +4270,32 @@ static inline void ghost_prepare_task_switch(struct rq *rq, if (prev == rq->ghost.agent) goto done; + /* + * An oncpu task may switch into ghost (e.g. via a third party calling + * sched_setscheduler(2)) while the rq->lock is dropped in one of the + * pick_next_task handlers (e.g. during CFS idle load balancing). + * + * It is not possible to distinguish this in switched_to_ghost() so + * we resolve 'prev->ghost.new_task' here. Failing to do this has + * dire consequences if 'prev' is runnable since it will languish + * in the kernel forever (in contrast to a blocked task there is + * no trigger forthcoming to produce a TASK_NEW in the future). + */ + if (unlikely(prev->ghost.new_task)) { + WARN_ON_ONCE(prev->ghost.yield_task); + WARN_ON_ONCE(prev->ghost.blocked_task); + + /* + * We just produced TASK_NEW so no need to consider 'prev' for + * a preemption edge (see ghost_produce_prev_msgs for details). + */ + rq->ghost.check_prev_preemption = false; + + prev->ghost.new_task = false; + ghost_task_new(rq, prev); + ghost_wake_agent_of(prev); + } + /* If we're on the ghost.tasks list, then we're runnable. */ if (!list_empty(&prev->ghost.run_list)) { /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 587d14efc43f..ca3677e69347 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -59,7 +59,6 @@ static DEFINE_PER_CPU(int64_t, sync_group_cookie); /* The load contribution that CFS sees for a running ghOSt task */ unsigned long sysctl_ghost_cfs_load_added = 1024; -static void ghost_task_new(struct rq *rq, struct task_struct *p); static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable); static void ghost_task_yield(struct rq *rq, struct task_struct *p); @@ -4281,7 +4280,7 @@ static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable) task_deliver_msg_task_new(rq, p, runnable); } -static void ghost_task_new(struct rq *rq, struct task_struct *p) +void ghost_task_new(struct rq *rq, struct task_struct *p) { _ghost_task_new(rq, p, task_on_rq_queued(p)); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 2486e0b97e42..80a9d358f340 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -283,6 +283,7 @@ extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); extern int64_t ghost_alloc_gtid(struct task_struct *p); extern void ghost_initialize_status_word(struct task_struct *p); +extern void ghost_task_new(struct rq *rq, struct task_struct *p); extern void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); From 934c481bd2e53a16239f650e9de8b898060c0be0 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 15 Sep 2021 12:53:57 -0400 Subject: [PATCH 049/165] sched/ghost: refactor message sending for bpf_ghost_msg To run a bpf program for every message, we'll need a common type: bpf_ghost_msg. This will be the 'context' for a new BPF_PROG_TYPE in an upcoming commit. Previously, message payloads were created on the stack. Now, we use per-cpu space for them. If we use the stack, we have to zero the message so as to not leak any padding fields when we copy the struct to userspace. That's true for the old code too. The sizeof the struct includes extra bytes for padding. This includes both "interior" padding, such as in struct ghost_msg_payload_task_new between runnable and sw_info, as well as trailing padding, such as the two bytes after struct ghost_msg_payload_task_preempt to make the struct size u64 aligned. No functional change. I'll update the uapi header file in a later commit. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I14c19d8e0024d971e12068fd1aff7fd19eac8455 --- include/uapi/linux/ghost.h | 20 +++ kernel/sched/ghost.c | 273 +++++++++++++++++++++++++------------ 2 files changed, 207 insertions(+), 86 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 642416c6c025..2a6d34643ed3 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -280,6 +280,26 @@ struct ghost_msg_payload_timer { uint64_t cookie; }; +struct bpf_ghost_msg { + union { + struct ghost_msg_payload_task_dead dead; + struct ghost_msg_payload_task_blocked blocked; + struct ghost_msg_payload_task_wakeup wakeup; + struct ghost_msg_payload_task_new newt; + struct ghost_msg_payload_task_preempt preempt; + struct ghost_msg_payload_task_yield yield; + struct ghost_msg_payload_task_departed departed; + struct ghost_msg_payload_task_switchto switchto; + struct ghost_msg_payload_task_affinity_changed affinity; + struct ghost_msg_payload_task_latched latched; + struct ghost_msg_payload_cpu_tick cpu_tick; + struct ghost_msg_payload_timer timer; + struct ghost_msg_payload_cpu_not_idle cpu_not_idle; + }; + uint16_t type; + uint32_t seqnum; +}; + #ifdef __cplusplus #include typedef std::atomic _ghost_ring_index_t; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index ca3677e69347..8e5ca8e2289f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -43,6 +43,11 @@ static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_txn *, ghost_txn); static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); static DEFINE_PER_CPU(int64_t, sync_group_cookie); +/* + * Use per-cpu memory instead of stack memory to avoid memsetting. We only + * send one message at a time per cpu. + */ +static DEFINE_PER_CPU(struct bpf_ghost_msg, bpf_ghost_msg); /* * We do not want to make 'SG_COOKIE_CPU_BITS' larger than necessary so that @@ -3560,21 +3565,94 @@ static int _produce(struct ghost_queue *q, uint32_t barrier, int type, return 0; } -static inline int produce_for_task(struct task_struct *p, int type, - void *payload, int payload_size) + +static inline int __produce_for_task(struct task_struct *p, + struct bpf_ghost_msg *msg, + uint32_t barrier) { - return _produce(p->ghost.dst_q, task_barrier_get(p), - type, payload, payload_size); + void *payload; + int payload_size; + + msg->seqnum = barrier; + + switch (msg->type) { + case MSG_TASK_DEAD: + payload = &msg->dead; + payload_size = sizeof(msg->dead); + break; + case MSG_TASK_BLOCKED: + payload = &msg->blocked; + payload_size = sizeof(msg->blocked); + break; + case MSG_TASK_WAKEUP: + payload = &msg->wakeup; + payload_size = sizeof(msg->wakeup); + break; + case MSG_TASK_NEW: + payload = &msg->newt; + payload_size = sizeof(msg->newt); + break; + case MSG_TASK_PREEMPT: + payload = &msg->preempt; + payload_size = sizeof(msg->preempt); + break; + case MSG_TASK_YIELD: + payload = &msg->yield; + payload_size = sizeof(msg->yield); + break; + case MSG_TASK_DEPARTED: + payload = &msg->departed; + payload_size = sizeof(msg->departed); + break; + case MSG_TASK_SWITCHTO: + payload = &msg->switchto; + payload_size = sizeof(msg->switchto); + break; + case MSG_TASK_AFFINITY_CHANGED: + payload = &msg->affinity; + payload_size = sizeof(msg->affinity); + break; + case MSG_TASK_LATCHED: + payload = &msg->latched; + payload_size = sizeof(msg->latched); + break; + case MSG_CPU_TICK: + payload = &msg->cpu_tick; + payload_size = sizeof(msg->cpu_tick); + break; + case MSG_CPU_TIMER_EXPIRED: + payload = &msg->timer; + payload_size = sizeof(msg->timer); + break; + case MSG_CPU_NOT_IDLE: + payload = &msg->cpu_not_idle; + payload_size = sizeof(msg->cpu_not_idle); + break; + default: + WARN(1, "unknown bpg_ghost_msg type %d!\n", msg->type); + return -EINVAL; + }; + /* + * TODO(brho): Call a BPF_PROG here that can modify msg. Note this + * means BPF can change the type if it wants. That's fine. + */ + return _produce(p->ghost.dst_q, barrier, msg->type, + payload, payload_size); +} + +static inline int produce_for_task(struct task_struct *p, + struct bpf_ghost_msg *msg) +{ + return __produce_for_task(p, msg, task_barrier_get(p)); } -static inline int produce_for_agent(struct rq *rq, int type, void *payload, - int payload_size) +static inline int produce_for_agent(struct rq *rq, + struct bpf_ghost_msg *msg) { struct task_struct *agent = rq->ghost.agent; agent_barrier_inc(rq); - return _produce(agent->ghost.dst_q, agent_barrier_get(agent), - type, payload, payload_size); + return __produce_for_task(agent, msg, agent_barrier_get(agent)); } /* @@ -3604,7 +3682,8 @@ static inline bool cpu_skip_message(struct rq *rq) static inline bool cpu_deliver_msg_tick(struct rq *rq) { - struct ghost_msg_payload_cpu_tick payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_cpu_tick *payload = &msg->cpu_tick; struct ghost_enclave *e; if (cpu_skip_message(rq)) @@ -3617,38 +3696,41 @@ static inline bool cpu_deliver_msg_tick(struct rq *rq) } rcu_read_unlock(); - payload.cpu = cpu_of(rq); + msg->type = MSG_CPU_TICK; + payload->cpu = cpu_of(rq); - return !produce_for_agent(rq, MSG_CPU_TICK, &payload, sizeof(payload)); + return !produce_for_agent(rq, msg); } /* Returns true if MSG_CPU_TIMER_EXPIRED was produced and false otherwise */ static inline bool cpu_deliver_timer_expired(struct rq *rq, uint64_t cookie) { - struct ghost_msg_payload_timer payload; - const int size = sizeof(payload); + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_timer *payload = &msg->timer; if (cpu_skip_message(rq)) return false; - payload.cpu = cpu_of(rq); - payload.cookie = cookie; + msg->type = MSG_CPU_TIMER_EXPIRED; + payload->cpu = cpu_of(rq); + payload->cookie = cookie; - return !produce_for_agent(rq, MSG_CPU_TIMER_EXPIRED, &payload, size); + return !produce_for_agent(rq, msg); } static bool cpu_deliver_msg_not_idle(struct rq *rq, struct task_struct *next) { - struct ghost_msg_payload_cpu_not_idle payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_cpu_not_idle *payload = &msg->cpu_not_idle; if (cpu_skip_message(rq)) return false; - payload.cpu = cpu_of(rq); - payload.next_gtid = gtid(next); + msg->type = MSG_CPU_NOT_IDLE; + payload->cpu = cpu_of(rq); + payload->next_gtid = gtid(next); - return !produce_for_agent(rq, MSG_CPU_NOT_IDLE, - &payload, sizeof(payload)); + return !produce_for_agent(rq, msg); } /* @@ -3729,44 +3811,48 @@ static inline int __task_deliver_common(struct rq *rq, struct task_struct *p) static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p, bool runnable) { - struct ghost_msg_payload_task_new payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_new *payload = &msg->newt; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.runnable = runnable; - payload.runtime = p->se.sum_exec_runtime; + msg->type = MSG_TASK_NEW; + payload->gtid = gtid(p); + payload->runnable = runnable; + payload->runtime = p->se.sum_exec_runtime; if (_get_sw_info(p->ghost.enclave, p->ghost.status_word, - &payload.sw_info)) { + &payload->sw_info)) { WARN(1, "New task PID %d didn't have a status word!", p->pid); return; } - produce_for_task(p, MSG_TASK_NEW, &payload, sizeof(payload)); + produce_for_task(p, msg); } static void task_deliver_msg_yield(struct rq *rq, struct task_struct *p) { - - struct ghost_msg_payload_task_yield payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_yield *payload = &msg->yield; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.runtime = p->se.sum_exec_runtime; - payload.cpu = cpu_of(rq); - payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; - payload.from_switchto = ghost_in_switchto(rq); + msg->type = MSG_TASK_YIELD; + payload->gtid = gtid(p); + payload->runtime = p->se.sum_exec_runtime; + payload->cpu = cpu_of(rq); + payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; + payload->from_switchto = ghost_in_switchto(rq); - produce_for_task(p, MSG_TASK_YIELD, &payload, sizeof(payload)); + produce_for_task(p, msg); } static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, bool from_switchto, bool was_latched) { - struct ghost_msg_payload_task_preempt payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_preempt *payload = &msg->preempt; if (__task_deliver_common(rq, p)) return; @@ -3785,67 +3871,75 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, */ WARN_ON_ONCE(from_switchto && rq->ghost.switchto_count > 0); - payload.gtid = gtid(p); - payload.runtime = p->se.sum_exec_runtime; - payload.cpu = cpu_of(rq); - payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; - payload.from_switchto = from_switchto; - payload.was_latched = was_latched; + msg->type = MSG_TASK_PREEMPT; + payload->gtid = gtid(p); + payload->runtime = p->se.sum_exec_runtime; + payload->cpu = cpu_of(rq); + payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; + payload->from_switchto = from_switchto; + payload->was_latched = was_latched; - produce_for_task(p, MSG_TASK_PREEMPT, &payload, sizeof(payload)); + produce_for_task(p, msg); } static void task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_blocked payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_blocked *payload = &msg->blocked; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.runtime = p->se.sum_exec_runtime; - payload.cpu = cpu_of(rq); - payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; - payload.from_switchto = ghost_in_switchto(rq); + msg->type = MSG_TASK_BLOCKED; + payload->gtid = gtid(p); + payload->runtime = p->se.sum_exec_runtime; + payload->cpu = cpu_of(rq); + payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; + payload->from_switchto = ghost_in_switchto(rq); - produce_for_task(p, MSG_TASK_BLOCKED, &payload, sizeof(payload)); + produce_for_task(p, msg); } static void task_deliver_msg_dead(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_dead payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_dead *payload = &msg->dead; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - produce_for_task(p, MSG_TASK_DEAD, &payload, sizeof(payload)); + msg->type = MSG_TASK_DEAD; + payload->gtid = gtid(p); + produce_for_task(p, msg); } static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_departed payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_departed *payload = &msg->departed; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.cpu = cpu_of(rq); - payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; + msg->type = MSG_TASK_DEAD; + payload->gtid = gtid(p); + payload->cpu = cpu_of(rq); + payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; if (task_current(rq, p) && ghost_in_switchto(rq)) - payload.from_switchto = true; + payload->from_switchto = true; else - payload.from_switchto = false; - payload.was_current = task_current(rq, p); + payload->from_switchto = false; + payload->was_current = task_current(rq, p); - produce_for_task(p, MSG_TASK_DEPARTED, &payload, - sizeof(payload)); + produce_for_task(p, msg); } static void task_deliver_msg_affinity_changed(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_affinity_changed payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_affinity_changed *payload = + &msg->affinity; /* * A running task can be switched into ghost while it is executing @@ -3859,27 +3953,29 @@ static void task_deliver_msg_affinity_changed(struct rq *rq, if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); + msg->type = MSG_TASK_AFFINITY_CHANGED; + payload->gtid = gtid(p); - produce_for_task(p, MSG_TASK_AFFINITY_CHANGED, &payload, - sizeof(payload)); + produce_for_task(p, msg); } static void task_deliver_msg_latched(struct rq *rq, struct task_struct *p, bool latched_preempt) { - struct ghost_msg_payload_task_latched payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_latched *payload = &msg->latched; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.commit_time = ktime_get_ns(); - payload.cpu = cpu_of(rq); - payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; - payload.latched_preempt = latched_preempt; + msg->type = MSG_TASK_LATCHED; + payload->gtid = gtid(p); + payload->commit_time = ktime_get_ns(); + payload->cpu = cpu_of(rq); + payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; + payload->latched_preempt = latched_preempt; - produce_for_task(p, MSG_TASK_LATCHED, &payload, sizeof(payload)); + produce_for_task(p, msg); } static inline bool deferrable_wakeup(struct task_struct *p) @@ -3898,32 +3994,37 @@ static inline bool deferrable_wakeup(struct task_struct *p) static void task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_wakeup payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_wakeup *payload = &msg->wakeup; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.deferrable = deferrable_wakeup(p); - payload.last_ran_cpu = p->ghost.twi.last_ran_cpu; - payload.wake_up_cpu = p->ghost.twi.wake_up_cpu; - payload.waker_cpu = p->ghost.twi.waker_cpu; - produce_for_task(p, MSG_TASK_WAKEUP, &payload, sizeof(payload)); + msg->type = MSG_TASK_WAKEUP; + payload->gtid = gtid(p); + payload->deferrable = deferrable_wakeup(p); + payload->last_ran_cpu = p->ghost.twi.last_ran_cpu; + payload->wake_up_cpu = p->ghost.twi.wake_up_cpu; + payload->waker_cpu = p->ghost.twi.waker_cpu; + + produce_for_task(p, msg); } static void task_deliver_msg_switchto(struct rq *rq, struct task_struct *p) { - struct ghost_msg_payload_task_switchto payload; + struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); + struct ghost_msg_payload_task_switchto *payload = &msg->switchto; if (__task_deliver_common(rq, p)) return; - payload.gtid = gtid(p); - payload.runtime = p->se.sum_exec_runtime; - payload.cpu = cpu_of(rq); - payload.cpu_seqnum = ++rq->ghost.cpu_seqnum; + msg->type = MSG_TASK_SWITCHTO; + payload->gtid = gtid(p); + payload->runtime = p->se.sum_exec_runtime; + payload->cpu = cpu_of(rq); + payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; - produce_for_task(p, MSG_TASK_SWITCHTO, &payload, sizeof(payload)); + produce_for_task(p, msg); } static void release_from_ghost(struct rq *rq, struct task_struct *p) From e9eb20701b92aea716bba9fa523067b4efc55271 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 15 Sep 2021 13:58:32 -0400 Subject: [PATCH 050/165] sched/ghost: refactor ghost_bpf_link() In anticipation of another BPF_PROG_TYPE_GHOST, refactor the old ghost_bpf_sched_link to handle multiple prog_types. Specifically: - rename ghost_sched_bpf_link_attach() -> ghost_bpf_link_attach() - move ghost_bpf_link_attach() later in bpf.c, after prog_type-specific code. - track prog_type in struct bpf_ghost_link - switch on prog_type in addition to ea_type in the link functions Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I93e83f1c2e6720dfb0370c391e9a5dce77545532 --- include/linux/sched.h | 4 +- kernel/bpf/syscall.c | 2 +- kernel/sched/bpf.c | 246 +++++++++++++++++++++++------------------- 3 files changed, 138 insertions(+), 114 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index f7aac6ae4df1..ce4caf8a2f14 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1459,8 +1459,8 @@ struct task_struct { struct bpf_prog; union bpf_attr; -extern int ghost_sched_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog); +extern int ghost_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog); static inline struct pid *task_pid(struct task_struct *task) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 154330189c39..726f58f160b9 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4072,7 +4072,7 @@ static int link_create(union bpf_attr *attr) break; #endif case BPF_PROG_TYPE_GHOST_SCHED: - ret = ghost_sched_bpf_link_attach(attr, prog); + ret = ghost_bpf_link_attach(attr, prog); break; default: ret = -EINVAL; diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index b638c8cbae57..435d10d3f893 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -157,56 +157,133 @@ static void ghost_sched_pnt_detach(struct ghost_enclave *e, spin_unlock_irqrestore(&e->lock, irq_fl); } -struct bpf_ghost_sched_link { +static const struct bpf_func_proto * +ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_ghost_wake_agent: + return &bpf_ghost_wake_agent_proto; + case BPF_FUNC_ghost_run_gtid: + switch (prog->expected_attach_type) { + case BPF_GHOST_SCHED_PNT: + return &bpf_ghost_run_gtid_proto; + default: + return NULL; + } + default: + return bpf_base_func_proto(func_id); + } +} + +static bool ghost_sched_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + /* The verifier guarantees that size > 0. */ + if (off < 0 || off + size > sizeof(struct bpf_ghost_sched) + || off % size) + return false; + + switch (off) { + default: + return false; + } +} + +#define SCHEDULER_ACCESS_FIELD(T, F) \ + T(BPF_FIELD_SIZEOF(struct bpf_ghost_sched_kern, F), \ + si->dst_reg, si->src_reg, \ + offsetof(struct bpf_ghost_sched_kern, F)) + +static u32 ghost_sched_convert_ctx_access(enum bpf_access_type type, + const struct bpf_insn *si, + struct bpf_insn *insn_buf, + struct bpf_prog *prog, + u32 *target_size) +{ + struct bpf_insn *insn = insn_buf; + + switch (si->off) { + default: + *target_size = 0; + break; + } + + *target_size = sizeof(u32); + + return insn - insn_buf; +} + +const struct bpf_verifier_ops ghost_sched_verifier_ops = { + .get_func_proto = ghost_sched_func_proto, + .is_valid_access = ghost_sched_is_valid_access, + .convert_ctx_access = ghost_sched_convert_ctx_access, +}; + +const struct bpf_prog_ops ghost_sched_prog_ops = { +}; + + +struct bpf_ghost_link { struct bpf_link link; struct ghost_enclave *e; + enum bpf_prog_type prog_type; enum bpf_attach_type ea_type; }; -static void bpf_ghost_sched_link_release(struct bpf_link *link) +static void bpf_ghost_link_dealloc(struct bpf_link *link) +{ + struct bpf_ghost_link *sc_link = + container_of(link, struct bpf_ghost_link, link); + + kfree(sc_link); +} + +static void bpf_ghost_link_release(struct bpf_link *link) { - struct bpf_ghost_sched_link *sc_link = - container_of(link, struct bpf_ghost_sched_link, link); + struct bpf_ghost_link *sc_link = + container_of(link, struct bpf_ghost_link, link); struct ghost_enclave *e = sc_link->e; if (WARN_ONCE(!e, "Missing enclave for bpf link ea_type %d!", sc_link->ea_type)) return; - switch (sc_link->ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - ghost_sched_tick_detach(e, link->prog); - break; - case BPF_GHOST_SCHED_PNT: - ghost_sched_pnt_detach(e, link->prog); + switch (sc_link->prog_type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (sc_link->ea_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + ghost_sched_tick_detach(e, link->prog); + break; + case BPF_GHOST_SCHED_PNT: + ghost_sched_pnt_detach(e, link->prog); + break; + default: + WARN_ONCE(1, "Unexpected release for ea_type %d", + sc_link->ea_type); + break; + } break; default: - WARN_ONCE(1, "Unexpected release for ea_type %d", - sc_link->ea_type); + WARN_ONCE(1, "Unexpected release for prog_type %d", + sc_link->prog_type); break; - }; + } kref_put(&e->kref, enclave_release); sc_link->e = NULL; } -static void bpf_ghost_sched_link_dealloc(struct bpf_link *link) -{ - struct bpf_ghost_sched_link *sc_link = - container_of(link, struct bpf_ghost_sched_link, link); - - kfree(sc_link); -} - -static const struct bpf_link_ops bpf_ghost_sched_link_ops = { - .release = bpf_ghost_sched_link_release, - .dealloc = bpf_ghost_sched_link_dealloc, +static const struct bpf_link_ops bpf_ghost_link_ops = { + .release = bpf_ghost_link_release, + .dealloc = bpf_ghost_link_dealloc, }; -int ghost_sched_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog) +int ghost_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog) { struct bpf_link_primer link_primer; - struct bpf_ghost_sched_link *sc_link; + struct bpf_ghost_link *sc_link; enum bpf_attach_type ea_type; struct ghost_enclave *e; struct fd f_enc = {0}; @@ -218,19 +295,26 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, return -EINVAL; ea_type = prog->expected_attach_type; - switch (ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - case BPF_GHOST_SCHED_PNT: + switch (prog->type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (ea_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + case BPF_GHOST_SCHED_PNT: + break; + default: + return -EINVAL; + } break; default: return -EINVAL; - }; + } sc_link = kzalloc(sizeof(*sc_link), GFP_USER); if (!sc_link) return -ENOMEM; bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, - &bpf_ghost_sched_link_ops, prog); + &bpf_ghost_link_ops, prog); + sc_link->prog_type = prog->type; sc_link->ea_type = ea_type; err = bpf_link_prime(&sc_link->link, &link_primer); @@ -256,19 +340,26 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, sc_link->e = e; ghost_fdput_enclave(e, &f_enc); - switch (ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - err = ghost_sched_tick_attach(e, prog); - break; - case BPF_GHOST_SCHED_PNT: - err = ghost_sched_pnt_attach(e, prog); + switch (prog->type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (ea_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + err = ghost_sched_tick_attach(e, prog); + break; + case BPF_GHOST_SCHED_PNT: + err = ghost_sched_pnt_attach(e, prog); + break; + default: + pr_warn("bad sched bpf ea_type %d, should be unreachable", + ea_type); + err = -EINVAL; + break; + } break; default: - pr_warn("bad sched bpf ea_type %d, should be unreachable", - ea_type); - err = -EINVAL; - break; - }; + return -EINVAL; + } + if (err) { /* bpf_link_cleanup() triggers .dealloc, but not .release. */ bpf_link_cleanup(&link_primer); @@ -279,79 +370,12 @@ int ghost_sched_bpf_link_attach(const union bpf_attr *attr, return bpf_link_settle(&link_primer); } -static const struct bpf_func_proto * -ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -{ - switch (func_id) { - case BPF_FUNC_ghost_wake_agent: - return &bpf_ghost_wake_agent_proto; - case BPF_FUNC_ghost_run_gtid: - switch (prog->expected_attach_type) { - case BPF_GHOST_SCHED_PNT: - return &bpf_ghost_run_gtid_proto; - default: - return NULL; - } - default: - return bpf_base_func_proto(func_id); - } -} - -static bool ghost_sched_is_valid_access(int off, int size, - enum bpf_access_type type, - const struct bpf_prog *prog, - struct bpf_insn_access_aux *info) -{ - /* The verifier guarantees that size > 0. */ - if (off < 0 || off + size > sizeof(struct bpf_ghost_sched) - || off % size) - return false; - - switch (off) { - default: - return false; - } -} - -#define SCHEDULER_ACCESS_FIELD(T, F) \ - T(BPF_FIELD_SIZEOF(struct bpf_ghost_sched_kern, F), \ - si->dst_reg, si->src_reg, \ - offsetof(struct bpf_ghost_sched_kern, F)) - -static u32 ghost_sched_convert_ctx_access(enum bpf_access_type type, - const struct bpf_insn *si, - struct bpf_insn *insn_buf, - struct bpf_prog *prog, - u32 *target_size) -{ - struct bpf_insn *insn = insn_buf; - - switch (si->off) { - default: - *target_size = 0; - break; - } - - *target_size = sizeof(u32); - - return insn - insn_buf; -} - -const struct bpf_verifier_ops ghost_sched_verifier_ops = { - .get_func_proto = ghost_sched_func_proto, - .is_valid_access = ghost_sched_is_valid_access, - .convert_ctx_access = ghost_sched_convert_ctx_access, -}; - -const struct bpf_prog_ops ghost_sched_prog_ops = { -}; #else /* !CONFIG_SCHED_CLASS_GHOST */ const struct bpf_verifier_ops ghost_sched_verifier_ops = {}; const struct bpf_prog_ops ghost_sched_prog_ops = {}; -int ghost_sched_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog) +int ghost_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { return -EINVAL; } From 834469bc84219e3c0e6f36c80fffff80154295ce Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 15 Sep 2021 14:41:20 -0400 Subject: [PATCH 051/165] sched/ghost: add BPF_PROG_TYPE_GHOST_MSG A BPF_PROG_TYPE_GHOST_MSG takes a struct bpf_ghost_msg as its argument. The agent can attach one of these programs (per enclave) at BPF_GHOST_MSG_SEND, and it will run for every message. The program should return 0 to send the message (the default action), or 1 to suppress the message. This hook allows the agent to quickly react to events, such as a task waking up or being preempted, before the global agent gets around to doing message dispatch. The set of messages in uapi/ghost.h and their semantics is the API by which we talk to both the agent and its bpf program; it is still the agent receiving the messages, just in its BPF_GHOST_MSG_SEND program. Long term, if we have every agent using bpf, we can move the message queues, the wakeup config, and even the message format into the bpf program - making all of it agent-specific. As of now, you could have bpf do its own messaging with a bpf ringbuffer map, and wake the agent on whatever cpu it wants with bpf_ghost_wake_agent(cpu). To be submitted in conjunction with cl/397090164. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I44c2dd805dc8d2c7fcf8c0e7a53642c790acf899 --- include/linux/bpf_types.h | 2 + include/linux/sched.h | 2 + include/uapi/linux/bpf.h | 4 +- include/uapi/linux/ghost.h | 6 +- kernel/bpf/syscall.c | 10 ++++ kernel/sched/bpf.c | 114 +++++++++++++++++++++++++++++++++++++ kernel/sched/ghost.c | 6 +- kernel/sched/sched.h | 8 +++ 8 files changed, 145 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index c3f020349925..058a489cc735 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -79,6 +79,8 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm, #endif BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_SCHED, ghost_sched, struct bpf_ghost_sched, struct bpf_ghost_sched_kern) +BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_MSG, ghost_msg, struct bpf_ghost_msg, + struct bpf_ghost_msg_kern) BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) diff --git a/include/linux/sched.h b/include/linux/sched.h index ce4caf8a2f14..eb5f60bd8157 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -8,6 +8,7 @@ */ #include +#include #include @@ -1456,6 +1457,7 @@ struct task_struct { }; #define bpf_ghost_sched_kern bpf_ghost_sched +#define bpf_ghost_msg_kern bpf_ghost_msg struct bpf_prog; union bpf_attr; diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c87293e2704f..77769e679bb6 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -201,6 +201,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_GHOST_SCHED = 35, + BPF_PROG_TYPE_GHOST_MSG, }; enum bpf_attach_type { @@ -243,7 +244,8 @@ enum bpf_attach_type { BPF_SK_LOOKUP, BPF_XDP, BPF_GHOST_SCHED_SKIP_TICK = 50, - BPF_GHOST_SCHED_PNT = 51, + BPF_GHOST_SCHED_PNT, + BPF_GHOST_MSG_SEND, __MAX_BPF_ATTACH_TYPE }; diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 2a6d34643ed3..d18ab1956941 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -16,7 +16,9 @@ #include -#ifndef __KERNEL__ +#ifdef __KERNEL__ +#include +#else #include #include #endif @@ -28,7 +30,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 47 +#define GHOST_VERSION 48 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 726f58f160b9..d7d5a7fc48d8 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2010,6 +2010,13 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, default: return -EINVAL; } + case BPF_PROG_TYPE_GHOST_MSG: + switch (expected_attach_type) { + case BPF_GHOST_MSG_SEND: + return 0; + default: + return -EINVAL; + } case BPF_PROG_TYPE_EXT: if (expected_attach_type) return -EINVAL; @@ -2963,6 +2970,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) case BPF_GHOST_SCHED_SKIP_TICK: case BPF_GHOST_SCHED_PNT: return BPF_PROG_TYPE_GHOST_SCHED; + case BPF_GHOST_MSG_SEND: + return BPF_PROG_TYPE_GHOST_MSG; default: return BPF_PROG_TYPE_UNSPEC; } @@ -4072,6 +4081,7 @@ static int link_create(union bpf_attr *attr) break; #endif case BPF_PROG_TYPE_GHOST_SCHED: + case BPF_PROG_TYPE_GHOST_MSG: ret = ghost_bpf_link_attach(attr, prog); break; default: diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 435d10d3f893..4d08054a76ff 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -107,6 +107,28 @@ void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) } } +/* + * Returns true (default) if the user wants us to send the message. + * Note that our caller holds an RQ lock, and we can't safely unlock it, so + * programs at the attach point can't call ghost_run_gtid(). + */ +bool ghost_bpf_msg_send(struct ghost_enclave *e, struct bpf_ghost_msg *msg) +{ + struct bpf_prog *prog; + bool send; + + rcu_read_lock(); + prog = rcu_dereference(e->bpf_msg_send); + if (!prog) { + rcu_read_unlock(); + return true; + } + /* Program returns 0 if they want us to send the message. */ + send = BPF_PROG_RUN(prog, msg) == 0; + rcu_read_unlock(); + return send; +} + static int ghost_sched_tick_attach(struct ghost_enclave *e, struct bpf_prog *prog) { @@ -225,6 +247,64 @@ const struct bpf_prog_ops ghost_sched_prog_ops = { }; +static int ghost_msg_send_attach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->bpf_msg_send) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + rcu_assign_pointer(e->bpf_msg_send, prog); + spin_unlock_irqrestore(&e->lock, irq_fl); + return 0; +} + +static void ghost_msg_send_detach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + rcu_replace_pointer(e->bpf_msg_send, NULL, lockdep_is_held(&e->lock)); + spin_unlock_irqrestore(&e->lock, irq_fl); +} + +static const struct bpf_func_proto * +ghost_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_ghost_wake_agent: + return &bpf_ghost_wake_agent_proto; + default: + return bpf_base_func_proto(func_id); + } +} + +static bool ghost_msg_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + /* The verifier guarantees that size > 0. */ + if (off < 0 || off + size > sizeof(struct bpf_ghost_msg) + || off % size) + return false; + + return true; +} + +const struct bpf_verifier_ops ghost_msg_verifier_ops = { + .get_func_proto = ghost_msg_func_proto, + .is_valid_access = ghost_msg_is_valid_access, +}; + +const struct bpf_prog_ops ghost_msg_prog_ops = { +}; + + struct bpf_ghost_link { struct bpf_link link; struct ghost_enclave *e; @@ -265,6 +345,17 @@ static void bpf_ghost_link_release(struct bpf_link *link) break; } break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (sc_link->ea_type) { + case BPF_GHOST_MSG_SEND: + ghost_msg_send_detach(e, link->prog); + break; + default: + WARN_ONCE(1, "Unexpected release for ea_type %d", + sc_link->ea_type); + break; + } + break; default: WARN_ONCE(1, "Unexpected release for prog_type %d", sc_link->prog_type); @@ -305,6 +396,14 @@ int ghost_bpf_link_attach(const union bpf_attr *attr, return -EINVAL; } break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (ea_type) { + case BPF_GHOST_MSG_SEND: + break; + default: + return -EINVAL; + } + break; default: return -EINVAL; } @@ -356,6 +455,18 @@ int ghost_bpf_link_attach(const union bpf_attr *attr, break; } break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (ea_type) { + case BPF_GHOST_MSG_SEND: + err = ghost_msg_send_attach(e, prog); + break; + default: + pr_warn("bad msg bpf ea_type %d, should be unreachable", + ea_type); + err = -EINVAL; + break; + } + break; default: return -EINVAL; } @@ -375,6 +486,9 @@ int ghost_bpf_link_attach(const union bpf_attr *attr, const struct bpf_verifier_ops ghost_sched_verifier_ops = {}; const struct bpf_prog_ops ghost_sched_prog_ops = {}; +const struct bpf_verifier_ops ghost_msg_verifier_ops = {}; +const struct bpf_prog_ops ghost_msg_prog_ops = {}; + int ghost_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { return -EINVAL; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8e5ca8e2289f..c5eef946cb2d 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3632,10 +3632,8 @@ static inline int __produce_for_task(struct task_struct *p, WARN(1, "unknown bpg_ghost_msg type %d!\n", msg->type); return -EINVAL; }; - /* - * TODO(brho): Call a BPF_PROG here that can modify msg. Note this - * means BPF can change the type if it wants. That's fine. - */ + if (!ghost_bpf_msg_send(p->ghost.enclave, msg)) + return -ENOMSG; return _produce(p->ghost.dst_q, barrier, msg->type, payload, payload_size); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 80a9d358f340..a2eef93ab0fd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -210,6 +210,7 @@ struct ghost_enclave { #ifdef CONFIG_BPF struct bpf_prog *bpf_tick; struct bpf_prog *bpf_pnt; + struct bpf_prog *bpf_msg_send; #endif }; @@ -267,6 +268,8 @@ struct rq_flags; extern bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq); extern void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf); +extern bool ghost_bpf_msg_send(struct ghost_enclave *e, + struct bpf_ghost_msg *msg); #else static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { @@ -276,6 +279,11 @@ static inline void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) { } +static inline bool ghost_bpf_msg_send(struct ghost_enclave *e, + struct bpf_ghost_msg *msg) +{ + return true; +} #endif extern void ghost_wait_for_rendezvous(struct rq *rq); From 2993dcd082a5f14d912d7dd60eec7266c3e2560b Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 16 Sep 2021 11:20:57 -0400 Subject: [PATCH 052/165] sched/ghost: add agent_data to common 'runnable' messages 'agent_data' is a blob that the agent can use to communicate between its BPF_GHOST_MSG_SEND program and its userspace message handler. For "runnable" messages, such as Preempt and Wakeup, SGS will try to push the task into a bpf ring. SGS will use blob to pass a pointer to the task's slot. This is a little inelegant - how do we know a u64 is enough to handle the variety of an agent's communication desires between bpf and userspace? Should we have 'agent_data' on all messages? Arguably, bpf can generate messages with a custom size and ABI for its particular agent! A u64 is fine for now. Patchset submitted in conjunction with cl/397090164. Tested: prototype agent with BPF-MSG pushing tasks into per-cpu rings for MSG_TASK_RUNNABLE Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I1f7801945dd353e463a960eb7920827918f7ab21 --- include/uapi/linux/ghost.h | 5 ++++- kernel/sched/ghost.c | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index d18ab1956941..a65eda33b945 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -30,7 +30,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 48 +#define GHOST_VERSION 49 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -197,6 +197,7 @@ struct ghost_msg_payload_task_preempt { uint64_t gtid; uint64_t runtime; /* cumulative runtime in ns */ uint64_t cpu_seqnum; /* cpu sequence number */ + uint64_t agent_data; /* used by bpf */ int cpu; char from_switchto; char was_latched; @@ -206,6 +207,7 @@ struct ghost_msg_payload_task_yield { uint64_t gtid; uint64_t runtime; /* cumulative runtime in ns */ uint64_t cpu_seqnum; + uint64_t agent_data; /* used by bpf */ int cpu; char from_switchto; }; @@ -236,6 +238,7 @@ struct ghost_msg_payload_task_affinity_changed { struct ghost_msg_payload_task_wakeup { uint64_t gtid; + uint64_t agent_data; /* used by bpf */ char deferrable; /* bool: 0 or 1 */ int last_ran_cpu; /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index c5eef946cb2d..69013091260e 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3841,6 +3841,7 @@ static void task_deliver_msg_yield(struct rq *rq, struct task_struct *p) payload->runtime = p->se.sum_exec_runtime; payload->cpu = cpu_of(rq); payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; + payload->agent_data = 0; payload->from_switchto = ghost_in_switchto(rq); produce_for_task(p, msg); @@ -3874,6 +3875,7 @@ static void task_deliver_msg_preempt(struct rq *rq, struct task_struct *p, payload->runtime = p->se.sum_exec_runtime; payload->cpu = cpu_of(rq); payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; + payload->agent_data = 0; payload->from_switchto = from_switchto; payload->was_latched = was_latched; @@ -4000,6 +4002,7 @@ static void task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p) msg->type = MSG_TASK_WAKEUP; payload->gtid = gtid(p); + payload->agent_data = 0; payload->deferrable = deferrable_wakeup(p); payload->last_ran_cpu = p->ghost.twi.last_ran_cpu; payload->wake_up_cpu = p->ghost.twi.wake_up_cpu; From e752fd570bbcf14450399d8fa7cffa4bb405e2df Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 22 Sep 2021 14:52:37 -0700 Subject: [PATCH 053/165] sched: fix typo in task_deliver_msg_departed() Fixes: 4cc5414085c1 ("refactor message sending for bpf_ghost_msg") Tested: api_test --gunit_filter=ApiTest.DepartedRace Effort: sched/ghost Change-Id: I4fa09c88ca368e1ba84781cf28a022f6009f9a95 --- kernel/sched/ghost.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 69013091260e..880df9639887 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3921,7 +3921,7 @@ static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) if (__task_deliver_common(rq, p)) return; - msg->type = MSG_TASK_DEAD; + msg->type = MSG_TASK_DEPARTED; payload->gtid = gtid(p); payload->cpu = cpu_of(rq); payload->cpu_seqnum = ++rq->ghost.cpu_seqnum; From 96d8a363e6646945ebc83966f4dd8bb52730e19f Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 12 Sep 2021 15:50:43 -0700 Subject: [PATCH 054/165] sched: prevent departed->new msg reordering. The ghost kernel promises in-order delivery of messages belonging to a particular task. Agents rely on this to track the evolution of the task state in userspace. When a task moves into the ghost sched_class the kernel produces a TASK_NEW msg on the enclave's default_queue. When a task moves out of ghost a TASK_DEPARTED msg is produced into the queue associated with the task (the agent controls the {task->queue} association). In certain situations it is possible for TASK_NEW and TASK_DEPARTED to be reordered which violates some fundamental assumptions in the agent. Restore sanity by ensuring that a TASK_NEW (beginning a new incarnation) is not produced until all msgs belonging to the previous incarnation have been consumed. We detect this by monitoring when the agent frees the status_word of the previous incarnation (until that point the task is not allowed to produce any msgs). The common case is straightforward but we must handle some corner cases like agent restart, enclave destruction and task's entering ghost while exiting. Note that ABI version is unchanged because we don't have anything in userspace that depends on the GHOST_SW_TASK_MSG_GATED status_word flag. Tested: - all unit tests passing in virtme - ghost smoketest - 2LCE/VIT - SLL: - baseline - move all batch tasks from SLL to CFS rapidly with cgroup_watcher frequency cranked up to 100 msec. - no warnings or (lasting) effect on QPS. Effort: sched/ghost Change-Id: I217ecb17d9d535634137c008bae77ecf44778ebc --- include/linux/sched.h | 2 + include/uapi/linux/ghost.h | 2 + kernel/fork.c | 2 + kernel/sched/core.c | 2 + kernel/sched/ghost.c | 308 +++++++++++++++++++++++++++++++++++-- kernel/sched/sched.h | 1 + 6 files changed, 301 insertions(+), 16 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index eb5f60bd8157..12a5ebd9aa1c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -768,6 +768,8 @@ struct task_struct { struct sched_rt_entity rt; #ifdef CONFIG_SCHED_CLASS_GHOST int64_t gtid; /* ghost tid */ + uint inhibit_task_msgs; /* don't produce msgs for this task */ + struct list_head inhibited_task_list; struct sched_ghost_entity ghost; #endif #ifdef CONFIG_CGROUP_SCHED diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index a65eda33b945..586f6ffcfb45 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -134,6 +134,8 @@ struct ghost_status_word { #define GHOST_SW_TASK_ONCPU (1U << 16) /* task is oncpu */ #define GHOST_SW_TASK_RUNNABLE (1U << 17) /* task is runnable */ #define GHOST_SW_TASK_IS_AGENT (1U << 18) +#define GHOST_SW_TASK_MSG_GATED (1U << 19) /* all task msgs are gated until + * status_word is freed by agent */ /* * Queue APIs. diff --git a/kernel/fork.c b/kernel/fork.c index 06a5bab2dfa2..d0210651e95f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2321,6 +2321,8 @@ static __latent_entropy struct task_struct *copy_process( p->gtid = ghost_alloc_gtid(p); if (ghost_class(p->sched_class)) ghost_initialize_status_word(p); + WARN_ON_ONCE(p->inhibit_task_msgs); + WARN_ON_ONCE(!list_empty(&p->inhibited_task_list)); #endif return p; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 87566bac9daf..6d6b9cc9c48a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3689,6 +3689,8 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->rt.on_list = 0; #ifdef CONFIG_SCHED_CLASS_GHOST + p->inhibit_task_msgs = 0; + INIT_LIST_HEAD(&p->inhibited_task_list); sched_ghost_entity_init(p); #endif diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 880df9639887..a39c8c9c1573 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -69,6 +69,8 @@ static void _ghost_task_new(struct rq *rq, struct task_struct *p, static void ghost_task_yield(struct rq *rq, struct task_struct *p); static void ghost_task_blocked(struct rq *rq, struct task_struct *p); static void task_dead_ghost(struct task_struct *p); +static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p, + bool runnable); static void task_deliver_msg_affinity_changed(struct rq *rq, struct task_struct *p); static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p); @@ -120,6 +122,11 @@ static bool check_same_enclave(int cpu_x, int cpu_y) return x == y; } +static inline uint enclave_abi(struct ghost_enclave *e) +{ + return GHOST_VERSION; /* for now all enclaves have the same abi */ +} + /* enclave::is_dying */ #define ENCLAVE_IS_DYING (1U << 0) #define ENCLAVE_IS_REAPABLE (1U << 1) @@ -1818,6 +1825,12 @@ static void enclave_actual_release(struct work_struct *w) list_for_each_entry_safe(swr, temp, &e->sw_region_list, list) __sw_region_free(swr); + /* + * Any inhibited tasks should have been "released" as a side-effect + * of freeing the status_word regions above. + */ + WARN_ON_ONCE(!list_empty(&e->inhibited_task_list)); + for_each_possible_cpu(cpu) vfree(e->cpu_data[cpu]); @@ -1923,6 +1936,7 @@ struct ghost_enclave *ghost_create_enclave(void) kref_init(&e->kref); INIT_LIST_HEAD(&e->sw_region_list); INIT_LIST_HEAD(&e->task_list); + INIT_LIST_HEAD(&e->inhibited_task_list); INIT_LIST_HEAD(&e->ew.link); INIT_WORK(&e->task_reaper, enclave_reap_tasks); INIT_WORK(&e->enclave_actual_release, enclave_actual_release); @@ -2428,6 +2442,31 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, error = -EXDEV; goto done; } + + /* + * If msg inhibition is in effect then make sure that the task does + * not jump ABI versions (i.e. it must setsched into an enclave that + * has the same abi version as the one it belonged to earlier). + * + * Note that this is trivially true if the task switches back into + * the same enclave. + * + * Why? different enclaves may be at different ABI versions so when + * the status_word is freed from the original enclave we may not have + * access to the function that produces TASK_NEW in the new ABI. + */ + if (p->inhibit_task_msgs) { + /* + * 'inhibit_task_msgs' is used as a boolean but it actually + * holds the abi of the enclave the task departed from. + */ + uint old_abi = p->inhibit_task_msgs; + if (old_abi != enclave_abi(e)) { + error = -EINVAL; + goto done; + } + } + if (q) { /* * It's possible for a client task to have no queue: it may have @@ -2477,6 +2516,112 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, return error; } +/* + * Caller must call ghost_uninhibit_task_msgs(p) on the task returned from + * this function. Ideally we would have had a single function that removes + * the task from the 'inhibited_task_list' and enables msgs for this task, + * but we must split them up due to locking requirements (removing the + * task requires enclave to be locked whereas ungating msgs requires + * that no locks are held). + */ +struct task_struct *__enclave_remove_inhibited_task(struct ghost_enclave *e, + gtid_t gtid) +{ + struct task_struct *p; + + if (kref_read(&e->kref)) { + lockdep_assert_held(&e->lock); + } else { + /* + * No more references to the enclave (called from + * enclave_actual_release()). + */ + } + + /* + * N.B. cannot use find_task_by_gtid() to do the {gtid->task_struct} + * lookup here since that function expects to be called in an agent + * context. + */ + list_for_each_entry(p, &e->inhibited_task_list, inhibited_task_list) { + if (p->gtid == gtid) { + list_del_init(&p->inhibited_task_list); + return p; + } + } + return NULL; +} + +static void ghost_uninhibit_task_msgs(struct task_struct *p) +{ + struct rq_flags rf; + struct rq *rq; + + if (WARN_ON_ONCE(!p->inhibit_task_msgs)) + goto done; + + rq = task_rq_lock(p, &rf); + if (p->state != TASK_DEAD) { + /* + * 'inhibit_task_msgs' is used as a boolean but it actually + * holds the abi of the enclave the task departed from. + */ + const uint old_abi = p->inhibit_task_msgs; + + p->inhibit_task_msgs = 0; /* open the gate */ + + if (ghost_class(p->sched_class)) { + WARN_ON_ONCE(old_abi != enclave_abi(p->ghost.enclave)); + if (p->ghost.new_task) { + /* + * Task is oncpu and will produce TASK_NEW on + * the next scheduling edge. + */ + WARN_ON_ONCE(!task_running(rq, p)); + } else { + const bool runnable = task_on_rq_queued(p); + task_deliver_msg_task_new(rq, p, runnable); + ghost_wake_agent_of(p); + } + } else { + /* + * Nothing: we opened the gate above so if/when the + * task switches into ghost we'll produce a TASK_NEW + * msg immediately. + */ + } + } else { + /* + * There are three possibilities here: + * + * 1. task has finished scheduling for the last time and the + * task_dead() callback is happening concurrently or has + * finished executing. + * + * 2. task is in the middle of its last __schedule() and we + * sneaked in while rq->lock was dropped in one of the PNT + * sched_class callbacks. + * + * 3. task is about to __schedule() for the very last time + * (running concurrently in do_task_dead() or spinning on + * rq->lock in __schedule()). + * + * It would be problematic if we produced TASK_NEW in the + * first scenario since there is no guarantee that we'll + * also produce the corresponding TASK_DEAD and because we + * cannot distinguish (1) from (2) or (3) we won't produce + * the TASK_NEW msg or clear p->inhibit_task_msgs. + * + * Note that release_from_ghost() will free the status_word + * in all of these cases. + */ + } + task_rq_unlock(rq, p, &rf); + +done: + put_task_struct(p); +} + /* * Only called from enclave_release. Any users of this SWR had an enclave kref, * so if we're here, they must all have stopped using their SW. This also @@ -2488,11 +2633,54 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, */ static int __sw_region_free(struct ghost_sw_region *swr) { - struct ghost_sw_region_header *h = swr->header; + struct ghost_sw_region_header *header = swr->header; + struct ghost_status_word *sw; + struct task_struct *p; + + /* + * Enclave is destroyed so scan all status words to find tasks + * where msg gating is in effect (and then turn it off because + * any agent that could have been handling msgs from an earlier + * incarnation is dead). + */ + for (sw = first_status_word(header); + sw < first_status_word(header) + header->capacity; sw++) { + /* + * inuse canfree + * 0 0 empty slot + * 0 1 not expected (see WARN below) + * 1 0 not expected (see WARN below) + * 1 1 task died or departed (either due to + * task_reaper or organically while the + * enclave was being destroyed). + */ + if (!status_word_inuse(sw)) { + WARN_ON_ONCE(status_word_canfree(sw)); + continue; + } + + if (WARN_ON_ONCE(!status_word_canfree(sw))) + continue; + + if (!(sw->flags & GHOST_SW_TASK_MSG_GATED)) { + /* + * task died while enclave was being destroyed or a + * misbehaving agent did not free status word in its + * TASK_DEAD handler. + */ + continue; + } + + p = __enclave_remove_inhibited_task(swr->enclave, sw->gtid); + if (WARN_ON_ONCE(!p)) + continue; + + ghost_uninhibit_task_msgs(p); + } list_del(&swr->list); - vfree(h); + vfree(header); kfree(swr); /* XXX memcg uncharge */ @@ -3145,7 +3333,10 @@ int ghost_sw_get_info(struct ghost_enclave *e, int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo) { int error; + bool gated; + gtid_t gtid; ulong flags; + struct task_struct *p = NULL; struct ghost_status_word *sw; struct ghost_sw_info info = { 0 }; @@ -3163,9 +3354,18 @@ int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo) error = -EINVAL; goto done; } + gtid = sw->gtid; + gated = sw->flags & GHOST_SW_TASK_MSG_GATED; error = free_status_word_locked(e, sw); + if (!error && gated) { + p = __enclave_remove_inhibited_task(e, gtid); + WARN_ON_ONCE(!p); + } done: spin_unlock_irqrestore(&e->lock, flags); + if (p) + ghost_uninhibit_task_msgs(p); + return error; } @@ -3247,6 +3447,24 @@ static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, if (p->ghost.new_task) status |= GHOST_ASSOC_SF_BRAND_NEW; + /* + * All task msgs may be inhibited (including TASK_NEW) because the + * agent hasn't finished handling TASK_DEPARTED from the previous + * incarnation. This is similar to 'p->ghost.new_task' check above + * in that TASK_NEW will be produced when the status_word from the + * earlier incarnation is freed. + */ + if (p->inhibit_task_msgs) { + /* + * The status_word associated with the previous incarnation + * of the task is orphaned (it cannot be reached from any + * task struct) so we should never see the GATED flag in + * any "live" status_word. + */ + WARN_ON_ONCE(sw->flags & GHOST_SW_TASK_MSG_GATED); + status |= GHOST_ASSOC_SF_BRAND_NEW; + } + if (ustatus && copy_to_user(ustatus, &status, sizeof(status))) { error = -EFAULT; goto done; @@ -3803,6 +4021,15 @@ static inline int __task_deliver_common(struct rq *rq, struct task_struct *p) if (!p->ghost.dst_q) return -1; + /* + * Inhibit tasks msgs until agent acknowledges receipt of an earlier + * TASK_DEPARTED (by freeing the task's status_word). This ensures + * that msgs belonging to the previous incarnation of the task are + * drained before any msg from its current incarnation is produced. + */ + if (p->inhibit_task_msgs) + return -1; + return 0; } @@ -4046,22 +4273,71 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) /* Remove before potentially clearing p->ghost.agent. */ __enclave_remove_task(e, p); - /* - * Annotate the status_word so it can be freed by the agent. - * - * N.B. this must be ordered before the TASK_DEAD message is - * visible to the agent. - * - * N.B. we grab the enclave lock to prevent a misbehaving agent - * from freeing the status_word (or its enclosing sw_region) - * before delivery of the TASK_DEAD message below. - */ - ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_F_CANFREE); - if (p->state == TASK_DEAD) - task_deliver_msg_dead(rq, p); - else + if (p->inhibit_task_msgs) { + /* + * Agent is not going to free the status_word since it + * never saw any msgs associated with this incarnation: + * 1. p->state != TASK_DEAD (task departed) + * task departed, came back into ghost and departed again + * before agent had a chance to handle the first DEPARTED. + * 2. p->state == TASK_DEAD (task is dead) + * task departed and came back into ghost while exiting + * before agent had a chance to handle the DEPARTED msg. + */ + free_status_word_locked(e, p->ghost.status_word); + } else if (p->state != TASK_DEAD) { + /* agents cannot setsched out of ghost */ + WARN_ON_ONCE(is_agent(rq, p)); + + ghost_sw_set_flag(p->ghost.status_word, + GHOST_SW_F_CANFREE|GHOST_SW_TASK_MSG_GATED); task_deliver_msg_departed(rq, p); + /* + * Inhibit all msgs produced by this task until the agent + * acknowledges receipt of TASK_DEPARTED msg (implicitly by + * freeing the status_word). This ensures that when TASK_NEW + * is produced any msgs associated with an earlier incarnation + * have been consumed. + */ + p->inhibit_task_msgs = enclave_abi(e); + + /* + * Track the task in the enclave's 'inhibited_task_list'. + * + * We must do this because a departed task can re-enter ghost + * while it is exiting and subsequently lose its identity + * (e.g. see go/kcl/390722). It is possible for the task to + * enter a different enclave in which case the fallback in + * kcl/390722 is ineffective. + * + * TODO: declare enclave failure if agent does not free the + * status_word of a departed task within X msec (X should be + * smaller than the e->max_unscheduled timeout): + * - misbehavior of one enclave should not result in a + * (false-positive) failure of a different enclave. + * - limit the task_struct memory held hostage while they are + * on the inhibited_task_list. + */ + get_task_struct(p); + WARN_ON_ONCE(!list_empty(&p->inhibited_task_list)); + list_add_tail(&p->inhibited_task_list, &e->inhibited_task_list); + } else { + /* + * Annotate the status_word so it can be freed by the agent. + * + * N.B. this must be ordered before the TASK_DEAD message is + * visible to the agent. + * + * N.B. we grab the enclave lock to prevent a misbehaving agent + * from freeing the status_word (or its enclosing sw_region) + * before delivery of the TASK_DEAD message below. + */ + ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_F_CANFREE); + task_deliver_msg_dead(rq, p); + } + WARN_ON_ONCE(!p->inhibit_task_msgs && p->state != TASK_DEAD); + /* status_word is off-limits to the kernel */ p->ghost.status_word = NULL; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a2eef93ab0fd..a624379ee4e0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -184,6 +184,7 @@ struct ghost_enclave { struct ghost_queue *def_q; /* default queue */ + struct list_head inhibited_task_list; struct list_head task_list; /* all non-agent tasks in the enclave */ unsigned long nr_tasks; struct work_struct task_reaper; From 6d07e3a82c34710e6005dd109cf0d57308e8b5b3 Mon Sep 17 00:00:00 2001 From: Josh Don Date: Thu, 12 Aug 2021 16:58:49 -0700 Subject: [PATCH 055/165] sched/ghost: fix TASK_DEPARTED race with validate_next_task() The agent might attempt to schedule a task that recently departed (ie. before consuming the TASK_DEPARTED message). In this case, we should return a TARGET_STALE status rather than INVALID_TARGET. Effort: sched/ghost Tested: no longer crashing due to invalid target when running test_2lce Change-Id: I5742f8216c900d4ba7681bc10d61fd052dc5c625 --- kernel/sched/ghost.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index a39c8c9c1573..1a89f49fe8c4 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -938,7 +938,21 @@ static int validate_next_task(struct rq *rq, struct task_struct *next, { lockdep_assert_held(&rq->lock); - if (!task_has_ghost_policy(next) || next->ghost.agent) { + if (next->ghost.agent) { + set_txn_state(state, GHOST_TXN_INVALID_TARGET); + return -EINVAL; + } + + /* + * Task departed, but the agent hasn't yet processed the + * TASK_DEPARTED message. + */ + if (next->inhibit_task_msgs) { + set_txn_state(state, GHOST_TXN_TARGET_STALE); + return -EINVAL; + } + + if (!task_has_ghost_policy(next)) { set_txn_state(state, GHOST_TXN_INVALID_TARGET); return -EINVAL; } From 8d69b3db1d877457dfe44a6a814e5700d3c3ed92 Mon Sep 17 00:00:00 2001 From: Josh Don Date: Tue, 7 Sep 2021 11:28:04 -0700 Subject: [PATCH 056/165] sched/ghost: allow queue wakeup to be configured to empty It is useful to configure queue wakeup to an empty cpuset (ie. if we have a spinning/polling agent). Tested: Switching the 2LCE agent between global and per-core scheduling. Effort: sched/ghost Change-Id: I1db7fd2fc2e759d81471c34380325d460a31bf91 --- kernel/sched/ghost.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 1a89f49fe8c4..c1afa87a7931 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3589,14 +3589,14 @@ static int ghost_config_queue_wakeup(int qfd, int ninfo, int flags) { struct ghost_queue *q; - struct queue_notifier *qn, *old; + struct queue_notifier *old, *qn = NULL; struct ghost_agent_wakeup wakeup[GHOST_MAX_WINFO]; struct file *f; ulong fl; int cpu = -1, i; int ret = 0; - if (ninfo <= 0 || ninfo > GHOST_MAX_WINFO) + if (ninfo < 0 || ninfo > GHOST_MAX_WINFO) return -EINVAL; if (flags) @@ -3617,26 +3617,28 @@ static int ghost_config_queue_wakeup(int qfd, goto out_fput; } - if (copy_from_user(&wakeup, w, - sizeof(struct ghost_agent_wakeup) * ninfo)) { - ret = -EFAULT; - goto out_fput; - } + if (ninfo) { + if (copy_from_user(&wakeup, w, + sizeof(struct ghost_agent_wakeup) * ninfo)) { + ret = -EFAULT; + goto out_fput; + } - for (i = 0; i < ninfo; i++) { - /* cpu == -1 implies that it is polling for messages. */ - cpu = wakeup[i].cpu; + for (i = 0; i < ninfo; i++) { + /* cpu == -1 implies that it is polling for messages. */ + cpu = wakeup[i].cpu; - if (wakeup[i].prio || (cpu < -1) || (cpu >= nr_cpu_ids) || - !cpu_online(cpu)) { - ret = -EINVAL; - goto out_fput; + if (wakeup[i].prio || (cpu < -1) || + (cpu >= nr_cpu_ids) || !cpu_online(cpu)) { + ret = -EINVAL; + goto out_fput; + } } - } - qn = kzalloc(sizeof(struct queue_notifier), GFP_KERNEL); - memcpy(qn->winfo, wakeup, sizeof(qn->winfo)); - qn->wnum = ninfo; + qn = kzalloc(sizeof(struct queue_notifier), GFP_KERNEL); + memcpy(qn->winfo, wakeup, sizeof(qn->winfo)); + qn->wnum = ninfo; + } spin_lock_irqsave(&q->lock, fl); old = rcu_dereference_protected(q->notifier, lockdep_is_held(&q->lock)); From c180a76ee91b1a4bd081638740f9c8613a06fbe0 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Tue, 7 Sep 2021 17:19:19 -0700 Subject: [PATCH 057/165] sched: deprecate GHOST_NULL syscall All the ghost syscalls will become enclave ioctls. Deprecate the null syscall as part of that process, since it is only used for benchmarking against the null ioctl. The benchmark is modified in cl/395329521 and the numbers are discussed there. To be submitted with cl/395329521. Tested: //third_party/ghost:ioctl_test in virtme (cl/395329521) Effort: sched/ghost Change-Id: I05d0cd94b820b798346f81279aec03841dea0ec9 --- include/uapi/linux/ghost.h | 3 +-- kernel/sched/ghost.c | 18 ------------------ 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 586f6ffcfb45..2ca18e5035bd 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -30,7 +30,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 49 +#define GHOST_VERSION 50 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -345,7 +345,6 @@ struct ghost_ring { * 'ops' supported by gsys_ghost(). */ enum ghost_ops { - GHOST_NULL, GHOST_CREATE_QUEUE, GHOST_ASSOCIATE_QUEUE, GHOST_CONFIG_QUEUE_WAKEUP, diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index c1afa87a7931..59029e329325 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6205,22 +6205,6 @@ static int ghost_get_gtid(int64_t __user *out) return 0; } -/* Null syscall for benchmarking */ -static int ghost_null(bool lookup_e, int e_fd) -{ - struct ghost_enclave *e; - struct fd f_enc = {0}; - int ret = 0; - - if (!lookup_e) - return 0; - e = ghost_fdget_enclave(e_fd, &f_enc); - if (!e) - ret = -EBADF; - ghost_fdput_enclave(e, &f_enc); - return ret; -} - /* * TODO: Access to the ghost syscalls needs to be restricted, probably via a * capability. @@ -6238,8 +6222,6 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, return -EPERM; switch (op) { - case GHOST_NULL: - return ghost_null(arg1, arg2); case GHOST_CREATE_QUEUE: return ghost_create_queue(arg1, arg2, arg3, (ulong __user *)arg4, arg5); From 3cacc7ef4b035df92325e1b6b9a564c1fbf36d4a Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Thu, 9 Sep 2021 14:32:39 -0700 Subject: [PATCH 058/165] sched: use ioctls for GHOST_CREATE_QUEUE and GHOST_ASSOCIATE_QUEUE Deprecating all ghost syscalls in favor of enclave ioctls. To be submitted with cl/395976383. Tested: see last kcl in series Effort: sched/ghost Change-Id: Ia013fba97042ba71d3c2e550e463558628e44b47 --- include/uapi/linux/ghost.h | 21 +++++++++++-- kernel/sched/ghost.c | 64 ++++++++++++++++++-------------------- kernel/sched/ghostfs.c | 6 ++++ kernel/sched/sched.h | 3 ++ 4 files changed, 58 insertions(+), 36 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 2ca18e5035bd..d1625bae1801 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -30,7 +30,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 50 +#define GHOST_VERSION 51 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -92,9 +92,26 @@ struct ghost_ioc_sw_get_info { struct ghost_sw_info response; }; +struct ghost_ioc_create_queue { + int elems; + int node; + int flags; + ulong mapsize; +}; + +struct ghost_ioc_assoc_queue { + int fd; + struct ghost_msg_src src; + int barrier; + int flags; + int status; +}; + #define GHOST_IOC_NULL _IO('g', 0) #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) +#define GHOST_IOC_CREATE_QUEUE _IOWR('g', 3, struct ghost_ioc_create_queue *) +#define GHOST_IOC_ASSOC_QUEUE _IOWR('g', 4, struct ghost_ioc_assoc_queue *) /* * Status word region APIs. @@ -345,8 +362,6 @@ struct ghost_ring { * 'ops' supported by gsys_ghost(). */ enum ghost_ops { - GHOST_CREATE_QUEUE, - GHOST_ASSOCIATE_QUEUE, GHOST_CONFIG_QUEUE_WAKEUP, GHOST_SET_OPTION, GHOST_GET_CPU_TIME, diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 59029e329325..b2b8f9102c31 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3094,18 +3094,25 @@ static const struct file_operations queue_fops = { .mmap = queue_mmap, }; -static int ghost_create_queue(int elems, int node, int flags, - ulong __user *mapsize, int e_fd) +int ghost_create_queue(struct ghost_enclave *e, + struct ghost_ioc_create_queue __user *arg) { ulong size; - int error = 0, fd; + int error = 0, fd, elems, node, flags; struct ghost_queue *q; struct ghost_queue_header *h; - struct ghost_enclave *e; - struct fd f_enc = {0}; + struct ghost_ioc_create_queue create_queue; const int valid_flags = 0; /* no flags for now */ + if (copy_from_user(&create_queue, arg, + sizeof(struct ghost_ioc_create_queue))) + return -EFAULT; + + elems = create_queue.elems; + node = create_queue.node; + flags = create_queue.flags; + /* * Validate that 'head' and 'tail' are large enough to distinguish * between an empty and full queue. In other words when the queue @@ -3130,20 +3137,14 @@ static int ghost_create_queue(int elems, int node, int flags, size += elems * sizeof(struct ghost_msg); size = PAGE_ALIGN(size); - error = put_user(size, mapsize); + error = put_user(size, &arg->mapsize); if (error) return error; - e = ghost_fdget_enclave(e_fd, &f_enc); - if (!e) { - error = -EBADF; - goto err_enc_fd; - } - q = kzalloc_node(sizeof(struct ghost_queue), GFP_KERNEL, node); if (!q) { error = -ENOMEM; - goto err_alloc_queue; + return error; } spin_lock_init(&q->lock); @@ -3180,16 +3181,12 @@ static int ghost_create_queue(int elems, int node, int flags, q->enclave = e; INIT_WORK(&q->free_work, __queue_free_work); - ghost_fdput_enclave(e, &f_enc); return fd; err_getfd: vfree(q->addr); err_vmalloc: kfree(q); -err_alloc_queue: -err_enc_fd: - ghost_fdput_enclave(e, &f_enc); return error; } @@ -3383,8 +3380,7 @@ int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo) return error; } -static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, - int barrier, int flags, int *ustatus) +int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg) { int error = 0; struct rq *rq; @@ -3396,12 +3392,21 @@ static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, struct ghost_queue *oldq, *newq; int status = 0; - if (flags != 0) /* no flags for now */ - return -EINVAL; + int fd, barrier, flags; + struct ghost_ioc_assoc_queue assoc_queue; - if (copy_from_user(&src, usrc, sizeof(struct ghost_msg_src))) + if (copy_from_user(&assoc_queue, arg, + sizeof(struct ghost_ioc_assoc_queue))) return -EFAULT; + fd = assoc_queue.fd; + barrier = assoc_queue.barrier; + flags = assoc_queue.flags; + src = assoc_queue.src; + + if (flags != 0) /* no flags for now */ + return -EINVAL; + /* For now only allow changing task-to-queue association. */ if (src.type != GHOST_TASK) return -EINVAL; @@ -3479,11 +3484,6 @@ static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, status |= GHOST_ASSOC_SF_BRAND_NEW; } - if (ustatus && copy_to_user(ustatus, &status, sizeof(status))) { - error = -EFAULT; - goto done; - } - queue_incref(newq); p->ghost.dst_q = newq; if (oldq) @@ -3493,6 +3493,10 @@ static int ghost_associate_queue(int fd, struct ghost_msg_src __user *usrc, if (p) task_rq_unlock(rq, p, &rf); fput(file); + /* TODO(b/202070945) */ + if (!error) + error = put_user(status, &arg->status); + return error; } @@ -6222,12 +6226,6 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, return -EPERM; switch (op) { - case GHOST_CREATE_QUEUE: - return ghost_create_queue(arg1, arg2, arg3, - (ulong __user *)arg4, arg5); - case GHOST_ASSOCIATE_QUEUE: - return ghost_associate_queue(arg1, (void *)arg2, arg3, arg4, - (int __user *)arg5); case GHOST_SET_DEFAULT_QUEUE: return ghost_set_default_queue(arg1); case GHOST_CONFIG_QUEUE_WAKEUP: diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 52974535d4d4..598884a1b8b0 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -494,6 +494,12 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, (struct ghost_ioc_sw_get_info __user *)arg); case GHOST_IOC_SW_FREE: return ghost_sw_free(e, (void __user *)arg); + case GHOST_IOC_CREATE_QUEUE: + return ghost_create_queue(e, + (struct ghost_ioc_create_queue __user *)arg); + case GHOST_IOC_ASSOC_QUEUE: + return ghost_associate_queue( + (struct ghost_ioc_assoc_queue __user *)arg); } return -ENOIOCTLCMD; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a624379ee4e0..cc842cd4a7ec 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -238,6 +238,9 @@ extern int ghost_sw_get_info(struct ghost_enclave *e, struct ghost_ioc_sw_get_info __user *arg); extern int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo); +extern int ghost_create_queue(struct ghost_enclave *e, + struct ghost_ioc_create_queue __user *arg); +extern int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); From 61315a0c17469d5a7f0233880a37137542c844c0 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Tue, 14 Sep 2021 12:13:38 -0700 Subject: [PATCH 059/165] sched: use ioctls for GHOST_SET_DEFAULT_QUEUE and GHOST_CONFIG_QUEUE_WAKEUP Deprecating all ghost syscalls in favor of enclave ioctls. To be submitted with cl/396859710. Tested: see last kcl in series Effort: sched/ghost Change-Id: Ie5eb7edcac4faa21c5854b702995d5791eb44cc5 --- include/uapi/linux/ghost.h | 17 +++++++++--- kernel/sched/ghost.c | 55 +++++++++++++++++++++++++++----------- kernel/sched/ghostfs.c | 6 +++++ kernel/sched/sched.h | 4 +++ 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index d1625bae1801..e16f14d2fc6a 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -30,7 +30,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 51 +#define GHOST_VERSION 52 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -107,11 +107,24 @@ struct ghost_ioc_assoc_queue { int status; }; +struct ghost_ioc_set_default_queue { + int fd; +}; + +struct ghost_ioc_config_queue_wakeup { + int qfd; + struct ghost_agent_wakeup *w; + int ninfo; + int flags; +}; + #define GHOST_IOC_NULL _IO('g', 0) #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) #define GHOST_IOC_CREATE_QUEUE _IOWR('g', 3, struct ghost_ioc_create_queue *) #define GHOST_IOC_ASSOC_QUEUE _IOWR('g', 4, struct ghost_ioc_assoc_queue *) +#define GHOST_IOC_SET_DEFAULT_QUEUE _IOW('g', 5, struct ghost_ioc_set_default_queue *) +#define GHOST_IOC_CONFIG_QUEUE_WAKEUP _IOW('g', 6, struct ghost_ioc_config_queue_wakeup *) /* * Status word region APIs. @@ -362,7 +375,6 @@ struct ghost_ring { * 'ops' supported by gsys_ghost(). */ enum ghost_ops { - GHOST_CONFIG_QUEUE_WAKEUP, GHOST_SET_OPTION, GHOST_GET_CPU_TIME, GHOST_COMMIT_TXN, @@ -371,7 +383,6 @@ enum ghost_ops { GHOST_GTID_LOOKUP, GHOST_GET_GTID_10, /* TODO: deprecate */ GHOST_GET_GTID_11, /* TODO: deprecate */ - GHOST_SET_DEFAULT_QUEUE, }; /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b2b8f9102c31..4ba4a4618b5d 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3500,21 +3500,41 @@ int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg) return error; } -static int ghost_set_default_queue(int qfd) +int ghost_set_default_queue(struct ghost_enclave *e, + struct ghost_ioc_set_default_queue __user *arg) { + int error = 0; struct fd f_que = {0}; struct ghost_queue *newq; + int qfd; + struct ghost_ioc_set_default_queue set_queue; + + if (copy_from_user(&set_queue, arg, + sizeof(struct ghost_ioc_set_default_queue))) + return -EFAULT; + + qfd = set_queue.fd; + f_que = fdget(qfd); newq = fd_to_queue(f_que); if (!newq) { - fdput(f_que); - return -EBADF; + error = -EBADF; + goto err_newq; + } + /* + * Make sure that 'newq' belongs to the same enclave as the one + * whose default queue we are updating. + */ + if (newq->enclave != e) { + error = -EINVAL; + goto err_newq; } - /* The implied target enclave is whichever newq belongs to. */ - enclave_set_default_queue(newq->enclave, newq); + enclave_set_default_queue(e, newq); + +err_newq: fdput(f_que); - return 0; + return error; } /* @@ -3588,9 +3608,7 @@ static int agent_target_cpu(struct rq *rq) return target_cpu(agent->ghost.dst_q, task_cpu(agent)); } -static int ghost_config_queue_wakeup(int qfd, - struct ghost_agent_wakeup __user *w, - int ninfo, int flags) +int ghost_config_queue_wakeup(struct ghost_ioc_config_queue_wakeup __user *arg) { struct ghost_queue *q; struct queue_notifier *old, *qn = NULL; @@ -3600,6 +3618,19 @@ static int ghost_config_queue_wakeup(int qfd, int cpu = -1, i; int ret = 0; + int qfd, ninfo, flags; + struct ghost_agent_wakeup *w; + struct ghost_ioc_config_queue_wakeup config_wakeup; + + if (copy_from_user(&config_wakeup, arg, + sizeof(struct ghost_ioc_config_queue_wakeup))) + return -EFAULT; + + qfd = config_wakeup.qfd; + w = config_wakeup.w; + ninfo = config_wakeup.ninfo; + flags = config_wakeup.flags; + if (ninfo < 0 || ninfo > GHOST_MAX_WINFO) return -EINVAL; @@ -6226,12 +6257,6 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, return -EPERM; switch (op) { - case GHOST_SET_DEFAULT_QUEUE: - return ghost_set_default_queue(arg1); - case GHOST_CONFIG_QUEUE_WAKEUP: - return ghost_config_queue_wakeup(arg1, - (struct ghost_agent_wakeup __user *) arg2, - arg3, arg4); case GHOST_SET_OPTION: return ghost_set_option(arg1, arg2, arg3, arg4); case GHOST_GET_CPU_TIME: diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 598884a1b8b0..2ff3c56b0736 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -500,6 +500,12 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, case GHOST_IOC_ASSOC_QUEUE: return ghost_associate_queue( (struct ghost_ioc_assoc_queue __user *)arg); + case GHOST_IOC_SET_DEFAULT_QUEUE: + return ghost_set_default_queue(e, + (struct ghost_ioc_set_default_queue __user *)arg); + case GHOST_IOC_CONFIG_QUEUE_WAKEUP: + return ghost_config_queue_wakeup( + (struct ghost_ioc_config_queue_wakeup __user *)arg); } return -ENOIOCTLCMD; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index cc842cd4a7ec..7d12a8ae5f7e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -241,6 +241,10 @@ extern int ghost_sw_free(struct ghost_enclave *e, extern int ghost_create_queue(struct ghost_enclave *e, struct ghost_ioc_create_queue __user *arg); extern int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg); +extern int ghost_set_default_queue(struct ghost_enclave *e, + struct ghost_ioc_set_default_queue __user *arg); +extern int ghost_config_queue_wakeup( + struct ghost_ioc_config_queue_wakeup __user *arg); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); From a53baec928f62bbc1dbe203ca49b34b879cb3f4b Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 15 Sep 2021 10:14:24 -0700 Subject: [PATCH 060/165] sched: use an ioctl for GHOST_GET_CPU_TIME and deprecate unused syscalls (GHOST_SET_OPTION, GHOST_GET_GTID_10, GHOST_GET_GTID_11) Deprecating all ghost syscalls in favor of enclave ioctls. To be submitted with cl/396933039. Tested: see last kcl in series Effort: sched/ghost Change-Id: I9a51706c410421e3c8f22fb7af3ef7b7054655f4 --- include/uapi/linux/ghost.h | 12 +++++++----- kernel/sched/ghost.c | 27 +++++++-------------------- kernel/sched/ghostfs.c | 3 +++ kernel/sched/sched.h | 1 + 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index e16f14d2fc6a..85a6b17d03dd 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -30,7 +30,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 52 +#define GHOST_VERSION 53 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -118,6 +118,11 @@ struct ghost_ioc_config_queue_wakeup { int flags; }; +struct ghost_ioc_get_cpu_time { + int64_t gtid; + uint64_t runtime; +}; + #define GHOST_IOC_NULL _IO('g', 0) #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) @@ -125,6 +130,7 @@ struct ghost_ioc_config_queue_wakeup { #define GHOST_IOC_ASSOC_QUEUE _IOWR('g', 4, struct ghost_ioc_assoc_queue *) #define GHOST_IOC_SET_DEFAULT_QUEUE _IOW('g', 5, struct ghost_ioc_set_default_queue *) #define GHOST_IOC_CONFIG_QUEUE_WAKEUP _IOW('g', 6, struct ghost_ioc_config_queue_wakeup *) +#define GHOST_IOC_GET_CPU_TIME _IOWR('g', 7, struct ghost_ioc_get_cpu_time *) /* * Status word region APIs. @@ -375,14 +381,10 @@ struct ghost_ring { * 'ops' supported by gsys_ghost(). */ enum ghost_ops { - GHOST_SET_OPTION, - GHOST_GET_CPU_TIME, GHOST_COMMIT_TXN, GHOST_SYNC_GROUP_TXN, GHOST_TIMERFD_SETTIME, GHOST_GTID_LOOKUP, - GHOST_GET_GTID_10, /* TODO: deprecate */ - GHOST_GET_GTID_11, /* TODO: deprecate */ }; /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 4ba4a4618b5d..39eda91e51d4 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3690,20 +3690,14 @@ int ghost_config_queue_wakeup(struct ghost_ioc_config_queue_wakeup __user *arg) return ret; } -static int ghost_set_option(int option, ulong val1, ulong val2, ulong val3) -{ - switch (option) { - default: - return -EINVAL; - } - - return 0; -} - -static int ghost_get_cpu_time(gtid_t gtid, u64 __user *uruntime) +int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg) { struct task_struct *p; u64 time; + gtid_t gtid; + + if (copy_from_user(>id, &arg->gtid, sizeof(gtid_t))) + return -EFAULT; rcu_read_lock(); p = find_task_by_gtid(gtid); @@ -3713,7 +3707,7 @@ static int ghost_get_cpu_time(gtid_t gtid, u64 __user *uruntime) } time = task_sched_runtime(p); rcu_read_unlock(); - if (copy_to_user(uruntime, &time, sizeof(*uruntime))) + if (copy_to_user(&arg->runtime, &time, sizeof(u64))) return -EFAULT; return 0; } @@ -6249,18 +6243,13 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, { bool be_nice = true; - if (op == GHOST_GET_GTID_10 || op == GHOST_GET_GTID_11 || - op == GHOST_BASE_GET_GTID) + if (op == GHOST_BASE_GET_GTID) be_nice = false; if (be_nice && !capable(CAP_SYS_NICE)) return -EPERM; switch (op) { - case GHOST_SET_OPTION: - return ghost_set_option(arg1, arg2, arg3, arg4); - case GHOST_GET_CPU_TIME: - return ghost_get_cpu_time((gtid_t)arg1, (u64 __user *)arg2); case GHOST_COMMIT_TXN: return gsys_ghost_commit_txn((ulong __user *)arg1, arg2, arg3); case GHOST_SYNC_GROUP_TXN: @@ -6273,8 +6262,6 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, case GHOST_GTID_LOOKUP: return ghost_gtid_lookup(arg1, arg2, arg3, (int64_t __user *)arg4); - case GHOST_GET_GTID_10: - case GHOST_GET_GTID_11: case GHOST_BASE_GET_GTID: return ghost_get_gtid((int64_t __user *)arg1); default: diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 2ff3c56b0736..130a89b6d48a 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -506,6 +506,9 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, case GHOST_IOC_CONFIG_QUEUE_WAKEUP: return ghost_config_queue_wakeup( (struct ghost_ioc_config_queue_wakeup __user *)arg); + case GHOST_IOC_GET_CPU_TIME: + return ghost_get_cpu_time( + (struct ghost_ioc_get_cpu_time __user *)arg); } return -ENOIOCTLCMD; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 7d12a8ae5f7e..01dc65b850f9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -245,6 +245,7 @@ extern int ghost_set_default_queue(struct ghost_enclave *e, struct ghost_ioc_set_default_queue __user *arg); extern int ghost_config_queue_wakeup( struct ghost_ioc_config_queue_wakeup __user *arg); +extern int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); From 6836153b09532665eb0353af6367c64b9050f9f2 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 15 Sep 2021 16:49:12 -0700 Subject: [PATCH 061/165] sched: use ioctls for GHOST_COMMIT_TXN and GHOST_SYNC_GROUP_TXN Deprecating all ghost syscalls in favor of enclave ioctls. To be submitted with cl/397221332. Tested: see last kcl in series Effort: sched/ghost Change-Id: I581735c6b53812095191f2c997a77a9149489cf7 --- include/uapi/linux/ghost.h | 17 ++++++++++++++--- kernel/sched/ghost.c | 38 ++++++++++++++++++++++++++++++-------- kernel/sched/ghostfs.c | 6 ++++++ kernel/sched/sched.h | 4 ++++ 4 files changed, 54 insertions(+), 11 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 85a6b17d03dd..96045889c953 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -20,6 +20,7 @@ #include #else #include +#include #include #endif @@ -30,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 53 +#define GHOST_VERSION 54 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -123,6 +124,16 @@ struct ghost_ioc_get_cpu_time { uint64_t runtime; }; +struct ghost_ioc_commit_txn { +#ifdef __KERNEL__ + ulong *mask_ptr; +#else + cpu_set_t *mask_ptr; +#endif + uint32_t mask_len; + int flags; +}; + #define GHOST_IOC_NULL _IO('g', 0) #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) @@ -131,6 +142,8 @@ struct ghost_ioc_get_cpu_time { #define GHOST_IOC_SET_DEFAULT_QUEUE _IOW('g', 5, struct ghost_ioc_set_default_queue *) #define GHOST_IOC_CONFIG_QUEUE_WAKEUP _IOW('g', 6, struct ghost_ioc_config_queue_wakeup *) #define GHOST_IOC_GET_CPU_TIME _IOWR('g', 7, struct ghost_ioc_get_cpu_time *) +#define GHOST_IOC_COMMIT_TXN _IOW('g', 8, struct ghost_ioc_commit_txn *) +#define GHOST_IOC_SYNC_GROUP_TXN _IOW('g', 9, struct ghost_ioc_commit_txn *) /* * Status word region APIs. @@ -381,8 +394,6 @@ struct ghost_ring { * 'ops' supported by gsys_ghost(). */ enum ghost_ops { - GHOST_COMMIT_TXN, - GHOST_SYNC_GROUP_TXN, GHOST_TIMERFD_SETTIME, GHOST_GTID_LOOKUP, }; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 39eda91e51d4..942637a432af 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -5786,8 +5786,8 @@ static inline int64_t ghost_sync_group_cookie(void) return val; } -static int gsys_ghost_sync_group(ulong __user *user_mask_ptr, - uint user_mask_len, int flags) +int ghost_sync_group(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg) { int64_t target; bool failed = false; @@ -5795,8 +5795,21 @@ static int gsys_ghost_sync_group(ulong __user *user_mask_ptr, int cpu, this_cpu, error, state; cpumask_var_t cpumask, ipimask, rendmask; + ulong *user_mask_ptr; + uint user_mask_len; + int flags; + struct ghost_ioc_commit_txn commit_txn; + const int valid_flags = 0; + if (copy_from_user(&commit_txn, arg, + sizeof(struct ghost_ioc_commit_txn))) + return -EFAULT; + + user_mask_ptr = commit_txn.mask_ptr; + user_mask_len = commit_txn.mask_len; + flags = commit_txn.flags; + if (flags & ~valid_flags) return -EINVAL; @@ -5984,8 +5997,8 @@ static int gsys_ghost_sync_group(ulong __user *user_mask_ptr, return !failed; } -static int gsys_ghost_commit_txn(ulong __user *user_mask_ptr, - uint user_mask_len, int flags) +int ioctl_ghost_commit_txn(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg) { int error, state; int cpu, this_cpu; @@ -5993,8 +6006,21 @@ static int gsys_ghost_commit_txn(ulong __user *user_mask_ptr, bool local_resched = false; cpumask_var_t cpumask, ipimask; + ulong *user_mask_ptr; + uint user_mask_len; + int flags; + struct ghost_ioc_commit_txn commit_txn; + const int valid_flags = 0; + if (copy_from_user(&commit_txn, arg, + sizeof(struct ghost_ioc_commit_txn))) + return -EFAULT; + + user_mask_ptr = commit_txn.mask_ptr; + user_mask_len = commit_txn.mask_len; + flags = commit_txn.flags; + if (flags & ~valid_flags) return -EINVAL; @@ -6250,10 +6276,6 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, return -EPERM; switch (op) { - case GHOST_COMMIT_TXN: - return gsys_ghost_commit_txn((ulong __user *)arg1, arg2, arg3); - case GHOST_SYNC_GROUP_TXN: - return gsys_ghost_sync_group((ulong __user *)arg1, arg2, arg3); case GHOST_TIMERFD_SETTIME: return ghost_timerfd_settime(arg1, arg2, (const struct __kernel_itimerspec __user *)arg3, diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 130a89b6d48a..a975741f5e1c 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -509,6 +509,12 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, case GHOST_IOC_GET_CPU_TIME: return ghost_get_cpu_time( (struct ghost_ioc_get_cpu_time __user *)arg); + case GHOST_IOC_COMMIT_TXN: + return ioctl_ghost_commit_txn(e, + (struct ghost_ioc_commit_txn __user *)arg); + case GHOST_IOC_SYNC_GROUP_TXN: + return ghost_sync_group(e, + (struct ghost_ioc_commit_txn __user *)arg); } return -ENOIOCTLCMD; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 01dc65b850f9..403ba5bc073b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -246,6 +246,10 @@ extern int ghost_set_default_queue(struct ghost_enclave *e, extern int ghost_config_queue_wakeup( struct ghost_ioc_config_queue_wakeup __user *arg); extern int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg); +extern int ioctl_ghost_commit_txn(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg); +extern int ghost_sync_group(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); From 85524e3ec4324c9cff30817bcaff45ebd1dfaf07 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Fri, 17 Sep 2021 13:11:29 -0700 Subject: [PATCH 062/165] sched: use an ioctl for GHOST_TIMERFD_SETTIME Deprecating all ghost syscalls in favor of enclave ioctls. Note: gtid-related syscalls are called from 'base' and can't be converted to an enclave ioctl; they will be removed when gtid is put in /proc. To be submitted with cl/397392218. Tested: 1) all unit tests pass in virtme 2) gtests 3) gtests with debug kernel: no warnings in dmesg 4) SLL: compared performance before and after 5) VIT: compared performance before and after Effort: sched/ghost Change-Id: I938a0c6d613a3d55cc9bc5ca53ba67f6ff76d16f --- include/uapi/linux/ghost.h | 33 +++++++++++++++++++++++---------- kernel/sched/ghost.c | 28 ++++++++++++++++------------ kernel/sched/ghostfs.c | 3 +++ kernel/sched/sched.h | 1 + 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 96045889c953..7b086365cd53 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 54 +#define GHOST_VERSION 55 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -83,6 +83,14 @@ struct ghost_msg_src { }; #define GHOST_THIS_CPU -1 +/* GHOST_TIMERFD_SETTIME */ +struct timerfd_ghost { + int cpu; + int flags; + uint64_t cookie; +}; +#define TIMERFD_GHOST_ENABLED (1 << 0) + struct ghost_sw_info { uint32_t id; /* status_word region id */ uint32_t index; /* index into the status_word array */ @@ -134,6 +142,19 @@ struct ghost_ioc_commit_txn { int flags; }; +struct ghost_ioc_timerfd_settime { + int timerfd; + int flags; +#ifdef __KERNEL__ + struct __kernel_itimerspec *in_tmr; + struct __kernel_itimerspec *out_tmr; +#else + struct itimerspec *in_tmr; + struct itimerspec *out_tmr; +#endif + struct timerfd_ghost timerfd_ghost; +}; + #define GHOST_IOC_NULL _IO('g', 0) #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) @@ -144,6 +165,7 @@ struct ghost_ioc_commit_txn { #define GHOST_IOC_GET_CPU_TIME _IOWR('g', 7, struct ghost_ioc_get_cpu_time *) #define GHOST_IOC_COMMIT_TXN _IOW('g', 8, struct ghost_ioc_commit_txn *) #define GHOST_IOC_SYNC_GROUP_TXN _IOW('g', 9, struct ghost_ioc_commit_txn *) +#define GHOST_IOC_TIMERFD_SETTIME _IOWR('g', 10, struct ghost_ioc_timerfd_settime *) /* * Status word region APIs. @@ -394,7 +416,6 @@ struct ghost_ring { * 'ops' supported by gsys_ghost(). */ enum ghost_ops { - GHOST_TIMERFD_SETTIME, GHOST_GTID_LOOKUP, }; @@ -548,14 +569,6 @@ struct ghost_cpu_data { #define GHOST_TXN_VERSION 0 -/* GHOST_TIMERFD_SETTIME */ -struct timerfd_ghost { - int cpu; - int flags; - uint64_t cookie; -}; -#define TIMERFD_GHOST_ENABLED (1 << 0) - /* GHOST_GTID_LOOKUP */ enum { GHOST_GTID_LOOKUP_TGID, /* return group_leader pid */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 942637a432af..dcbe39f81fa0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6168,19 +6168,28 @@ static int ghost_timerfd_validate(struct timerfd_ghost *timerfd_ghost) int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, struct itimerspec64 *old, struct timerfd_ghost *tfdl); -static int ghost_timerfd_settime(int timerfd, int flags, - const struct __kernel_itimerspec __user *utmr, - struct __kernel_itimerspec __user *otmr, - struct timerfd_ghost __user *utfdl) +int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) { - struct timerfd_ghost timerfd_ghost; struct itimerspec64 new, old; int ret; - if (get_itimerspec64(&new, utmr)) + int timerfd, flags; + const struct __kernel_itimerspec *itmr; + struct __kernel_itimerspec *otmr; + struct timerfd_ghost timerfd_ghost; + struct ghost_ioc_timerfd_settime settime_data; + + if (copy_from_user(&settime_data, arg, + sizeof(struct ghost_ioc_timerfd_settime))) return -EFAULT; - if (copy_from_user(&timerfd_ghost, utfdl, sizeof(timerfd_ghost))) + timerfd = settime_data.timerfd; + flags = settime_data.flags; + itmr = settime_data.in_tmr; + otmr = settime_data.out_tmr; + timerfd_ghost = settime_data.timerfd_ghost; + + if (get_itimerspec64(&new, itmr)) return -EFAULT; ret = ghost_timerfd_validate(&timerfd_ghost); @@ -6276,11 +6285,6 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, return -EPERM; switch (op) { - case GHOST_TIMERFD_SETTIME: - return ghost_timerfd_settime(arg1, arg2, - (const struct __kernel_itimerspec __user *)arg3, - (struct __kernel_itimerspec __user *)arg4, - (struct timerfd_ghost __user *)arg5); case GHOST_GTID_LOOKUP: return ghost_gtid_lookup(arg1, arg2, arg3, (int64_t __user *)arg4); diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index a975741f5e1c..34eb3c896d03 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -515,6 +515,9 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, case GHOST_IOC_SYNC_GROUP_TXN: return ghost_sync_group(e, (struct ghost_ioc_commit_txn __user *)arg); + case GHOST_IOC_TIMERFD_SETTIME: + return ghost_timerfd_settime( + (struct ghost_ioc_timerfd_settime __user *)arg); } return -ENOIOCTLCMD; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 403ba5bc073b..c025136f6d39 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -250,6 +250,7 @@ extern int ioctl_ghost_commit_txn(struct ghost_enclave *e, struct ghost_ioc_commit_txn __user *arg); extern int ghost_sync_group(struct ghost_enclave *e, struct ghost_ioc_commit_txn __user *arg); +extern int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); From 64f3996b1d794c4adeba3518dc88a0e31d6d6868 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 29 Sep 2021 14:48:43 -0700 Subject: [PATCH 063/165] sched: don't pass pointer to ghost ioctls The third argument is used to encode the size of the ioctl-specific argument. Remove the pointer modifier from the types passed to the ioctl macros. See comment discussion in kcl/423609. To be submitted with cl/399803634. Tested: see last ioctl kcl (kcl/424873) Effort: sched/ghost Change-Id: I081988e47d533678a3071dfc0ec66cd83aabaabf --- include/uapi/linux/ghost.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 7b086365cd53..6dbd9d8cd934 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -156,16 +156,16 @@ struct ghost_ioc_timerfd_settime { }; #define GHOST_IOC_NULL _IO('g', 0) -#define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info *) -#define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info *) -#define GHOST_IOC_CREATE_QUEUE _IOWR('g', 3, struct ghost_ioc_create_queue *) -#define GHOST_IOC_ASSOC_QUEUE _IOWR('g', 4, struct ghost_ioc_assoc_queue *) -#define GHOST_IOC_SET_DEFAULT_QUEUE _IOW('g', 5, struct ghost_ioc_set_default_queue *) -#define GHOST_IOC_CONFIG_QUEUE_WAKEUP _IOW('g', 6, struct ghost_ioc_config_queue_wakeup *) -#define GHOST_IOC_GET_CPU_TIME _IOWR('g', 7, struct ghost_ioc_get_cpu_time *) -#define GHOST_IOC_COMMIT_TXN _IOW('g', 8, struct ghost_ioc_commit_txn *) -#define GHOST_IOC_SYNC_GROUP_TXN _IOW('g', 9, struct ghost_ioc_commit_txn *) -#define GHOST_IOC_TIMERFD_SETTIME _IOWR('g', 10, struct ghost_ioc_timerfd_settime *) +#define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info) +#define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info) +#define GHOST_IOC_CREATE_QUEUE _IOWR('g', 3, struct ghost_ioc_create_queue) +#define GHOST_IOC_ASSOC_QUEUE _IOWR('g', 4, struct ghost_ioc_assoc_queue) +#define GHOST_IOC_SET_DEFAULT_QUEUE _IOW('g', 5, struct ghost_ioc_set_default_queue) +#define GHOST_IOC_CONFIG_QUEUE_WAKEUP _IOW('g', 6, struct ghost_ioc_config_queue_wakeup) +#define GHOST_IOC_GET_CPU_TIME _IOWR('g', 7, struct ghost_ioc_get_cpu_time) +#define GHOST_IOC_COMMIT_TXN _IOW('g', 8, struct ghost_ioc_commit_txn) +#define GHOST_IOC_SYNC_GROUP_TXN _IOW('g', 9, struct ghost_ioc_commit_txn) +#define GHOST_IOC_TIMERFD_SETTIME _IOWR('g', 10, struct ghost_ioc_timerfd_settime) /* * Status word region APIs. From 6d839789a01fafbf70917f52253b45af83cdaf52 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Tue, 12 Oct 2021 15:39:59 -0700 Subject: [PATCH 064/165] sched: add gtid to /proc Add a "/ghost" subdirectory to /proc/ with a file containing the task's gtid. This will eventually replace the syscall used to obtain the gtid. To be submitted with cl/402677877. Tested: //third_party/ghost and //prodkernel/ghost unit tests in virtme Effort: sched/ghost Change-Id: I6738d296439d433fa3f2470fde4f03ad8cd5d7a3 --- fs/proc/base.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/fs/proc/base.c b/fs/proc/base.c index b3422cda2a91..2a52d9885747 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3156,6 +3156,11 @@ static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns, static const struct file_operations proc_task_operations; static const struct inode_operations proc_task_inode_operations; +#ifdef CONFIG_SCHED_CLASS_GHOST +static const struct file_operations proc_ghost_dir_operations; +static const struct inode_operations proc_ghost_dir_inode_operations; +#endif + static const struct pid_entry tgid_base_stuff[] = { DIR("task", S_IRUGO|S_IXUGO, proc_task_inode_operations, proc_task_operations), DIR("fd", S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations), @@ -3266,6 +3271,10 @@ static const struct pid_entry tgid_base_stuff[] = { #ifdef CONFIG_SECCOMP_CACHE_DEBUG ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache), #endif +#ifdef CONFIG_SCHED_CLASS_GHOST + DIR("ghost", S_IRUGO|S_IXUGO, proc_ghost_dir_inode_operations, + proc_ghost_dir_operations), +#endif }; static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx) @@ -3831,3 +3840,58 @@ void __init set_proc_pid_nlink(void) nlink_tid = pid_entry_nlink(tid_base_stuff, ARRAY_SIZE(tid_base_stuff)); nlink_tgid = pid_entry_nlink(tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff)); } + +#ifdef CONFIG_SCHED_CLASS_GHOST + +static ssize_t proc_ghost_gtid_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[32]; + size_t len; + + if (!task) + return -ESRCH; + + len = snprintf(buffer, sizeof(buffer), "%ld\n", + task->gtid); + WARN_ON_ONCE(len >= sizeof(buffer)); + put_task_struct(task); + return simple_read_from_buffer(buf, count, ppos, buffer, len); +} + +static const struct file_operations proc_ghost_gtid_operations = { + .read = proc_ghost_gtid_read, + .llseek = generic_file_llseek, +}; + +static const struct pid_entry ghost_dir_stuff[] = { + REG("gtid", S_IRUGO, proc_ghost_gtid_operations), +}; + +static struct dentry *proc_ghost_dir_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + return proc_pident_lookup(dir, dentry, ghost_dir_stuff, + ghost_dir_stuff + ARRAY_SIZE(ghost_dir_stuff)); +} + +static int proc_ghost_dir_readdir(struct file *file, struct dir_context *ctx) +{ + return proc_pident_readdir(file, ctx, ghost_dir_stuff, + ARRAY_SIZE(ghost_dir_stuff)); +} + +static const struct inode_operations proc_ghost_dir_inode_operations = { + .lookup = proc_ghost_dir_lookup, + .getattr = pid_getattr, +}; + +static const struct file_operations proc_ghost_dir_operations = { + .read = generic_read_dir, + .iterate_shared = proc_ghost_dir_readdir, + .llseek = default_llseek, +}; + +#endif From 1ac2bdac4f923d2bd88a5d10de4135d42d5402e2 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 17 Oct 2021 17:05:48 -0700 Subject: [PATCH 065/165] sched: get rid of usused GHOST_MAX_SW_REGIONS and sw_region_ids[] Tested: builds Effort: sched/ghost Change-Id: Ia18891596bb905e045944eb5f8daee263bb5511a --- kernel/sched/sched.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c025136f6d39..1d00bf42b46d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -149,7 +149,6 @@ struct ghost_sw_region { struct ghost_enclave *enclave; }; -#define GHOST_MAX_SW_REGIONS 64 #define GHOST_CPU_DATA_REGION_SIZE \ (sizeof(struct ghost_cpu_data) * num_possible_cpus()) @@ -177,7 +176,6 @@ struct ghost_enclave { spinlock_t lock; struct kref kref; struct list_head sw_region_list; - ulong sw_region_ids[BITS_TO_LONGS(GHOST_MAX_SW_REGIONS)]; struct ghost_cpu_data **cpu_data; struct cpumask cpus; From 0b1012d9e291e3c7296925aebe3d7377cf4963bf Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 19 Oct 2021 09:04:56 -0700 Subject: [PATCH 066/165] sched: add an empty include/linux/ghost.h Empty header to satisfy inclusions like which if this file were not present would end up resolving to We care about this because including has abi implications so we want it to be included explicitly. Tested: builds Effort: sched/ghost/abi Change-Id: I0f97373d7a288caae0eaadca81a4968c843a6d6e --- fs/timerfd.c | 3 +-- include/linux/ghost.h | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 include/linux/ghost.h diff --git a/fs/timerfd.c b/fs/timerfd.c index d3e0f0733dc1..33339793b414 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -29,8 +29,7 @@ #include #ifdef CONFIG_SCHED_CLASS_GHOST -#include -#include +#include #endif struct timerfd_ctx { diff --git a/include/linux/ghost.h b/include/linux/ghost.h new file mode 100644 index 000000000000..527b514a22f7 --- /dev/null +++ b/include/linux/ghost.h @@ -0,0 +1,13 @@ +#ifndef _LINUX_GHOST_H_ +#define _LINUX_GHOST_H_ + +/* + * Empty header to satisfy inclusions like which if this + * file were not present would end up resolving to + * + * We care about this because including has abi + * implications so we want this header to be included explicitly. + */ + +#endif /* _LINUX_GHOST_H_ */ + From 79821de6aafec2de186545b8983bbe54f19fcdc2 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 19 Oct 2021 10:08:14 -0700 Subject: [PATCH 067/165] sched: don't include ghost uapi header from The header file can be included without any restrictions from any source or header file in the linux kernel. As such this makes keeping track of ghost UAPI dependencies difficult. While here add CONFIG_SCHED_CLASS_GHOST ifdef guards to bpf_types.h. Tested: builds Effort: sched/ghost/abi Change-Id: I7a0b77342e9d627097704fb7a92a8859fad4460f --- include/linux/bpf_types.h | 3 ++- include/linux/sched.h | 1 - kernel/bpf/btf.c | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 058a489cc735..589a5d89a9b0 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -77,11 +77,12 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm, void *, void *) #endif /* CONFIG_BPF_LSM */ #endif +#ifdef CONFIG_SCHED_CLASS_GHOST BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_SCHED, ghost_sched, struct bpf_ghost_sched, struct bpf_ghost_sched_kern) BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_MSG, ghost_msg, struct bpf_ghost_msg, struct bpf_ghost_msg_kern) - +#endif BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_PROG_ARRAY, prog_array_map_ops) diff --git a/include/linux/sched.h b/include/linux/sched.h index 12a5ebd9aa1c..40296ba2a13d 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -8,7 +8,6 @@ */ #include -#include #include diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 84a36ee4a4c2..0d1fcc699741 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -26,6 +26,10 @@ #include #include +#ifdef CONFIG_SCHED_CLASS_GHOST +#include +#endif + /* BTF (BPF Type Format) is the meta data format which describes * the data types of BPF program/map. Hence, it basically focus * on the C programming language which the modern BPF is primary From 178d442a8073da80aa727274731c66076ac6ddf6 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 19 Oct 2021 09:09:35 -0700 Subject: [PATCH 068/165] sched: move ghost status_word helper functions into the uapi header The status_word is part of the ABI and thus these helper functions are also tied to a specific ABI. Tested: builds Effort: sched/ghost/abi Change-Id: Ie9c4d3ee7ee1bf6fd3a3a3db5b76bad0bb9cd651 --- include/uapi/linux/ghost.h | 28 ++++++++++++++++++++++++++++ kernel/sched/sched.h | 21 --------------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 6dbd9d8cd934..672f2434b9ea 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -208,6 +208,34 @@ struct ghost_status_word { #define GHOST_SW_TASK_MSG_GATED (1U << 19) /* all task msgs are gated until * status_word is freed by agent */ +#ifdef __KERNEL__ +static inline void ghost_sw_set_flag(struct ghost_status_word *sw, + uint32_t flag) +{ + /* Pairs with an acquire load in StatusWord::sw_flags() in the agent */ + smp_store_release(&sw->flags, sw->flags | flag); +} + +static inline void ghost_sw_clear_flag(struct ghost_status_word *sw, + uint32_t flag) +{ + /* Pairs with an acquire load in StatusWord::sw_flags() in the agent */ + smp_store_release(&sw->flags, sw->flags & ~flag); +} + +static inline void ghost_sw_set_time(struct ghost_status_word *sw, + s64 time) +{ + /* + * Do a relaxed store since userspace syncs with the release store to + * `sw->flags` for setting the oncpu bit in `ghost_sw_set_flag`. We set + * the time in this function before setting the oncpu bit, so we use + * that release store as a barrier. + */ + WRITE_ONCE(sw->switch_time, time); +} +#endif /* __KERNEL__ */ + /* * Queue APIs. */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1d00bf42b46d..f860e0c78347 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -314,27 +314,6 @@ static inline void sched_ghost_entity_init(struct task_struct *p) INIT_LIST_HEAD(&p->ghost.task_list); } -static inline void ghost_sw_set_flag(struct ghost_status_word *sw, - uint32_t flag) { - smp_store_release(&sw->flags, sw->flags | flag); -} - -static inline void ghost_sw_clear_flag(struct ghost_status_word *sw, - uint32_t flag) { - smp_store_release(&sw->flags, sw->flags & ~flag); -} - -static inline void ghost_sw_set_time(struct ghost_status_word *sw, - s64 time) { - /* - * Do a relaxed store since userspace syncs with the release store to - * `sw->flags` for setting the oncpu bit in `ghost_sw_set_flag`. We set - * the time in this function before setting the oncpu bit, so we use - * that release store as a barrier. - */ - WRITE_ONCE(sw->switch_time, time); -} - static inline int ghost_schedattr_to_enclave_fd(const struct sched_attr *attr) { return attr->sched_runtime; From 5ab35a10fc0decf3ceee87c4bbc78ff6181eecd6 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 19 Oct 2021 09:15:08 -0700 Subject: [PATCH 069/165] sched: get rid of unused 'ghost_run_gtid_on_check()' Tested: builds Effort: sched/ghost/abi Change-Id: Ibd1b881ec6b2d14030246f71cc953bab4295de69 --- kernel/sched/ghost.c | 13 ------------- kernel/sched/sched.h | 2 -- 2 files changed, 15 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index dcbe39f81fa0..c880a7a18487 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -5092,19 +5092,6 @@ int ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, int cpu) /*check_caller_enclave=*/ false); } -/* - * Attempts to run gtid on cpu. Returns 0 or -error. - * - * Like ghost_run_gtid_on, but checks that the calling CPU and the target cpu - * are in the same enclave. - */ -int ghost_run_gtid_on_check(gtid_t gtid, u32 task_barrier, int run_flags, - int cpu) -{ - return __ghost_run_gtid_on(gtid, task_barrier, run_flags, cpu, - /*check_caller_enclave=*/ true); -} - static inline bool _ghost_txn_ready(int cpu, int *commit_flags) { struct ghost_txn *txn; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f860e0c78347..3b95cbdf0025 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -271,8 +271,6 @@ extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); extern int ghost_run_gtid_on(s64 gtid, u32 task_barrier, int run_flags, int cpu); -extern int ghost_run_gtid_on_check(s64 gtid, u32 task_barrier, int run_flags, - int cpu); extern void ghost_cpu_idle(void); struct rq_flags; From 10b3fc7e532917b49a4cb249d7258b780fef832b Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 19 Oct 2021 10:31:39 -0700 Subject: [PATCH 070/165] sched: add ghost_uapi.h This is a "local" include file that contains the ABI that ghost.c and ghostfs.c are compiled with. For now this is a simple wrapper around However when we have per-abi subdirectories it will contain a snapshot of at a specific ABI. The reason for adding this indirection is so that we can snapshot an ABI by simply branching ghost.c and ghostfs.c from kernel/sched and ghost_uapi.h from uapi/linux/ghost.h (no need to modify source code which preserves integrity of the snapshot). Tested: builds Effort: sched/ghost/abi Change-Id: Ife564d0b886dad2027c95885d7bc7ba9d2bed43f --- kernel/sched/ghost.c | 6 +++--- kernel/sched/ghost_uapi.h | 18 ++++++++++++++++++ kernel/sched/ghostfs.c | 3 +++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 kernel/sched/ghost_uapi.h diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index c880a7a18487..880f81bab6bd 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -24,13 +24,13 @@ #include #endif -#include - #include -#include "ghost.h" #include "sched.h" +#include "ghost_uapi.h" +#include "ghost.h" + /* ghost tid */ typedef int64_t gtid_t; diff --git a/kernel/sched/ghost_uapi.h b/kernel/sched/ghost_uapi.h new file mode 100644 index 000000000000..3170b2c0d67f --- /dev/null +++ b/kernel/sched/ghost_uapi.h @@ -0,0 +1,18 @@ +/* + * Copyright 2021 Google LLC + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef _KERNEL_SCHED_GHOST_UAPI_H_ +#define _KERNEL_SCHED_GHOST_UAPI_H_ + +#include + +#endif diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index 34eb3c896d03..d479d49c66ec 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -13,6 +13,9 @@ #include "sched.h" +#include "ghost_uapi.h" +#include "ghost.h" + #include #include #include From 7ed673850cf885a193d0e7b522cc435e72adc529 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 19 Oct 2021 10:46:03 -0700 Subject: [PATCH 071/165] sched: GHOST_CPU_DATA_REGION_SIZE is specific to an ABI Move from sched.h to ghostfs.c since this is an abi-specific concern. Tested: builds Effort: sched/ghost/abi Change-Id: I48bc737dfb78729b26fbf377fe20ab3f693e1e4e --- kernel/sched/ghostfs.c | 3 +++ kernel/sched/sched.h | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index d479d49c66ec..e1606d6c0129 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -918,6 +918,9 @@ static void runtime_adjust_dirtabs(void) { struct gf_dirent *enc_txn; + const loff_t GHOST_CPU_DATA_REGION_SIZE = + sizeof(struct ghost_cpu_data) * num_possible_cpus(); + enc_txn = gf_find_file(enclave_dirtab, "cpu_data"); if (WARN_ON(!enc_txn)) return; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3b95cbdf0025..8ef91adef0d0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -149,9 +149,6 @@ struct ghost_sw_region { struct ghost_enclave *enclave; }; -#define GHOST_CPU_DATA_REGION_SIZE \ - (sizeof(struct ghost_cpu_data) * num_possible_cpus()) - struct enclave_work { struct list_head link; unsigned int nr_decrefs; From 81e5d38ba17b7e51cbf6700860b732b78267fa29 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sat, 16 Oct 2021 11:18:40 -0700 Subject: [PATCH 072/165] sched: separate headers along the ABI boundaries. At this point we have the following: uapi/linux/ghost.h: this is user-kernel ABI and any inclusion outside of ghost.c, ghostfs.c and ghost_uapi.h must be properly vetted. All such inclusions must either be worked around or fixed. At this time we have: - fs/timerfd.c: needs fixing - sched/core.c: needs fixing - sched/bpf.c: needs fixing - bpf/btf.c: not fixable so work around by not using BPF CO-RE APIs to access ghost related struct members. include/linux/ghost.h: currently empty but generally this will advertise APIs that span ABI boundary provided by ghost to rest of the kernel. It also captures inclusions like that would have implicitly resolved to (we don't ever want implicit inclusion of the ghost uapi). kernel/sched/sched.h: APIs that span ABI boundary provided by ghost to code in kernel/sched. At the moment the implementation of these APIs is in ghost.c but will move shortly to ghost_core.c which is independent of a particular ABI version. kernel/sched/ghost.h: APIs between ghost.c and ghostfs.c specific to a particular ABI. ghost_uapi.h: - in the kernel/sched directory this is just a wrapper on top of - in the kernel/sched/ghost_abi_ subdirectory this file is a snapshot of the contents at abi . Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: If8e5ff76e278c6347d146b248c3237397b856f97 --- include/uapi/linux/ghost.h | 6 ++--- kernel/sched/bpf.c | 1 + kernel/sched/core.c | 1 - kernel/sched/ghost.c | 1 + kernel/sched/ghost.h | 49 ++++++++++++++++++++++++++++++++++++++ kernel/sched/ghostfs.c | 7 +++--- kernel/sched/sched.h | 44 +--------------------------------- 7 files changed, 59 insertions(+), 50 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 672f2434b9ea..173fcbbdc0ba 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -11,8 +11,8 @@ * GNU General Public License for more details. */ -#ifndef _SCHED_GHOST_H_ -#define _SCHED_GHOST_H_ +#ifndef _UAPI_LINUX_GHOST_H_ +#define _UAPI_LINUX_GHOST_H_ #include @@ -614,4 +614,4 @@ enum { // NOLINTEND -#endif /* _SCHED_GHOST_H_ */ +#endif /* _UAPI_LINUX_GHOST_H_ */ diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 4d08054a76ff..e3365f1d6795 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -283,6 +283,7 @@ ghost_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) } } +#include static bool ghost_msg_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6d6b9cc9c48a..f145a76efaaa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -10,7 +10,6 @@ #include #undef CREATE_TRACE_POINTS -#include "ghost.h" #include "sched.h" #include diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 880f81bab6bd..d40ce6760a4f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -28,6 +28,7 @@ #include "sched.h" +#define __INCLUDE_KERNEL_SCHED_GHOST #include "ghost_uapi.h" #include "ghost.h" diff --git a/kernel/sched/ghost.h b/kernel/sched/ghost.h index 9093627f89a2..6720af394039 100644 --- a/kernel/sched/ghost.h +++ b/kernel/sched/ghost.h @@ -10,9 +10,58 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ +#ifndef _KERNEL_SCHED_GHOST_H_ +#define _KERNEL_SCHED_GHOST_H_ + +#ifndef __INCLUDE_KERNEL_SCHED_GHOST +#error "ghost.h can only be included from ghost.c or ghostfs.c" +#endif + +struct ghost_sw_region { + struct list_head list; /* ghost_enclave glue */ + uint32_t alloc_scan_start; /* allocator starts scan here */ + struct ghost_sw_region_header *header; /* pointer to vmalloc memory */ + size_t mmap_sz; /* size of mmapped region */ + struct ghost_enclave *enclave; +}; + +/* In kernel/sched/ghostfs.c */ +struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); +void ghostfs_put_enclave_ctl(struct file *f); +void ghostfs_remove_enclave(struct ghost_enclave *e); + +/* In kernel/sched/ghost.c */ +struct ghost_enclave *ghost_create_enclave(void); +void ghost_destroy_enclave(struct ghost_enclave *e); +int ghost_enclave_set_cpus(struct ghost_enclave *e, const struct cpumask *cpus); +int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, + void *addr, ulong mapsize); +int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, + struct ghost_cpu_data **cpu_data, ulong mapsize); +struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, + unsigned int id, + unsigned int node); + +int ghost_sw_get_info(struct ghost_enclave *e, + struct ghost_ioc_sw_get_info __user *arg); +int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo); +int ghost_create_queue(struct ghost_enclave *e, + struct ghost_ioc_create_queue __user *arg); +int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg); +int ghost_set_default_queue(struct ghost_enclave *e, + struct ghost_ioc_set_default_queue __user *arg); +int ghost_config_queue_wakeup(struct ghost_ioc_config_queue_wakeup __user *arg); +int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg); +int ioctl_ghost_commit_txn(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg); +int ghost_sync_group(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg); +int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg); /* Stub functions for open source ghOSt code. */ /* These functions should not be open sourced. */ static inline void kvm_register_core_conflict(int cpu) {} static inline bool try_ipiless_wakeup(int cpu) { return false; } + +#endif /* _KERNEL_SCHED_GHOST_H_ */ diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c index e1606d6c0129..2d942f4bd282 100644 --- a/kernel/sched/ghostfs.c +++ b/kernel/sched/ghostfs.c @@ -13,9 +13,6 @@ #include "sched.h" -#include "ghost_uapi.h" -#include "ghost.h" - #include #include #include @@ -27,6 +24,10 @@ #include "../../fs/kernfs/kernfs-internal.h" +#define __INCLUDE_KERNEL_SCHED_GHOST +#include "ghost_uapi.h" +#include "ghost.h" + /* Helper for when you "echo foo > ctl" without the -n. */ static void strip_slash_n(char *buf, size_t count) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 8ef91adef0d0..28995a89542c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -102,8 +102,6 @@ extern void call_trace_sched_update_nr_running(struct rq *rq, int count); #ifdef CONFIG_SCHED_CLASS_GHOST -#include - struct ghost_rq { struct task_struct *agent; /* protected by e->lock and rq->lock */ uint32_t agent_barrier; @@ -141,14 +139,6 @@ struct ghost_rq { int64_t rendezvous; }; -struct ghost_sw_region { - struct list_head list; /* ghost_enclave glue */ - uint32_t alloc_scan_start; /* allocator starts scan here */ - struct ghost_sw_region_header *header; /* pointer to vmalloc memory */ - size_t mmap_sz; /* size of mmapped region */ - struct ghost_enclave *enclave; -}; - struct enclave_work { struct list_head link; unsigned int nr_decrefs; @@ -210,42 +200,9 @@ struct ghost_enclave { #endif }; -/* In kernel/sched/ghostfs.c */ -extern struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); -extern void ghostfs_put_enclave_ctl(struct file *f); -extern void ghostfs_remove_enclave(struct ghost_enclave *e); - /* In kernel/sched/ghost.c */ extern bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev); -extern struct ghost_enclave *ghost_create_enclave(void); extern void enclave_release(struct kref *k); -extern void ghost_destroy_enclave(struct ghost_enclave *e); -extern int ghost_enclave_set_cpus(struct ghost_enclave *e, - const struct cpumask *cpus); -extern int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, - void *addr, ulong mapsize); -extern int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, - struct ghost_cpu_data **cpu_data, ulong mapsize); -extern struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, - unsigned int id, - unsigned int node); -extern int ghost_sw_get_info(struct ghost_enclave *e, - struct ghost_ioc_sw_get_info __user *arg); -extern int ghost_sw_free(struct ghost_enclave *e, - struct ghost_sw_info __user *uinfo); -extern int ghost_create_queue(struct ghost_enclave *e, - struct ghost_ioc_create_queue __user *arg); -extern int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg); -extern int ghost_set_default_queue(struct ghost_enclave *e, - struct ghost_ioc_set_default_queue __user *arg); -extern int ghost_config_queue_wakeup( - struct ghost_ioc_config_queue_wakeup __user *arg); -extern int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg); -extern int ioctl_ghost_commit_txn(struct ghost_enclave *e, - struct ghost_ioc_commit_txn __user *arg); -extern int ghost_sync_group(struct ghost_enclave *e, - struct ghost_ioc_commit_txn __user *arg); -extern int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); @@ -271,6 +228,7 @@ extern int ghost_run_gtid_on(s64 gtid, u32 task_barrier, int run_flags, extern void ghost_cpu_idle(void); struct rq_flags; +struct bpf_ghost_msg; #ifdef CONFIG_BPF extern bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq); extern void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, From 9f025ee1e85781b5ce40d8ca9eb1f019d6742e0a Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 20 Oct 2021 09:28:33 -0700 Subject: [PATCH 073/165] sched: embed ABI in the 'expected_attach_type' of ghost/bpf programs. We don't have access to the 'enclave_fd' to infer the ghost ABI when a BPF program is loaded into the kernel. The ABI version is required to do proper abi-specific verification of the BPF program. We also now validate that the ABI version at program load time matches the enclave's ABI version (derived from the enclave_fd passed to ghost_bpf_link_attach(). Submitted in conjunction with cl/404566797 Suggested by: brho@ Tested: all unit tests pass successfully in virtme Effort: sched/ghost/abi Change-Id: Ibdb48d01879e0ced483f1660ce3fcbc89789cb11 --- include/uapi/linux/ghost.h | 2 +- kernel/bpf/syscall.c | 10 ++++++++++ kernel/sched/bpf.c | 25 ++++++++++++++++++++----- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 173fcbbdc0ba..7970fd8ab27a 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 55 +#define GHOST_VERSION 56 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index d7d5a7fc48d8..73907a4f5c36 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1928,6 +1928,11 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, struct btf *attach_btf, u32 btf_id, struct bpf_prog *dst_prog) { +#ifdef CONFIG_SCHED_CLASS_GHOST + BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE > 0xFFFF); + expected_attach_type &= 0xFFFF; +#endif + if (btf_id) { if (btf_id > BTF_MAX_TYPE) return -EINVAL; @@ -2921,6 +2926,11 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, static enum bpf_prog_type attach_type_to_prog_type(enum bpf_attach_type attach_type) { +#ifdef CONFIG_SCHED_CLASS_GHOST + BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE > 0xFFFF); + attach_type &= 0xFFFF; +#endif + switch (attach_type) { case BPF_CGROUP_INET_INGRESS: case BPF_CGROUP_INET_EGRESS: diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index e3365f1d6795..5d131dc3df9e 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -11,9 +11,11 @@ * GNU General Public License for more details. */ -#include "sched.h" #include +#include "sched.h" +#include "ghost_uapi.h" + #ifdef CONFIG_SCHED_CLASS_GHOST BPF_CALL_1(bpf_ghost_wake_agent, u32, cpu) @@ -182,11 +184,13 @@ static void ghost_sched_pnt_detach(struct ghost_enclave *e, static const struct bpf_func_proto * ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + int eat = prog->expected_attach_type & 0xFFFF; + switch (func_id) { case BPF_FUNC_ghost_wake_agent: return &bpf_ghost_wake_agent_proto; case BPF_FUNC_ghost_run_gtid: - switch (prog->expected_attach_type) { + switch (eat) { case BPF_GHOST_SCHED_PNT: return &bpf_ghost_run_gtid_proto; default: @@ -371,6 +375,11 @@ static const struct bpf_link_ops bpf_ghost_link_ops = { .dealloc = bpf_ghost_link_dealloc, }; +static inline uint enclave_abi(struct ghost_enclave *e) +{ + return GHOST_VERSION; /* for now all enclaves have the same abi */ +} + int ghost_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { @@ -379,13 +388,18 @@ int ghost_bpf_link_attach(const union bpf_attr *attr, enum bpf_attach_type ea_type; struct ghost_enclave *e; struct fd f_enc = {0}; - int err; + int err, ea_abi; + + BUILD_BUG_ON(GHOST_VERSION > 0xFFFF); if (attr->link_create.flags) return -EINVAL; if (prog->expected_attach_type != attr->link_create.attach_type) return -EINVAL; - ea_type = prog->expected_attach_type; + + // Mask the ABI value encoded in the upper 16 bits. + ea_type = prog->expected_attach_type & 0xFFFF; + ea_abi = (u32)prog->expected_attach_type >> 16; switch (prog->type) { case BPF_PROG_TYPE_GHOST_SCHED: @@ -424,12 +438,13 @@ int ghost_bpf_link_attach(const union bpf_attr *attr, } e = ghost_fdget_enclave(attr->link_create.target_fd, &f_enc); - if (!e) { + if (!e || ea_abi != enclave_abi(e)) { ghost_fdput_enclave(e, &f_enc); /* bpf_link_cleanup() triggers .dealloc, but not .release. */ bpf_link_cleanup(&link_primer); return -EBADF; } + /* * On success, sc_link will hold a kref on the enclave, which will get * put when the link's FD is closed (and thus bpf_link_put -> From 26017f5ff86896434f0e49fd2a80a88eac4f1c59 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 24 Oct 2021 17:09:58 -0700 Subject: [PATCH 074/165] sched: move contents of ghostfs.c into ghost.c Until now ghostfs.c and ghost.c communicated via functions visible in the global namespace. In a world where we have multiple ABIs this won't be possible anymore (unless we rewrite globally visible names and make them unique to an abi). We'll also get rid of ghost.h in a subsequent commit since it is now unncessary. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I2e917245477eb21847be9c2195825354df975781 --- kernel/sched/Makefile | 1 - kernel/sched/ghost.c | 1020 ++++++++++++++++++++++++++++++++++++++ kernel/sched/ghostfs.c | 1047 ---------------------------------------- 3 files changed, 1020 insertions(+), 1048 deletions(-) delete mode 100644 kernel/sched/ghostfs.c diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index f97342908659..4008a8251d4e 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -36,6 +36,5 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o -obj-$(CONFIG_SCHED_CLASS_GHOST) += ghostfs.o obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost.o obj-$(CONFIG_BPF) += bpf.o diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index d40ce6760a4f..99ae91bbedb0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6398,3 +6398,1023 @@ unsigned long ghost_cfs_added_load(struct rq *rq) return sysctl_ghost_cfs_load_added; return 0; } + +#include "../../fs/kernfs/kernfs-internal.h" + +/* Helper for when you "echo foo > ctl" without the -n. */ +static void strip_slash_n(char *buf, size_t count) +{ + char *n = strnchr(buf, count, '\n'); + + if (n) + *n = '\0'; +} + +static struct kernfs_root *ghost_kfs_root; + +struct gf_dirent { + char *name; + umode_t mode; + struct kernfs_ops *ops; + loff_t size; + bool is_dir; +}; + +/* + * Finds a specific file in a directory table. This is useful for runtime + * initialization, though keep in mind the dirtab is a global structure. + */ +static struct gf_dirent *gf_find_file(struct gf_dirent *dirtab, + const char *name) +{ + struct gf_dirent *gft; + + for (gft = dirtab; gft->name; gft++) { + if (!strcmp(gft->name, name)) + return gft; + } + return NULL; +} + +/* + * Every open file in an enclave holds a kref on the enclave. This includes the + * ctl, txn, the status words, etc. + * + * Each of those file's priv points to the enclave struct, either directly or + * indirectly. For instance, the SW files have their own object that points to + * the enclave. We can differentiate which file is which based on the kernfs + * ops. In some cases, we can reuse ops and differentiate on the object's name. + * + * This does not include the enclave directory; it's priv is NULL, and it has no + * function ops. + * + * Destroying the enclave involves removing it from the filesystem by writing + * "destroy" into ctl. That triggers kernfs_remove() and whatever things we + * want to do for the enclave itself, like killing the agents. The enclave + * object will exist until the last FD is closed. Note that when we call + * kernfs_remove(), it'll "drain" any open files, which involves unmapping any + * mmapped files. + * + * Most of the heavy lifting for destroying an enclave is done in + * ghost_destroy_enclave(). The kernel scheduler may call that function at any + * moment if it determines an enclave misbehaved. + */ + +static struct ghost_enclave *of_to_e(struct kernfs_open_file *of) +{ + return of->kn->priv; +} + +static struct ghost_enclave *seq_to_e(struct seq_file *sf) +{ + struct kernfs_open_file *of = sf->private; + + return of_to_e(of); +} + +/* For enclave files whose priv directly points to a ghost enclave. */ +static int gf_e_open(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + + kref_get(&e->kref); + return 0; +} + +/* For enclave files whose priv directly points to a ghost enclave. */ +static void gf_e_release(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + + kref_put(&e->kref, enclave_release); +} + +/* + * There can be at most one writable open, and there can be any number of + * read-only opens. + * + * If there is a writable open, that means userspace has advertised this enclave + * as having an agent_online, and the value of agent_online will change + * accordingly. + */ +static int gf_agent_online_open(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long irq_fl; + + if (!(of->file->f_mode & FMODE_WRITE)) + goto done; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->agent_online) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + e->agent_online = true; + spin_unlock_irqrestore(&e->lock, irq_fl); + + kernfs_notify(of->kn); + +done: + kref_get(&e->kref); + return 0; +} + +static void gf_agent_online_release(struct kernfs_open_file *of) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long irq_fl; + + if (!(of->file->f_mode & FMODE_WRITE)) + goto done; + + spin_lock_irqsave(&e->lock, irq_fl); + WARN_ONCE(!e->agent_online, + "closing RW agent_online for enclave_%lu, but it's not online!", + e->id); + e->agent_online = false; + spin_unlock_irqrestore(&e->lock, irq_fl); + + /* + * Kicks any epoll/notifiers about the change in state. This is the + * "agent death edge". + * + * Note that this doesn't necessarily mean the enclave will be + * destroyed, merely that userspace no longer thinks there is a valid + * agent. This FD was probably closed because the agent crashed. + */ + kernfs_notify(of->kn); + +done: + kref_put(&e->kref, enclave_release); +} + +static int gf_agent_online_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%u", e->agent_online ? 1 : 0); + return 0; +} + +static ssize_t gf_agent_online_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + /* We need a write op so kernfs allows us to be opened for writing. */ + return -EINVAL; +} + +static struct kernfs_ops gf_ops_e_agent_online = { + .open = gf_agent_online_open, + .release = gf_agent_online_release, + .seq_show = gf_agent_online_show, + .write = gf_agent_online_write, +}; + +static int gf_cpu_data_mmap(struct kernfs_open_file *of, + struct vm_area_struct *vma) +{ + struct ghost_enclave *e = of_to_e(of); + + return ghost_cpu_data_mmap(of->file, vma, e->cpu_data, + of->kn->attr.size); +} + +static struct kernfs_ops gf_ops_e_cpu_data = { + .open = gf_e_open, + .release = gf_e_release, + .mmap = gf_cpu_data_mmap, +}; + +/* Returns an ASCII list of the cpus in the enclave */ +static ssize_t gf_cpulist_read(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long fl; + cpumask_var_t cpus; + char *pagebuf; + ssize_t strlen; + + if (off > PAGE_SIZE) + return -EINVAL; + bytes = min_t(size_t, bytes, PAGE_SIZE - off); + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + spin_lock_irqsave(&e->lock, fl); + memcpy(cpus, &e->cpus, cpumask_size()); + spin_unlock_irqrestore(&e->lock, fl); + + pagebuf = (char *)get_zeroed_page(GFP_KERNEL); + if (!pagebuf) { + free_cpumask_var(cpus); + return -ENOMEM; + } + + strlen = cpumap_print_to_pagebuf(/*list=*/true, pagebuf, cpus); + bytes = memory_read_from_buffer(buf, bytes, &off, pagebuf, strlen); + + free_page((unsigned long)pagebuf); + free_cpumask_var(cpus); + + return bytes; +} + + +static ssize_t __cpu_set_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off, bool is_list) +{ + cpumask_var_t cpus; + int err; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + if (is_list) + err = cpulist_parse(buf, cpus); + else + err = cpumask_parse(buf, cpus); + if (err) { + free_cpumask_var(cpus); + return err; + } + err = ghost_enclave_set_cpus(of_to_e(of), cpus); + free_cpumask_var(cpus); + return err ? err : bytes; +} + +/* Sets the enclave's cpus to buf, an ASCII list of cpus. */ +static ssize_t gf_cpulist_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + return __cpu_set_write(of, buf, bytes, off, /*is_list=*/true); +} + +static struct kernfs_ops gf_ops_e_cpulist = { + .open = gf_e_open, + .release = gf_e_release, + .read = gf_cpulist_read, + .write = gf_cpulist_write, +}; + +/* Returns an ASCII hex mask of the cpus in the enclave */ +static ssize_t gf_cpumask_read(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + unsigned long fl; + cpumask_var_t cpus; + char *mask_str; + int len; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + spin_lock_irqsave(&e->lock, fl); + memcpy(cpus, &e->cpus, cpumask_size()); + spin_unlock_irqrestore(&e->lock, fl); + + /* + * +1 for the \0. We won't return the \0 to userspace, but the string + * will be null-terminated while in the kernel. + */ + len = snprintf(NULL, 0, "%*pb\n", cpumask_pr_args(cpus)) + 1; + mask_str = kmalloc(len, GFP_KERNEL); + if (!mask_str) { + free_cpumask_var(cpus); + return -ENOMEM; + } + len = snprintf(mask_str, len, "%*pb\n", cpumask_pr_args(cpus)); + bytes = memory_read_from_buffer(buf, bytes, &off, mask_str, len); + + kfree(mask_str); + free_cpumask_var(cpus); + + return bytes; +} + +/* Sets the enclave's cpus to buf, an ASCII hex cpumask. */ +static ssize_t gf_cpumask_write(struct kernfs_open_file *of, char *buf, + size_t bytes, loff_t off) +{ + return __cpu_set_write(of, buf, bytes, off, /*is_list=*/false); +} + +static struct kernfs_ops gf_ops_e_cpumask = { + .open = gf_e_open, + .release = gf_e_release, + .read = gf_cpumask_read, + .write = gf_cpumask_write, +}; + +static int gf_ctl_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%lu", e->id); + return 0; +} + +/* Called from the scheduler when it destroys the enclave. */ +void ghostfs_remove_enclave(struct ghost_enclave *e) +{ + kernfs_remove(e->enclave_dir); +} + +static void destroy_enclave(struct kernfs_open_file *ctl_of) +{ + struct kernfs_node *ctl = ctl_of->kn; + struct ghost_enclave *e = of_to_e(ctl_of); + + /* + * kernfs_remove() works recursively, removing the entire enclave + * directory. We need to remove the ctl file first, since we're in the + * middle of a kn op. + * + * Multiple threads can call kernfs_remove_self() at once. Whichever + * succeeds will remove the directory and release e. + */ + if (!kernfs_remove_self(ctl)) + return; + if (WARN_ON(ctl->parent != e->enclave_dir)) + return; + ghost_destroy_enclave(e); +} + +static struct ghost_sw_region *of_to_swr(struct kernfs_open_file *of) +{ + return of->kn->priv; +} + +/* + * swr_kns refcount the enclave, even though they point to the swr. An swr will + * never be deleted until the enclave is released. + */ +static int gf_swr_open(struct kernfs_open_file *of) +{ + struct ghost_sw_region *swr = of_to_swr(of); + + kref_get(&swr->enclave->kref); + return 0; +} + +static void gf_swr_release(struct kernfs_open_file *of) +{ + struct ghost_sw_region *swr = of_to_swr(of); + + kref_put(&swr->enclave->kref, enclave_release); +} + +static int gf_swr_mmap(struct kernfs_open_file *of, struct vm_area_struct *vma) +{ + struct ghost_sw_region *swr = of_to_swr(of); + + if (vma->vm_flags & VM_WRITE) + return -EINVAL; + vma->vm_flags &= ~VM_MAYWRITE; + + return ghost_region_mmap(of->file, vma, swr->header, of->kn->attr.size); +} + +static struct kernfs_ops gf_ops_e_swr = { + .open = gf_swr_open, + .release = gf_swr_release, + .mmap = gf_swr_mmap, +}; + +static int create_sw_region(struct kernfs_open_file *ctl_of, unsigned int id, + unsigned int numa_node) +{ + struct ghost_enclave *e = of_to_e(ctl_of); + struct kernfs_node *dir, *swr_kn; + struct ghost_sw_region *swr; + char name[31]; + int err; + + dir = kernfs_find_and_get(e->enclave_dir, "sw_regions"); + if (WARN_ON_ONCE(!dir)) + return -EINVAL; + + if (snprintf(name, sizeof(name), "sw_%u", id) >= sizeof(name)) { + err = -ENOSPC; + goto err_snprintf; + } + + swr_kn = kernfs_create_file(dir, name, 0440, 0, &gf_ops_e_swr, e); + if (IS_ERR(swr_kn)) { + err = PTR_ERR(swr_kn); + goto err_create_kn; + } + /* swr_kn is inaccessible until kernfs_activate. */ + + swr = ghost_create_sw_region(e, id, numa_node); + if (IS_ERR(swr)) { + err = PTR_ERR(swr); + goto err_create_swr; + } + + swr_kn->attr.size = swr->mmap_sz; + swr_kn->priv = swr; + kernfs_activate(swr_kn); + + return 0; +err_create_swr: + kernfs_remove(swr_kn); +err_create_kn: +err_snprintf: + kernfs_put(dir); + return err; +} + +static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + unsigned int arg1, arg2; + int err; + + /* + * Ignore the offset for ctl commands, so userspace doesn't have to + * worry about lseeking after each command. Userspace should submit a + * single write per command too, and not write("des"), write ("troy"). + * Otherwise they'll fail. + */ + + strip_slash_n(buf, len); + + if (!strcmp(buf, "destroy")) { + destroy_enclave(of); + } else if (sscanf(buf, "create sw_region %u %u", &arg1, &arg2) == 2) { + err = create_sw_region(of, arg1, arg2); + if (err) + return err; + } else { + pr_err("%s: bad cmd :%s:", __func__, buf); + return -EINVAL; + } + + return len; +} + +static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, + unsigned long arg) +{ + struct ghost_enclave *e = of_to_e(of); + + if (!(of->file->f_mode & FMODE_WRITE)) + return -EACCES; + switch (cmd) { + case GHOST_IOC_NULL: + return 0; + case GHOST_IOC_SW_GET_INFO: + return ghost_sw_get_info(e, + (struct ghost_ioc_sw_get_info __user *)arg); + case GHOST_IOC_SW_FREE: + return ghost_sw_free(e, (void __user *)arg); + case GHOST_IOC_CREATE_QUEUE: + return ghost_create_queue(e, + (struct ghost_ioc_create_queue __user *)arg); + case GHOST_IOC_ASSOC_QUEUE: + return ghost_associate_queue( + (struct ghost_ioc_assoc_queue __user *)arg); + case GHOST_IOC_SET_DEFAULT_QUEUE: + return ghost_set_default_queue(e, + (struct ghost_ioc_set_default_queue __user *)arg); + case GHOST_IOC_CONFIG_QUEUE_WAKEUP: + return ghost_config_queue_wakeup( + (struct ghost_ioc_config_queue_wakeup __user *)arg); + case GHOST_IOC_GET_CPU_TIME: + return ghost_get_cpu_time( + (struct ghost_ioc_get_cpu_time __user *)arg); + case GHOST_IOC_COMMIT_TXN: + return ioctl_ghost_commit_txn(e, + (struct ghost_ioc_commit_txn __user *)arg); + case GHOST_IOC_SYNC_GROUP_TXN: + return ghost_sync_group(e, + (struct ghost_ioc_commit_txn __user *)arg); + case GHOST_IOC_TIMERFD_SETTIME: + return ghost_timerfd_settime( + (struct ghost_ioc_timerfd_settime __user *)arg); + } + return -ENOIOCTLCMD; +} + +static struct kernfs_ops gf_ops_e_ctl = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_ctl_show, + .write = gf_ctl_write, + .ioctl = gf_ctl_ioctl, +}; + +/* + * Returns the enclave for f, if f is a ghostfs ctl file. We could support + * other file types, but since this is a backdoor into the FS, we only need to + * support ctl. + * + * Successful callers must call ghostfs_put_ctl_enclave(f). + * + * The kernfs_ops don't need this helper. kernfs manages the the refcounts. We + * need to do it manually here, because this is is a "backdoor" function to get + * the enclave pointer. That pointer is kept alive by kernfs. + */ +struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) +{ + struct kernfs_node *kn; + + kn = kernfs_node_from_file(f); + if (!kn) + return NULL; + if (kernfs_type(kn) != KERNFS_FILE) + return NULL; + if (kn->attr.ops != &gf_ops_e_ctl) + return NULL; + if (!kernfs_get_active(kn)) + return NULL; + WARN_ON(!kn->priv); + return kn->priv; +} + +/* Pair this with a successful ghostfs_ctl_to_enclave call. */ +void ghostfs_put_enclave_ctl(struct file *f) +{ + struct kernfs_node *kn; + + kn = kernfs_node_from_file(f); + if (WARN_ON(!kn)) + return; + if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) + return; + if (WARN_ON(kn->attr.ops != &gf_ops_e_ctl)) + return; + kernfs_put_active(kn); +} + +static int gf_runnable_timeout_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%lld", ktime_to_ms(READ_ONCE(e->max_unscheduled))); + return 0; +} + +static ssize_t gf_runnable_timeout_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + unsigned long msec; + + err = kstrtoul(buf, 0, &msec); + if (err) + return -EINVAL; + + WRITE_ONCE(e->max_unscheduled, ms_to_ktime(msec)); + + return len; +} + +static struct kernfs_ops gf_ops_e_runnable_timeout = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_runnable_timeout_show, + .write = gf_runnable_timeout_write, +}; + +static int gf_switchto_disabled_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->switchto_disabled)); + return 0; +} + +static ssize_t gf_switchto_disabled_write(struct kernfs_open_file *of, + char *buf, size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->switchto_disabled, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_switchto_disabled = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_switchto_disabled_show, + .write = gf_switchto_disabled_write, +}; + +static int gf_wake_on_waker_cpu_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->wake_on_waker_cpu)); + return 0; +} + +static ssize_t gf_wake_on_waker_cpu_write(struct kernfs_open_file *of, + char *buf, size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->wake_on_waker_cpu, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_wake_on_waker_cpu = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_wake_on_waker_cpu_show, + .write = gf_wake_on_waker_cpu_write, +}; + +static int gf_commit_at_tick_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->commit_at_tick)); + return 0; +} + +static ssize_t gf_commit_at_tick_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->commit_at_tick, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_commit_at_tick = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_commit_at_tick_show, + .write = gf_commit_at_tick_write, +}; + +static int gf_status_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + unsigned long fl; + bool is_active; + unsigned long nr_tasks; + + /* + * Userspace uses this to find any active enclave, since they don't have + * any other methods yet to know which enclave to use. + */ + spin_lock_irqsave(&e->lock, fl); + /* + * We don't need to lock to read agent_online, but eventually we'll + * check for the presence of an interstitial scheduler too. This status + * is for the *enclave*, not the *agent*. + */ + is_active = e->agent_online; + nr_tasks = e->nr_tasks; + spin_unlock_irqrestore(&e->lock, fl); + + seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); + seq_printf(sf, "nr_tasks %lu\n", nr_tasks); + return 0; +} + +static struct kernfs_ops gf_ops_e_status = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_status_show, +}; + +static struct gf_dirent enclave_dirtab[] = { + { + .name = "sw_regions", + .mode = 0555, + .is_dir = true, + }, + { + .name = "agent_online", + .mode = 0664, + .ops = &gf_ops_e_agent_online, + }, + { + .name = "cpu_data", + .mode = 0660, + .ops = &gf_ops_e_cpu_data, + }, + { + .name = "cpulist", + .mode = 0664, + .ops = &gf_ops_e_cpulist, + }, + { + .name = "cpumask", + .mode = 0664, + .ops = &gf_ops_e_cpumask, + }, + { + .name = "ctl", + .mode = 0664, + .ops = &gf_ops_e_ctl, + }, + { + .name = "runnable_timeout", + .mode = 0664, + .ops = &gf_ops_e_runnable_timeout, + }, + { + .name = "switchto_disabled", + .mode = 0664, + .ops = &gf_ops_e_switchto_disabled, + }, + { + .name = "wake_on_waker_cpu", + .mode = 0664, + .ops = &gf_ops_e_wake_on_waker_cpu, + }, + { + .name = "commit_at_tick", + .mode = 0664, + .ops = &gf_ops_e_commit_at_tick, + }, + { + .name = "status", + .mode = 0444, + .ops = &gf_ops_e_status, + }, + {0}, +}; + +/* Caller is responsible for cleanup. Removing the parent will suffice. */ +static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, + struct ghost_enclave *priv) +{ + struct gf_dirent *gft; + struct kernfs_node *kn; + + for (gft = dirtab; gft->name; gft++) { + if (gft->is_dir) { + kn = kernfs_create_dir(parent, gft->name, gft->mode, + NULL); + } else { + kn = kernfs_create_file(parent, gft->name, gft->mode, + gft->size, gft->ops, priv); + } + if (IS_ERR(kn)) + return PTR_ERR(kn); + } + return 0; +} + +static int make_enclave(struct kernfs_node *parent, unsigned long id) +{ + struct kernfs_node *dir; + struct ghost_enclave *e; + char name[31]; + int ret; + + /* + * ghost_create_enclave() is mostly just "alloc and initialize". + * Anything done by it gets undone in enclave_release, and it is not + * discoverable, usable, or otherwise hooked into the kernel until + * kernfs_active(). + */ + e = ghost_create_enclave(); + if (!e) + return -ENOMEM; + e->id = id; + if (snprintf(name, sizeof(name), "enclave_%lu", id) >= sizeof(name)) { + ret = -ENOSPC; + goto out_e; + } + + dir = kernfs_create_dir(parent, name, 0555, NULL); + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + goto out_e; + } + e->enclave_dir = dir; + + ret = gf_add_files(dir, enclave_dirtab, e); + if (ret) + goto out_dir; + + /* + * Once the enclave has been activated, it is available to userspace and + * can be used for scheduling. After that, we must destroy it by + * calling ghost_destroy_enclave(), not by releasing the reference. + */ + kernfs_activate(dir); /* recursive */ + + return 0; + +out_dir: + kernfs_remove(dir); /* recursive */ +out_e: + kref_put(&e->kref, enclave_release); + return ret; +} + +static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct kernfs_node *ctl = of->kn; + struct kernfs_node *top_dir = ctl->parent; + unsigned long x; + int ret; + + strip_slash_n(buf, len); + + /* This will ignore any extra digits or characters beyond the %u. */ + ret = sscanf(buf, "create %lu", &x); + if (ret == 1) { + ret = make_enclave(top_dir, x); + return ret ? ret : len; + } + + return -EINVAL; +} + +static struct kernfs_ops gf_ops_top_ctl = { + .write = gf_top_ctl_write, +}; + +static int gf_top_version_show(struct seq_file *sf, void *v) +{ + seq_printf(sf, "%u", GHOST_VERSION); + return 0; +} + +static struct kernfs_ops gf_ops_top_version = { + .seq_show = gf_top_version_show, +}; + +static struct gf_dirent top_dirtab[] = { + { + .name = "ctl", + .mode = 0660, + .ops = &gf_ops_top_ctl, + }, + { + .name = "version", + .mode = 0444, + .ops = &gf_ops_top_version, + }, + {0} +}; + +/* + * Most gf_dirent file sizes are not known at compile time. Most don't matter + * for sysfs and we can leave them as 0. But for anything that gets mmapped, + * it's convenient for userspace for the kernel to say how big it is. + */ +static void runtime_adjust_dirtabs(void) +{ + struct gf_dirent *enc_txn; + + const loff_t GHOST_CPU_DATA_REGION_SIZE = + sizeof(struct ghost_cpu_data) * num_possible_cpus(); + + enc_txn = gf_find_file(enclave_dirtab, "cpu_data"); + if (WARN_ON(!enc_txn)) + return; + enc_txn->size = GHOST_CPU_DATA_REGION_SIZE; +} + +static int __init ghost_setup_root(void) +{ + int ret; + struct kernfs_root *fs_root; + + fs_root = kernfs_create_root(NULL, KERNFS_ROOT_CREATE_DEACTIVATED | + KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, NULL); + if (IS_ERR(fs_root)) + return PTR_ERR(fs_root); + + ret = gf_add_files(fs_root->kn, top_dirtab, NULL); + if (ret) { + kernfs_destroy_root(fs_root); + return ret; + } + + ghost_kfs_root = fs_root; + + runtime_adjust_dirtabs(); + + kernfs_activate(ghost_kfs_root->kn); + + return ret; +} + +static int ghost_get_tree(struct fs_context *fc) +{ + int ret; + + /* Technically, this should be in uapi/linux/magic.h. */ + #define GHOST_SUPER_MAGIC 0xBAD1DEA2 + + VM_BUG_ON(!ghost_kfs_root); + ret = kernfs_get_tree(fc); + if (ret) + WARN(1, "Failed to mount ghostfs!"); + return ret; +} + +static void ghost_fs_context_free(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + + kernfs_free_fs_context(fc); + kfree(kfc); +} + +static const struct fs_context_operations ghost_fs_context_ops = { + .free = ghost_fs_context_free, + .get_tree = ghost_get_tree, +}; + +static int ghost_init_fs_context(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc; + + kfc = kzalloc(sizeof(struct kernfs_fs_context), GFP_KERNEL); + if (!kfc) + return -ENOMEM; + + kfc->root = ghost_kfs_root; + kfc->magic = GHOST_SUPER_MAGIC; + fc->fs_private = kfc; + fc->ops = &ghost_fs_context_ops; + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(&init_user_ns); + fc->global = true; + return 0; +} + +static void ghost_kill_sb(struct super_block *sb) +{ + kernfs_kill_sb(sb); +} + +static struct file_system_type ghost_fs_type = { + .name = "ghost", + .init_fs_context = ghost_init_fs_context, + .kill_sb = ghost_kill_sb, +}; + +static int __init ghostfs_init(void) +{ + int ret = 0; + + ret = ghost_setup_root(); + if (ret) + return ret; + + ret = sysfs_create_mount_point(fs_kobj, "ghost"); + if (ret) + goto cleanup_root; + + ret = register_filesystem(&ghost_fs_type); + if (ret) + goto cleanup_mountpoint; + + return 0; + +cleanup_mountpoint: + sysfs_remove_mount_point(fs_kobj, "ghost"); +cleanup_root: + kernfs_destroy_root(ghost_kfs_root); + ghost_kfs_root = NULL; + + return ret; +} + +static void __exit ghostfs_exit(void) +{ + unregister_filesystem(&ghost_fs_type); + sysfs_remove_mount_point(fs_kobj, "ghost"); + kernfs_destroy_root(ghost_kfs_root); +} + +late_initcall(ghostfs_init); +__exitcall(ghostfs_exit); diff --git a/kernel/sched/ghostfs.c b/kernel/sched/ghostfs.c deleted file mode 100644 index 2d942f4bd282..000000000000 --- a/kernel/sched/ghostfs.c +++ /dev/null @@ -1,1047 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#include "sched.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../../fs/kernfs/kernfs-internal.h" - -#define __INCLUDE_KERNEL_SCHED_GHOST -#include "ghost_uapi.h" -#include "ghost.h" - -/* Helper for when you "echo foo > ctl" without the -n. */ -static void strip_slash_n(char *buf, size_t count) -{ - char *n = strnchr(buf, count, '\n'); - - if (n) - *n = '\0'; -} - -static struct kernfs_root *ghost_kfs_root; - -struct gf_dirent { - char *name; - umode_t mode; - struct kernfs_ops *ops; - loff_t size; - bool is_dir; -}; - -/* - * Finds a specific file in a directory table. This is useful for runtime - * initialization, though keep in mind the dirtab is a global structure. - */ -static struct gf_dirent *gf_find_file(struct gf_dirent *dirtab, - const char *name) -{ - struct gf_dirent *gft; - - for (gft = dirtab; gft->name; gft++) { - if (!strcmp(gft->name, name)) - return gft; - } - return NULL; -} - -/* - * Every open file in an enclave holds a kref on the enclave. This includes the - * ctl, txn, the status words, etc. - * - * Each of those file's priv points to the enclave struct, either directly or - * indirectly. For instance, the SW files have their own object that points to - * the enclave. We can differentiate which file is which based on the kernfs - * ops. In some cases, we can reuse ops and differentiate on the object's name. - * - * This does not include the enclave directory; it's priv is NULL, and it has no - * function ops. - * - * Destroying the enclave involves removing it from the filesystem by writing - * "destroy" into ctl. That triggers kernfs_remove() and whatever things we - * want to do for the enclave itself, like killing the agents. The enclave - * object will exist until the last FD is closed. Note that when we call - * kernfs_remove(), it'll "drain" any open files, which involves unmapping any - * mmapped files. - * - * Most of the heavy lifting for destroying an enclave is done in - * ghost_destroy_enclave(). The kernel scheduler may call that function at any - * moment if it determines an enclave misbehaved. - */ - -static struct ghost_enclave *of_to_e(struct kernfs_open_file *of) -{ - return of->kn->priv; -} - -static struct ghost_enclave *seq_to_e(struct seq_file *sf) -{ - struct kernfs_open_file *of = sf->private; - - return of_to_e(of); -} - -/* For enclave files whose priv directly points to a ghost enclave. */ -static int gf_e_open(struct kernfs_open_file *of) -{ - struct ghost_enclave *e = of_to_e(of); - - kref_get(&e->kref); - return 0; -} - -/* For enclave files whose priv directly points to a ghost enclave. */ -static void gf_e_release(struct kernfs_open_file *of) -{ - struct ghost_enclave *e = of_to_e(of); - - kref_put(&e->kref, enclave_release); -} - -/* - * There can be at most one writable open, and there can be any number of - * read-only opens. - * - * If there is a writable open, that means userspace has advertised this enclave - * as having an agent_online, and the value of agent_online will change - * accordingly. - */ -static int gf_agent_online_open(struct kernfs_open_file *of) -{ - struct ghost_enclave *e = of_to_e(of); - unsigned long irq_fl; - - if (!(of->file->f_mode & FMODE_WRITE)) - goto done; - - spin_lock_irqsave(&e->lock, irq_fl); - if (e->agent_online) { - spin_unlock_irqrestore(&e->lock, irq_fl); - return -EBUSY; - } - e->agent_online = true; - spin_unlock_irqrestore(&e->lock, irq_fl); - - kernfs_notify(of->kn); - -done: - kref_get(&e->kref); - return 0; -} - -static void gf_agent_online_release(struct kernfs_open_file *of) -{ - struct ghost_enclave *e = of_to_e(of); - unsigned long irq_fl; - - if (!(of->file->f_mode & FMODE_WRITE)) - goto done; - - spin_lock_irqsave(&e->lock, irq_fl); - WARN_ONCE(!e->agent_online, - "closing RW agent_online for enclave_%lu, but it's not online!", - e->id); - e->agent_online = false; - spin_unlock_irqrestore(&e->lock, irq_fl); - - /* - * Kicks any epoll/notifiers about the change in state. This is the - * "agent death edge". - * - * Note that this doesn't necessarily mean the enclave will be - * destroyed, merely that userspace no longer thinks there is a valid - * agent. This FD was probably closed because the agent crashed. - */ - kernfs_notify(of->kn); - -done: - kref_put(&e->kref, enclave_release); -} - -static int gf_agent_online_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%u", e->agent_online ? 1 : 0); - return 0; -} - -static ssize_t gf_agent_online_write(struct kernfs_open_file *of, char *buf, - size_t bytes, loff_t off) -{ - /* We need a write op so kernfs allows us to be opened for writing. */ - return -EINVAL; -} - -static struct kernfs_ops gf_ops_e_agent_online = { - .open = gf_agent_online_open, - .release = gf_agent_online_release, - .seq_show = gf_agent_online_show, - .write = gf_agent_online_write, -}; - -static int gf_cpu_data_mmap(struct kernfs_open_file *of, - struct vm_area_struct *vma) -{ - struct ghost_enclave *e = of_to_e(of); - - return ghost_cpu_data_mmap(of->file, vma, e->cpu_data, - of->kn->attr.size); -} - -static struct kernfs_ops gf_ops_e_cpu_data = { - .open = gf_e_open, - .release = gf_e_release, - .mmap = gf_cpu_data_mmap, -}; - -/* Returns an ASCII list of the cpus in the enclave */ -static ssize_t gf_cpulist_read(struct kernfs_open_file *of, char *buf, - size_t bytes, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - unsigned long fl; - cpumask_var_t cpus; - char *pagebuf; - ssize_t strlen; - - if (off > PAGE_SIZE) - return -EINVAL; - bytes = min_t(size_t, bytes, PAGE_SIZE - off); - - if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) - return -ENOMEM; - - spin_lock_irqsave(&e->lock, fl); - memcpy(cpus, &e->cpus, cpumask_size()); - spin_unlock_irqrestore(&e->lock, fl); - - pagebuf = (char *)get_zeroed_page(GFP_KERNEL); - if (!pagebuf) { - free_cpumask_var(cpus); - return -ENOMEM; - } - - strlen = cpumap_print_to_pagebuf(/*list=*/true, pagebuf, cpus); - bytes = memory_read_from_buffer(buf, bytes, &off, pagebuf, strlen); - - free_page((unsigned long)pagebuf); - free_cpumask_var(cpus); - - return bytes; -} - - -static ssize_t __cpu_set_write(struct kernfs_open_file *of, char *buf, - size_t bytes, loff_t off, bool is_list) -{ - cpumask_var_t cpus; - int err; - - if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) - return -ENOMEM; - if (is_list) - err = cpulist_parse(buf, cpus); - else - err = cpumask_parse(buf, cpus); - if (err) { - free_cpumask_var(cpus); - return err; - } - err = ghost_enclave_set_cpus(of_to_e(of), cpus); - free_cpumask_var(cpus); - return err ? err : bytes; -} - -/* Sets the enclave's cpus to buf, an ASCII list of cpus. */ -static ssize_t gf_cpulist_write(struct kernfs_open_file *of, char *buf, - size_t bytes, loff_t off) -{ - return __cpu_set_write(of, buf, bytes, off, /*is_list=*/true); -} - -static struct kernfs_ops gf_ops_e_cpulist = { - .open = gf_e_open, - .release = gf_e_release, - .read = gf_cpulist_read, - .write = gf_cpulist_write, -}; - -/* Returns an ASCII hex mask of the cpus in the enclave */ -static ssize_t gf_cpumask_read(struct kernfs_open_file *of, char *buf, - size_t bytes, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - unsigned long fl; - cpumask_var_t cpus; - char *mask_str; - int len; - - if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) - return -ENOMEM; - - spin_lock_irqsave(&e->lock, fl); - memcpy(cpus, &e->cpus, cpumask_size()); - spin_unlock_irqrestore(&e->lock, fl); - - /* - * +1 for the \0. We won't return the \0 to userspace, but the string - * will be null-terminated while in the kernel. - */ - len = snprintf(NULL, 0, "%*pb\n", cpumask_pr_args(cpus)) + 1; - mask_str = kmalloc(len, GFP_KERNEL); - if (!mask_str) { - free_cpumask_var(cpus); - return -ENOMEM; - } - len = snprintf(mask_str, len, "%*pb\n", cpumask_pr_args(cpus)); - bytes = memory_read_from_buffer(buf, bytes, &off, mask_str, len); - - kfree(mask_str); - free_cpumask_var(cpus); - - return bytes; -} - -/* Sets the enclave's cpus to buf, an ASCII hex cpumask. */ -static ssize_t gf_cpumask_write(struct kernfs_open_file *of, char *buf, - size_t bytes, loff_t off) -{ - return __cpu_set_write(of, buf, bytes, off, /*is_list=*/false); -} - -static struct kernfs_ops gf_ops_e_cpumask = { - .open = gf_e_open, - .release = gf_e_release, - .read = gf_cpumask_read, - .write = gf_cpumask_write, -}; - -static int gf_ctl_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%lu", e->id); - return 0; -} - -/* Called from the scheduler when it destroys the enclave. */ -void ghostfs_remove_enclave(struct ghost_enclave *e) -{ - kernfs_remove(e->enclave_dir); -} - -static void destroy_enclave(struct kernfs_open_file *ctl_of) -{ - struct kernfs_node *ctl = ctl_of->kn; - struct ghost_enclave *e = of_to_e(ctl_of); - - /* - * kernfs_remove() works recursively, removing the entire enclave - * directory. We need to remove the ctl file first, since we're in the - * middle of a kn op. - * - * Multiple threads can call kernfs_remove_self() at once. Whichever - * succeeds will remove the directory and release e. - */ - if (!kernfs_remove_self(ctl)) - return; - if (WARN_ON(ctl->parent != e->enclave_dir)) - return; - ghost_destroy_enclave(e); -} - -static struct ghost_sw_region *of_to_swr(struct kernfs_open_file *of) -{ - return of->kn->priv; -} - -/* - * swr_kns refcount the enclave, even though they point to the swr. An swr will - * never be deleted until the enclave is released. - */ -static int gf_swr_open(struct kernfs_open_file *of) -{ - struct ghost_sw_region *swr = of_to_swr(of); - - kref_get(&swr->enclave->kref); - return 0; -} - -static void gf_swr_release(struct kernfs_open_file *of) -{ - struct ghost_sw_region *swr = of_to_swr(of); - - kref_put(&swr->enclave->kref, enclave_release); -} - -static int gf_swr_mmap(struct kernfs_open_file *of, struct vm_area_struct *vma) -{ - struct ghost_sw_region *swr = of_to_swr(of); - - if (vma->vm_flags & VM_WRITE) - return -EINVAL; - vma->vm_flags &= ~VM_MAYWRITE; - - return ghost_region_mmap(of->file, vma, swr->header, of->kn->attr.size); -} - -static struct kernfs_ops gf_ops_e_swr = { - .open = gf_swr_open, - .release = gf_swr_release, - .mmap = gf_swr_mmap, -}; - -static int create_sw_region(struct kernfs_open_file *ctl_of, unsigned int id, - unsigned int numa_node) -{ - struct ghost_enclave *e = of_to_e(ctl_of); - struct kernfs_node *dir, *swr_kn; - struct ghost_sw_region *swr; - char name[31]; - int err; - - dir = kernfs_find_and_get(e->enclave_dir, "sw_regions"); - if (WARN_ON_ONCE(!dir)) - return -EINVAL; - - if (snprintf(name, sizeof(name), "sw_%u", id) >= sizeof(name)) { - err = -ENOSPC; - goto err_snprintf; - } - - swr_kn = kernfs_create_file(dir, name, 0440, 0, &gf_ops_e_swr, e); - if (IS_ERR(swr_kn)) { - err = PTR_ERR(swr_kn); - goto err_create_kn; - } - /* swr_kn is inaccessible until kernfs_activate. */ - - swr = ghost_create_sw_region(e, id, numa_node); - if (IS_ERR(swr)) { - err = PTR_ERR(swr); - goto err_create_swr; - } - - swr_kn->attr.size = swr->mmap_sz; - swr_kn->priv = swr; - kernfs_activate(swr_kn); - - return 0; -err_create_swr: - kernfs_remove(swr_kn); -err_create_kn: -err_snprintf: - kernfs_put(dir); - return err; -} - -static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, - size_t len, loff_t off) -{ - unsigned int arg1, arg2; - int err; - - /* - * Ignore the offset for ctl commands, so userspace doesn't have to - * worry about lseeking after each command. Userspace should submit a - * single write per command too, and not write("des"), write ("troy"). - * Otherwise they'll fail. - */ - - strip_slash_n(buf, len); - - if (!strcmp(buf, "destroy")) { - destroy_enclave(of); - } else if (sscanf(buf, "create sw_region %u %u", &arg1, &arg2) == 2) { - err = create_sw_region(of, arg1, arg2); - if (err) - return err; - } else { - pr_err("%s: bad cmd :%s:", __func__, buf); - return -EINVAL; - } - - return len; -} - -static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, - unsigned long arg) -{ - struct ghost_enclave *e = of_to_e(of); - - if (!(of->file->f_mode & FMODE_WRITE)) - return -EACCES; - switch (cmd) { - case GHOST_IOC_NULL: - return 0; - case GHOST_IOC_SW_GET_INFO: - return ghost_sw_get_info(e, - (struct ghost_ioc_sw_get_info __user *)arg); - case GHOST_IOC_SW_FREE: - return ghost_sw_free(e, (void __user *)arg); - case GHOST_IOC_CREATE_QUEUE: - return ghost_create_queue(e, - (struct ghost_ioc_create_queue __user *)arg); - case GHOST_IOC_ASSOC_QUEUE: - return ghost_associate_queue( - (struct ghost_ioc_assoc_queue __user *)arg); - case GHOST_IOC_SET_DEFAULT_QUEUE: - return ghost_set_default_queue(e, - (struct ghost_ioc_set_default_queue __user *)arg); - case GHOST_IOC_CONFIG_QUEUE_WAKEUP: - return ghost_config_queue_wakeup( - (struct ghost_ioc_config_queue_wakeup __user *)arg); - case GHOST_IOC_GET_CPU_TIME: - return ghost_get_cpu_time( - (struct ghost_ioc_get_cpu_time __user *)arg); - case GHOST_IOC_COMMIT_TXN: - return ioctl_ghost_commit_txn(e, - (struct ghost_ioc_commit_txn __user *)arg); - case GHOST_IOC_SYNC_GROUP_TXN: - return ghost_sync_group(e, - (struct ghost_ioc_commit_txn __user *)arg); - case GHOST_IOC_TIMERFD_SETTIME: - return ghost_timerfd_settime( - (struct ghost_ioc_timerfd_settime __user *)arg); - } - return -ENOIOCTLCMD; -} - -static struct kernfs_ops gf_ops_e_ctl = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_ctl_show, - .write = gf_ctl_write, - .ioctl = gf_ctl_ioctl, -}; - -/* - * Returns the enclave for f, if f is a ghostfs ctl file. We could support - * other file types, but since this is a backdoor into the FS, we only need to - * support ctl. - * - * Successful callers must call ghostfs_put_ctl_enclave(f). - * - * The kernfs_ops don't need this helper. kernfs manages the the refcounts. We - * need to do it manually here, because this is is a "backdoor" function to get - * the enclave pointer. That pointer is kept alive by kernfs. - */ -struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) -{ - struct kernfs_node *kn; - - kn = kernfs_node_from_file(f); - if (!kn) - return NULL; - if (kernfs_type(kn) != KERNFS_FILE) - return NULL; - if (kn->attr.ops != &gf_ops_e_ctl) - return NULL; - if (!kernfs_get_active(kn)) - return NULL; - WARN_ON(!kn->priv); - return kn->priv; -} - -/* Pair this with a successful ghostfs_ctl_to_enclave call. */ -void ghostfs_put_enclave_ctl(struct file *f) -{ - struct kernfs_node *kn; - - kn = kernfs_node_from_file(f); - if (WARN_ON(!kn)) - return; - if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) - return; - if (WARN_ON(kn->attr.ops != &gf_ops_e_ctl)) - return; - kernfs_put_active(kn); -} - -static int gf_runnable_timeout_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%lld", ktime_to_ms(READ_ONCE(e->max_unscheduled))); - return 0; -} - -static ssize_t gf_runnable_timeout_write(struct kernfs_open_file *of, char *buf, - size_t len, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - int err; - unsigned long msec; - - err = kstrtoul(buf, 0, &msec); - if (err) - return -EINVAL; - - WRITE_ONCE(e->max_unscheduled, ms_to_ktime(msec)); - - return len; -} - -static struct kernfs_ops gf_ops_e_runnable_timeout = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_runnable_timeout_show, - .write = gf_runnable_timeout_write, -}; - -static int gf_switchto_disabled_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%d", READ_ONCE(e->switchto_disabled)); - return 0; -} - -static ssize_t gf_switchto_disabled_write(struct kernfs_open_file *of, - char *buf, size_t len, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - int err; - int tunable; - - err = kstrtoint(buf, 0, &tunable); - if (err) - return -EINVAL; - - WRITE_ONCE(e->switchto_disabled, !!tunable); - - return len; -} - -static struct kernfs_ops gf_ops_e_switchto_disabled = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_switchto_disabled_show, - .write = gf_switchto_disabled_write, -}; - -static int gf_wake_on_waker_cpu_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%d", READ_ONCE(e->wake_on_waker_cpu)); - return 0; -} - -static ssize_t gf_wake_on_waker_cpu_write(struct kernfs_open_file *of, - char *buf, size_t len, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - int err; - int tunable; - - err = kstrtoint(buf, 0, &tunable); - if (err) - return -EINVAL; - - WRITE_ONCE(e->wake_on_waker_cpu, !!tunable); - - return len; -} - -static struct kernfs_ops gf_ops_e_wake_on_waker_cpu = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_wake_on_waker_cpu_show, - .write = gf_wake_on_waker_cpu_write, -}; - -static int gf_commit_at_tick_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%d", READ_ONCE(e->commit_at_tick)); - return 0; -} - -static ssize_t gf_commit_at_tick_write(struct kernfs_open_file *of, char *buf, - size_t len, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - int err; - int tunable; - - err = kstrtoint(buf, 0, &tunable); - if (err) - return -EINVAL; - - WRITE_ONCE(e->commit_at_tick, !!tunable); - - return len; -} - -static struct kernfs_ops gf_ops_e_commit_at_tick = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_commit_at_tick_show, - .write = gf_commit_at_tick_write, -}; - -static int gf_status_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - unsigned long fl; - bool is_active; - unsigned long nr_tasks; - - /* - * Userspace uses this to find any active enclave, since they don't have - * any other methods yet to know which enclave to use. - */ - spin_lock_irqsave(&e->lock, fl); - /* - * We don't need to lock to read agent_online, but eventually we'll - * check for the presence of an interstitial scheduler too. This status - * is for the *enclave*, not the *agent*. - */ - is_active = e->agent_online; - nr_tasks = e->nr_tasks; - spin_unlock_irqrestore(&e->lock, fl); - - seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); - seq_printf(sf, "nr_tasks %lu\n", nr_tasks); - return 0; -} - -static struct kernfs_ops gf_ops_e_status = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_status_show, -}; - -static struct gf_dirent enclave_dirtab[] = { - { - .name = "sw_regions", - .mode = 0555, - .is_dir = true, - }, - { - .name = "agent_online", - .mode = 0664, - .ops = &gf_ops_e_agent_online, - }, - { - .name = "cpu_data", - .mode = 0660, - .ops = &gf_ops_e_cpu_data, - }, - { - .name = "cpulist", - .mode = 0664, - .ops = &gf_ops_e_cpulist, - }, - { - .name = "cpumask", - .mode = 0664, - .ops = &gf_ops_e_cpumask, - }, - { - .name = "ctl", - .mode = 0664, - .ops = &gf_ops_e_ctl, - }, - { - .name = "runnable_timeout", - .mode = 0664, - .ops = &gf_ops_e_runnable_timeout, - }, - { - .name = "switchto_disabled", - .mode = 0664, - .ops = &gf_ops_e_switchto_disabled, - }, - { - .name = "wake_on_waker_cpu", - .mode = 0664, - .ops = &gf_ops_e_wake_on_waker_cpu, - }, - { - .name = "commit_at_tick", - .mode = 0664, - .ops = &gf_ops_e_commit_at_tick, - }, - { - .name = "status", - .mode = 0444, - .ops = &gf_ops_e_status, - }, - {0}, -}; - -/* Caller is responsible for cleanup. Removing the parent will suffice. */ -static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, - struct ghost_enclave *priv) -{ - struct gf_dirent *gft; - struct kernfs_node *kn; - - for (gft = dirtab; gft->name; gft++) { - if (gft->is_dir) { - kn = kernfs_create_dir(parent, gft->name, gft->mode, - NULL); - } else { - kn = kernfs_create_file(parent, gft->name, gft->mode, - gft->size, gft->ops, priv); - } - if (IS_ERR(kn)) - return PTR_ERR(kn); - } - return 0; -} - -static int make_enclave(struct kernfs_node *parent, unsigned long id) -{ - struct kernfs_node *dir; - struct ghost_enclave *e; - char name[31]; - int ret; - - /* - * ghost_create_enclave() is mostly just "alloc and initialize". - * Anything done by it gets undone in enclave_release, and it is not - * discoverable, usable, or otherwise hooked into the kernel until - * kernfs_active(). - */ - e = ghost_create_enclave(); - if (!e) - return -ENOMEM; - e->id = id; - if (snprintf(name, sizeof(name), "enclave_%lu", id) >= sizeof(name)) { - ret = -ENOSPC; - goto out_e; - } - - dir = kernfs_create_dir(parent, name, 0555, NULL); - if (IS_ERR(dir)) { - ret = PTR_ERR(dir); - goto out_e; - } - e->enclave_dir = dir; - - ret = gf_add_files(dir, enclave_dirtab, e); - if (ret) - goto out_dir; - - /* - * Once the enclave has been activated, it is available to userspace and - * can be used for scheduling. After that, we must destroy it by - * calling ghost_destroy_enclave(), not by releasing the reference. - */ - kernfs_activate(dir); /* recursive */ - - return 0; - -out_dir: - kernfs_remove(dir); /* recursive */ -out_e: - kref_put(&e->kref, enclave_release); - return ret; -} - -static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, - size_t len, loff_t off) -{ - struct kernfs_node *ctl = of->kn; - struct kernfs_node *top_dir = ctl->parent; - unsigned long x; - int ret; - - strip_slash_n(buf, len); - - /* This will ignore any extra digits or characters beyond the %u. */ - ret = sscanf(buf, "create %lu", &x); - if (ret == 1) { - ret = make_enclave(top_dir, x); - return ret ? ret : len; - } - - return -EINVAL; -} - -static struct kernfs_ops gf_ops_top_ctl = { - .write = gf_top_ctl_write, -}; - -static int gf_top_version_show(struct seq_file *sf, void *v) -{ - seq_printf(sf, "%u", GHOST_VERSION); - return 0; -} - -static struct kernfs_ops gf_ops_top_version = { - .seq_show = gf_top_version_show, -}; - -static struct gf_dirent top_dirtab[] = { - { - .name = "ctl", - .mode = 0660, - .ops = &gf_ops_top_ctl, - }, - { - .name = "version", - .mode = 0444, - .ops = &gf_ops_top_version, - }, - {0} -}; - -/* - * Most gf_dirent file sizes are not known at compile time. Most don't matter - * for sysfs and we can leave them as 0. But for anything that gets mmapped, - * it's convenient for userspace for the kernel to say how big it is. - */ -static void runtime_adjust_dirtabs(void) -{ - struct gf_dirent *enc_txn; - - const loff_t GHOST_CPU_DATA_REGION_SIZE = - sizeof(struct ghost_cpu_data) * num_possible_cpus(); - - enc_txn = gf_find_file(enclave_dirtab, "cpu_data"); - if (WARN_ON(!enc_txn)) - return; - enc_txn->size = GHOST_CPU_DATA_REGION_SIZE; -} - -static int __init ghost_setup_root(void) -{ - int ret; - struct kernfs_root *fs_root; - - fs_root = kernfs_create_root(NULL, KERNFS_ROOT_CREATE_DEACTIVATED | - KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, NULL); - if (IS_ERR(fs_root)) - return PTR_ERR(fs_root); - - ret = gf_add_files(fs_root->kn, top_dirtab, NULL); - if (ret) { - kernfs_destroy_root(fs_root); - return ret; - } - - ghost_kfs_root = fs_root; - - runtime_adjust_dirtabs(); - - kernfs_activate(ghost_kfs_root->kn); - - return ret; -} - -static int ghost_get_tree(struct fs_context *fc) -{ - int ret; - - ret = kernfs_get_tree(fc); - if (ret) - return ret; - - return 0; -} - -static void ghost_fs_context_free(struct fs_context *fc) -{ - struct kernfs_fs_context *kfc = fc->fs_private; - - kernfs_free_fs_context(fc); - kfree(kfc); -} - -static const struct fs_context_operations ghost_fs_context_ops = { - .free = ghost_fs_context_free, - .get_tree = ghost_get_tree, -}; - -static int ghost_init_fs_context(struct fs_context *fc) -{ - struct kernfs_fs_context *kfc; - - /* Technically, this should be in uapi/linux/magic.h. */ - #define GHOST_SUPER_MAGIC 0xBAD1DEA2 - - kfc = kzalloc(sizeof(struct kernfs_fs_context), GFP_KERNEL); - if (!kfc) - return -ENOMEM; - - kfc->root = ghost_kfs_root; - kfc->magic = GHOST_SUPER_MAGIC; - fc->fs_private = kfc; - fc->ops = &ghost_fs_context_ops; - put_user_ns(fc->user_ns); - fc->user_ns = get_user_ns(&init_user_ns); - fc->global = true; - return 0; -} - -static void ghost_kill_sb(struct super_block *sb) -{ - kernfs_kill_sb(sb); -} - -static struct file_system_type ghost_fs_type = { - .name = "ghost", - .init_fs_context = ghost_init_fs_context, - .kill_sb = ghost_kill_sb, -}; - -static int __init ghostfs_init(void) -{ - int ret = 0; - - ret = ghost_setup_root(); - if (ret) - return ret; - - ret = sysfs_create_mount_point(fs_kobj, "ghost"); - if (ret) - goto cleanup_root; - - ret = register_filesystem(&ghost_fs_type); - if (ret) - goto cleanup_mountpoint; - - return 0; - -cleanup_mountpoint: - sysfs_remove_mount_point(fs_kobj, "ghost"); -cleanup_root: - kernfs_destroy_root(ghost_kfs_root); - ghost_kfs_root = NULL; - - return ret; -} - -static void __exit ghostfs_exit(void) -{ - unregister_filesystem(&ghost_fs_type); - sysfs_remove_mount_point(fs_kobj, "ghost"); - kernfs_destroy_root(ghost_kfs_root); -} - -late_initcall(ghostfs_init); -__exitcall(ghostfs_exit); From d3f5839aec3c5cbe4b4b422036f7d23eda8b8ffa Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 24 Oct 2021 17:31:54 -0700 Subject: [PATCH 075/165] sched: restrict symbols leaked into global scope from ghost.c Following the merge of contents of ghostfs.c into ghost.c we don't need global linkage to communicate between functions that were originally in different compilation units (ghostfs.c and ghost.c). Make these functions static since they are only referenced from a single compilation unit (ghost.c) While here replace calls to 'ghostfs_remove_enclave(e)' directly with 'kernfs_remove(e->enclave_dir)'. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I4f77885af4cbcbaea12a3be50756589d8c968501 --- kernel/sched/ghost.c | 75 +++++++++++++++++++++++++------------------- kernel/sched/ghost.h | 43 +------------------------ 2 files changed, 43 insertions(+), 75 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 99ae91bbedb0..3fa7804cb17d 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -62,6 +62,14 @@ static DEFINE_PER_CPU(struct bpf_ghost_msg, bpf_ghost_msg); #endif #define SG_COOKIE_CPU_SHIFT (63 - SG_COOKIE_CPU_BITS) /* MSB=0 */ +struct ghost_sw_region { + struct list_head list; /* ghost_enclave glue */ + uint32_t alloc_scan_start; /* allocator starts scan here */ + struct ghost_sw_region_header *header; /* pointer to vmalloc memory */ + size_t mmap_sz; /* size of mmapped region */ + struct ghost_enclave *enclave; +}; + /* The load contribution that CFS sees for a running ghOSt task */ unsigned long sysctl_ghost_cfs_load_added = 1024; @@ -103,6 +111,10 @@ static void __enclave_remove_task(struct ghost_enclave *e, static int __sw_region_free(struct ghost_sw_region *region); static const struct file_operations queue_fops; +static struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); +static void ghostfs_put_enclave_ctl(struct file *f); +static void ghost_destroy_enclave(struct ghost_enclave *e); + /* True if X and Y have the same enclave, including having no enclave. */ static bool check_same_enclave(int cpu_x, int cpu_y) { @@ -1642,8 +1654,8 @@ int _ghost_mmap_common(struct vm_area_struct *vma, ulong mapsize) * * 'addr' must have been obtained from vmalloc_user(). */ -int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, - void *addr, ulong mapsize) +static int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, + void *addr, ulong mapsize) { int error; @@ -1654,8 +1666,8 @@ int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, return error; } -int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, - struct ghost_cpu_data **cpu_data, ulong mapsize) +static int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, + struct ghost_cpu_data **cpu_data, ulong mapsize) { int error; unsigned long uaddr; @@ -1937,7 +1949,7 @@ static void enclave_destroyer(struct work_struct *work) kref_put(&e->kref, enclave_release); } -struct ghost_enclave *ghost_create_enclave(void) +static struct ghost_enclave *ghost_create_enclave(void) { struct ghost_enclave *e = kzalloc(sizeof(*e), GFP_KERNEL); bool vmalloc_failed = false; @@ -2102,7 +2114,7 @@ void ghost_destroy_enclave(struct ghost_enclave *e) * still be open FDs, each of which holds a reference. When the last FD * is closed, we'll release and free. */ - ghostfs_remove_enclave(e); + kernfs_remove(e->enclave_dir); kref_put(&e->kref, enclave_release); } @@ -2112,7 +2124,8 @@ void ghost_destroy_enclave(struct ghost_enclave *e) * Returns an error code on failure and does not change the enclave. Will fail * if a cpu is in another enclave. */ -int ghost_enclave_set_cpus(struct ghost_enclave *e, const struct cpumask *cpus) +static int ghost_enclave_set_cpus(struct ghost_enclave *e, + const struct cpumask *cpus) { unsigned long flags; unsigned int cpu; @@ -2727,9 +2740,9 @@ static size_t calculate_sw_region_size(void) * * Returns an ERR_PTR on error. */ -struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, - unsigned int id, - unsigned int node) +static struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, + unsigned int id, + unsigned int node) { struct ghost_sw_region *swr; struct ghost_sw_region_header *h; @@ -3095,8 +3108,8 @@ static const struct file_operations queue_fops = { .mmap = queue_mmap, }; -int ghost_create_queue(struct ghost_enclave *e, - struct ghost_ioc_create_queue __user *arg) +static int ghost_create_queue(struct ghost_enclave *e, + struct ghost_ioc_create_queue __user *arg) { ulong size; int error = 0, fd, elems, node, flags; @@ -3289,8 +3302,8 @@ static int _get_sw_info(struct ghost_enclave *e, return error; } -int ghost_sw_get_info(struct ghost_enclave *e, - struct ghost_ioc_sw_get_info __user *arg) +static int ghost_sw_get_info(struct ghost_enclave *e, + struct ghost_ioc_sw_get_info __user *arg) { int error = -EINVAL; struct task_struct *p; @@ -3342,7 +3355,8 @@ int ghost_sw_get_info(struct ghost_enclave *e, return error; } -int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo) +static int ghost_sw_free(struct ghost_enclave *e, + struct ghost_sw_info __user *uinfo) { int error; bool gated; @@ -3381,7 +3395,7 @@ int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo) return error; } -int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg) +static int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg) { int error = 0; struct rq *rq; @@ -3501,8 +3515,8 @@ int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg) return error; } -int ghost_set_default_queue(struct ghost_enclave *e, - struct ghost_ioc_set_default_queue __user *arg) +static int ghost_set_default_queue(struct ghost_enclave *e, + struct ghost_ioc_set_default_queue __user *arg) { int error = 0; struct fd f_que = {0}; @@ -3609,7 +3623,8 @@ static int agent_target_cpu(struct rq *rq) return target_cpu(agent->ghost.dst_q, task_cpu(agent)); } -int ghost_config_queue_wakeup(struct ghost_ioc_config_queue_wakeup __user *arg) +static int ghost_config_queue_wakeup( + struct ghost_ioc_config_queue_wakeup __user *arg) { struct ghost_queue *q; struct queue_notifier *old, *qn = NULL; @@ -3691,7 +3706,7 @@ int ghost_config_queue_wakeup(struct ghost_ioc_config_queue_wakeup __user *arg) return ret; } -int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg) +static int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg) { struct task_struct *p; u64 time; @@ -5774,8 +5789,8 @@ static inline int64_t ghost_sync_group_cookie(void) return val; } -int ghost_sync_group(struct ghost_enclave *e, - struct ghost_ioc_commit_txn __user *arg) +static int ghost_sync_group(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg) { int64_t target; bool failed = false; @@ -5985,8 +6000,8 @@ int ghost_sync_group(struct ghost_enclave *e, return !failed; } -int ioctl_ghost_commit_txn(struct ghost_enclave *e, - struct ghost_ioc_commit_txn __user *arg) +static int ioctl_ghost_commit_txn(struct ghost_enclave *e, + struct ghost_ioc_commit_txn __user *arg) { int error, state; int cpu, this_cpu; @@ -6156,7 +6171,7 @@ static int ghost_timerfd_validate(struct timerfd_ghost *timerfd_ghost) int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, struct itimerspec64 *old, struct timerfd_ghost *tfdl); -int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) +static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) { struct itimerspec64 new, old; int ret; @@ -6716,12 +6731,6 @@ static int gf_ctl_show(struct seq_file *sf, void *v) return 0; } -/* Called from the scheduler when it destroys the enclave. */ -void ghostfs_remove_enclave(struct ghost_enclave *e) -{ - kernfs_remove(e->enclave_dir); -} - static void destroy_enclave(struct kernfs_open_file *ctl_of) { struct kernfs_node *ctl = ctl_of->kn; @@ -6918,7 +6927,7 @@ static struct kernfs_ops gf_ops_e_ctl = { * need to do it manually here, because this is is a "backdoor" function to get * the enclave pointer. That pointer is kept alive by kernfs. */ -struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) +static struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) { struct kernfs_node *kn; @@ -6936,7 +6945,7 @@ struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) } /* Pair this with a successful ghostfs_ctl_to_enclave call. */ -void ghostfs_put_enclave_ctl(struct file *f) +static void ghostfs_put_enclave_ctl(struct file *f) { struct kernfs_node *kn; diff --git a/kernel/sched/ghost.h b/kernel/sched/ghost.h index 6720af394039..7fae64630a7f 100644 --- a/kernel/sched/ghost.h +++ b/kernel/sched/ghost.h @@ -14,50 +14,9 @@ #define _KERNEL_SCHED_GHOST_H_ #ifndef __INCLUDE_KERNEL_SCHED_GHOST -#error "ghost.h can only be included from ghost.c or ghostfs.c" +#error "ghost.h can only be included from ghost.c" #endif -struct ghost_sw_region { - struct list_head list; /* ghost_enclave glue */ - uint32_t alloc_scan_start; /* allocator starts scan here */ - struct ghost_sw_region_header *header; /* pointer to vmalloc memory */ - size_t mmap_sz; /* size of mmapped region */ - struct ghost_enclave *enclave; -}; - -/* In kernel/sched/ghostfs.c */ -struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); -void ghostfs_put_enclave_ctl(struct file *f); -void ghostfs_remove_enclave(struct ghost_enclave *e); - -/* In kernel/sched/ghost.c */ -struct ghost_enclave *ghost_create_enclave(void); -void ghost_destroy_enclave(struct ghost_enclave *e); -int ghost_enclave_set_cpus(struct ghost_enclave *e, const struct cpumask *cpus); -int ghost_region_mmap(struct file *file, struct vm_area_struct *vma, - void *addr, ulong mapsize); -int ghost_cpu_data_mmap(struct file *file, struct vm_area_struct *vma, - struct ghost_cpu_data **cpu_data, ulong mapsize); -struct ghost_sw_region *ghost_create_sw_region(struct ghost_enclave *e, - unsigned int id, - unsigned int node); - -int ghost_sw_get_info(struct ghost_enclave *e, - struct ghost_ioc_sw_get_info __user *arg); -int ghost_sw_free(struct ghost_enclave *e, struct ghost_sw_info __user *uinfo); -int ghost_create_queue(struct ghost_enclave *e, - struct ghost_ioc_create_queue __user *arg); -int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg); -int ghost_set_default_queue(struct ghost_enclave *e, - struct ghost_ioc_set_default_queue __user *arg); -int ghost_config_queue_wakeup(struct ghost_ioc_config_queue_wakeup __user *arg); -int ghost_get_cpu_time(struct ghost_ioc_get_cpu_time __user *arg); -int ioctl_ghost_commit_txn(struct ghost_enclave *e, - struct ghost_ioc_commit_txn __user *arg); -int ghost_sync_group(struct ghost_enclave *e, - struct ghost_ioc_commit_txn __user *arg); -int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg); - /* Stub functions for open source ghOSt code. */ /* These functions should not be open sourced. */ From d877db6ac844cca7c0760cb2e2382ddd996ba18b Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 24 Oct 2021 19:56:07 -0700 Subject: [PATCH 076/165] sched: beginnings of ghost_abi Annotate all 'struct ghost_abi' so they are emitted in a named section (".ghost_abi") so they can be later collected together by the linker. The linker-defined symbols __begin_ghost_abi and __end_ghost_abi are the bookends of the ghost_abi collection. Move the implementation of the ghostfs top-level directory into ghost_core.c (this part is abi-independent). Use the ghost_abi to do abi-specific initialization and enclave creation. Add a "version " field to the "status" file in the enclave directory. Tested: all unit tests passing in virtme. Effort: sched/ghost/abi Change-Id: I30a0fd8f1fd322554fdef4f1638908d2c2781391 --- include/asm-generic/vmlinux.lds.h | 11 ++ kernel/sched/Makefile | 1 + kernel/sched/ghost.c | 308 ++++++------------------------ kernel/sched/ghost_core.c | 291 ++++++++++++++++++++++++++++ kernel/sched/sched.h | 14 ++ 5 files changed, 379 insertions(+), 246 deletions(-) create mode 100644 kernel/sched/ghost_core.c diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 6bb060dc2f19..7ff1fde7ea4d 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -423,6 +423,16 @@ __end_ro_after_init = .; #endif +#ifdef CONFIG_SCHED_CLASS_GHOST +#define GHOST_ABI_RODATA \ + STRUCT_ALIGN(); \ + __begin_ghost_abi = .; \ + *(.rodata.ghost_abi) \ + __end_ghost_abi = .; +#else +#define GHOST_ABI_RODATA +#endif + /* * Read only Data */ @@ -430,6 +440,7 @@ . = ALIGN((align)); \ .rodata : AT(ADDR(.rodata) - LOAD_OFFSET) { \ __start_rodata = .; \ + GHOST_ABI_RODATA \ *(.rodata) *(.rodata.*) \ SCHED_DATA \ RO_AFTER_INIT_DATA /* Read only after init */ \ diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 4008a8251d4e..d20bf0aa6786 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -37,4 +37,5 @@ obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost.o +obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost_core.o obj-$(CONFIG_BPF) += bpf.o diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 3fa7804cb17d..8270954925f9 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1949,61 +1949,13 @@ static void enclave_destroyer(struct work_struct *work) kref_put(&e->kref, enclave_release); } -static struct ghost_enclave *ghost_create_enclave(void) -{ - struct ghost_enclave *e = kzalloc(sizeof(*e), GFP_KERNEL); - bool vmalloc_failed = false; - int cpu; - - BUILD_BUG_ON(sizeof(struct ghost_cpu_data) != PAGE_SIZE); - - if (!e) - return NULL; - spin_lock_init(&e->lock); - kref_init(&e->kref); - INIT_LIST_HEAD(&e->sw_region_list); - INIT_LIST_HEAD(&e->task_list); - INIT_LIST_HEAD(&e->inhibited_task_list); - INIT_LIST_HEAD(&e->ew.link); - INIT_WORK(&e->task_reaper, enclave_reap_tasks); - INIT_WORK(&e->enclave_actual_release, enclave_actual_release); - INIT_WORK(&e->enclave_destroyer, enclave_destroyer); - - e->cpu_data = kcalloc(num_possible_cpus(), - sizeof(struct ghost_cpu_data *), GFP_KERNEL); - if (!e->cpu_data) { - kfree(e); - return NULL; - } - - for_each_possible_cpu(cpu) { - e->cpu_data[cpu] = vmalloc_user_node_flags(PAGE_SIZE, - cpu_to_node(cpu), - GFP_KERNEL); - if (e->cpu_data[cpu] == NULL) { - vmalloc_failed = true; - break; - } - } - - if (vmalloc_failed) { - for_each_possible_cpu(cpu) - vfree(e->cpu_data[cpu]); - kfree(e->cpu_data); - kfree(e); - return NULL; - } - - return e; -} - /* * Here we can trigger any destruction we want, such as killing agents, kicking * tasks to CFS, etc. Undo things that were done at runtime. * * The enclave itself isn't freed until enclave_release(). */ -void ghost_destroy_enclave(struct ghost_enclave *e) +static void ghost_destroy_enclave(struct ghost_enclave *e) { ulong flags; int cpu; @@ -6425,8 +6377,6 @@ static void strip_slash_n(char *buf, size_t count) *n = '\0'; } -static struct kernfs_root *ghost_kfs_root; - struct gf_dirent { char *name; umode_t mode; @@ -7104,6 +7054,7 @@ static int gf_status_show(struct seq_file *sf, void *v) nr_tasks = e->nr_tasks; spin_unlock_irqrestore(&e->lock, fl); + seq_printf(sf, "version %u\n", GHOST_VERSION); seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); seq_printf(sf, "nr_tasks %lu\n", nr_tasks); return 0; @@ -7195,103 +7146,6 @@ static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, return 0; } -static int make_enclave(struct kernfs_node *parent, unsigned long id) -{ - struct kernfs_node *dir; - struct ghost_enclave *e; - char name[31]; - int ret; - - /* - * ghost_create_enclave() is mostly just "alloc and initialize". - * Anything done by it gets undone in enclave_release, and it is not - * discoverable, usable, or otherwise hooked into the kernel until - * kernfs_active(). - */ - e = ghost_create_enclave(); - if (!e) - return -ENOMEM; - e->id = id; - if (snprintf(name, sizeof(name), "enclave_%lu", id) >= sizeof(name)) { - ret = -ENOSPC; - goto out_e; - } - - dir = kernfs_create_dir(parent, name, 0555, NULL); - if (IS_ERR(dir)) { - ret = PTR_ERR(dir); - goto out_e; - } - e->enclave_dir = dir; - - ret = gf_add_files(dir, enclave_dirtab, e); - if (ret) - goto out_dir; - - /* - * Once the enclave has been activated, it is available to userspace and - * can be used for scheduling. After that, we must destroy it by - * calling ghost_destroy_enclave(), not by releasing the reference. - */ - kernfs_activate(dir); /* recursive */ - - return 0; - -out_dir: - kernfs_remove(dir); /* recursive */ -out_e: - kref_put(&e->kref, enclave_release); - return ret; -} - -static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, - size_t len, loff_t off) -{ - struct kernfs_node *ctl = of->kn; - struct kernfs_node *top_dir = ctl->parent; - unsigned long x; - int ret; - - strip_slash_n(buf, len); - - /* This will ignore any extra digits or characters beyond the %u. */ - ret = sscanf(buf, "create %lu", &x); - if (ret == 1) { - ret = make_enclave(top_dir, x); - return ret ? ret : len; - } - - return -EINVAL; -} - -static struct kernfs_ops gf_ops_top_ctl = { - .write = gf_top_ctl_write, -}; - -static int gf_top_version_show(struct seq_file *sf, void *v) -{ - seq_printf(sf, "%u", GHOST_VERSION); - return 0; -} - -static struct kernfs_ops gf_ops_top_version = { - .seq_show = gf_top_version_show, -}; - -static struct gf_dirent top_dirtab[] = { - { - .name = "ctl", - .mode = 0660, - .ops = &gf_ops_top_ctl, - }, - { - .name = "version", - .mode = 0444, - .ops = &gf_ops_top_version, - }, - {0} -}; - /* * Most gf_dirent file sizes are not known at compile time. Most don't matter * for sysfs and we can leave them as 0. But for anything that gets mmapped, @@ -7310,120 +7164,82 @@ static void runtime_adjust_dirtabs(void) enc_txn->size = GHOST_CPU_DATA_REGION_SIZE; } -static int __init ghost_setup_root(void) +static int __init abi_init(ghost_abi_ptr_t abi) { - int ret; - struct kernfs_root *fs_root; - - fs_root = kernfs_create_root(NULL, KERNFS_ROOT_CREATE_DEACTIVATED | - KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, NULL); - if (IS_ERR(fs_root)) - return PTR_ERR(fs_root); - - ret = gf_add_files(fs_root->kn, top_dirtab, NULL); - if (ret) { - kernfs_destroy_root(fs_root); - return ret; - } - - ghost_kfs_root = fs_root; + if (WARN_ON_ONCE(abi->version != GHOST_VERSION)) + return -1; runtime_adjust_dirtabs(); - - kernfs_activate(ghost_kfs_root->kn); - - return ret; -} - -static int ghost_get_tree(struct fs_context *fc) -{ - int ret; - - /* Technically, this should be in uapi/linux/magic.h. */ - #define GHOST_SUPER_MAGIC 0xBAD1DEA2 - - VM_BUG_ON(!ghost_kfs_root); - ret = kernfs_get_tree(fc); - if (ret) - WARN(1, "Failed to mount ghostfs!"); - return ret; -} - -static void ghost_fs_context_free(struct fs_context *fc) -{ - struct kernfs_fs_context *kfc = fc->fs_private; - - kernfs_free_fs_context(fc); - kfree(kfc); + return 0; } -static const struct fs_context_operations ghost_fs_context_ops = { - .free = ghost_fs_context_free, - .get_tree = ghost_get_tree, -}; - -static int ghost_init_fs_context(struct fs_context *fc) +static struct ghost_enclave *create_enclave(ghost_abi_ptr_t abi, + struct kernfs_node *dir, + ulong id) { - struct kernfs_fs_context *kfc; - - kfc = kzalloc(sizeof(struct kernfs_fs_context), GFP_KERNEL); - if (!kfc) - return -ENOMEM; + bool vmalloc_failed = false; + struct ghost_enclave *e; + int cpu, ret; - kfc->root = ghost_kfs_root; - kfc->magic = GHOST_SUPER_MAGIC; - fc->fs_private = kfc; - fc->ops = &ghost_fs_context_ops; - put_user_ns(fc->user_ns); - fc->user_ns = get_user_ns(&init_user_ns); - fc->global = true; - return 0; -} + BUILD_BUG_ON(sizeof(struct ghost_cpu_data) != PAGE_SIZE); -static void ghost_kill_sb(struct super_block *sb) -{ - kernfs_kill_sb(sb); -} + if (WARN_ON_ONCE(abi->version != GHOST_VERSION)) + return ERR_PTR(-EINVAL); -static struct file_system_type ghost_fs_type = { - .name = "ghost", - .init_fs_context = ghost_init_fs_context, - .kill_sb = ghost_kill_sb, -}; + e = kzalloc(sizeof(*e), GFP_KERNEL); + if (e == NULL) + return ERR_PTR(-ENOMEM); -static int __init ghostfs_init(void) -{ - int ret = 0; + spin_lock_init(&e->lock); + kref_init(&e->kref); + INIT_LIST_HEAD(&e->sw_region_list); + INIT_LIST_HEAD(&e->task_list); + INIT_LIST_HEAD(&e->inhibited_task_list); + INIT_LIST_HEAD(&e->ew.link); + INIT_WORK(&e->task_reaper, enclave_reap_tasks); + INIT_WORK(&e->enclave_actual_release, enclave_actual_release); + INIT_WORK(&e->enclave_destroyer, enclave_destroyer); - ret = ghost_setup_root(); - if (ret) - return ret; + e->cpu_data = kcalloc(num_possible_cpus(), + sizeof(struct ghost_cpu_data *), GFP_KERNEL); + if (!e->cpu_data) { + kfree(e); + return ERR_PTR(-ENOMEM); + } - ret = sysfs_create_mount_point(fs_kobj, "ghost"); - if (ret) - goto cleanup_root; + for_each_possible_cpu(cpu) { + e->cpu_data[cpu] = vmalloc_user_node_flags(PAGE_SIZE, + cpu_to_node(cpu), + GFP_KERNEL); + if (e->cpu_data[cpu] == NULL) { + vmalloc_failed = true; + break; + } + } - ret = register_filesystem(&ghost_fs_type); - if (ret) - goto cleanup_mountpoint; + if (vmalloc_failed) { + for_each_possible_cpu(cpu) + vfree(e->cpu_data[cpu]); + kfree(e->cpu_data); + kfree(e); + return ERR_PTR(-ENOMEM); + } - return 0; + ret = gf_add_files(dir, enclave_dirtab, e); + if (ret) { + kref_put(&e->kref, enclave_release); + return ERR_PTR(ret); + } -cleanup_mountpoint: - sysfs_remove_mount_point(fs_kobj, "ghost"); -cleanup_root: - kernfs_destroy_root(ghost_kfs_root); - ghost_kfs_root = NULL; + e->id = id; + e->enclave_dir = dir; - return ret; + return e; } -static void __exit ghostfs_exit(void) -{ - unregister_filesystem(&ghost_fs_type); - sysfs_remove_mount_point(fs_kobj, "ghost"); - kernfs_destroy_root(ghost_kfs_root); -} +DEFINE_GHOST_ABI(current_abi) = { + .version = GHOST_VERSION, + .abi_init = abi_init, + .create_enclave = create_enclave, +}; -late_initcall(ghostfs_init); -__exitcall(ghostfs_exit); diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c new file mode 100644 index 000000000000..e933cb3e3eaf --- /dev/null +++ b/kernel/sched/ghost_core.c @@ -0,0 +1,291 @@ +#include + +#include "sched.h" +#include "ghost_uapi.h" + +static struct kernfs_root *ghost_kfs_root; + +extern const struct ghost_abi __begin_ghost_abi[]; +extern const struct ghost_abi __end_ghost_abi[]; + +#define first_ghost_abi (&__begin_ghost_abi[0]) +#define last_ghost_abi (&__end_ghost_abi[-1]) + +#define for_each_abi(abi) \ + for (abi = first_ghost_abi; abi <= last_ghost_abi; abi++) + +struct gf_dirent { + char *name; + umode_t mode; + struct kernfs_ops *ops; + loff_t size; + bool is_dir; +}; + +/* Helper for when you "echo foo > ctl" without the -n. */ +static void gf_strip_slash_n(char *buf, size_t count) +{ + char *n = strnchr(buf, count, '\n'); + + if (n) + *n = '\0'; +} + +static ghost_abi_ptr_t ghost_abi_lookup(uint version) +{ + ghost_abi_ptr_t abi; + + for_each_abi(abi) { + if (abi->version == version) + return abi; + } + + return NULL; +} + +static int make_enclave(struct kernfs_node *parent, unsigned long id, + unsigned int version) +{ + struct kernfs_node *dir; + struct ghost_enclave *e; + ghost_abi_ptr_t abi; + char name[31]; + + abi = ghost_abi_lookup(version); + if (!abi) + return -ESRCH; + + if (snprintf(name, sizeof(name), "enclave_%lu", id) >= sizeof(name)) + return -ENOSPC; + + dir = kernfs_create_dir(parent, name, 0555, NULL); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + /* + * ghost_create_enclave() is mostly just "alloc and initialize". + * Anything done by it gets undone in enclave_release, and it is not + * discoverable, usable, or otherwise hooked into the kernel until + * kernfs_active(). + */ + e = abi->create_enclave(abi, dir, id); + if (IS_ERR(e)) { + kernfs_remove(dir); /* recursive */ + return PTR_ERR(e); + } + + /* + * Once the enclave has been activated, it is available to userspace and + * can be used for scheduling. After that, we must destroy it by + * calling ghost_destroy_enclave(), not by releasing the reference. + */ + kernfs_activate(dir); /* recursive */ + return 0; +} + +static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct kernfs_node *ctl = of->kn; + struct kernfs_node *top_dir = ctl->parent; + unsigned long x; + int ret; + + gf_strip_slash_n(buf, len); + + /* This will ignore any extra digits or characters beyond the %u. */ + ret = sscanf(buf, "create %lu", &x); + if (ret == 1) { + ret = make_enclave(top_dir, x, GHOST_VERSION); + return ret ? ret : len; + } + + return -EINVAL; +} + +static struct kernfs_ops gf_ops_top_ctl = { + .write = gf_top_ctl_write, +}; + +static int gf_top_version_show(struct seq_file *sf, void *v) +{ + /* For now we only advertise a single abi from the kernel */ + WARN_ON_ONCE(first_ghost_abi != last_ghost_abi); + + seq_printf(sf, "%u", first_ghost_abi->version); + return 0; +} + +static struct kernfs_ops gf_ops_top_version = { + .seq_show = gf_top_version_show, +}; + +static struct gf_dirent top_dirtab[] = { + { + .name = "ctl", + .mode = 0660, + .ops = &gf_ops_top_ctl, + }, + { + .name = "version", + .mode = 0444, + .ops = &gf_ops_top_version, + }, + {0} +}; + +/* Caller is responsible for cleanup. Removing the parent will suffice. */ +static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, + void *priv) +{ + struct gf_dirent *gft; + struct kernfs_node *kn; + + for (gft = dirtab; gft->name; gft++) { + if (gft->is_dir) { + kn = kernfs_create_dir(parent, gft->name, gft->mode, + NULL); + } else { + kn = kernfs_create_file(parent, gft->name, gft->mode, + gft->size, gft->ops, priv); + } + if (IS_ERR(kn)) + return PTR_ERR(kn); + } + return 0; +} + +static void __init ghost_abi_init(void) +{ + ghost_abi_ptr_t abi; + int ret; + + for_each_abi(abi) { + ret = abi->abi_init(abi); + if (ret) { + WARN(true, "Skipping ghost abi %d due to error %d", + abi->version, ret); + continue; + } + } +} + +static int __init ghost_setup_root(void) +{ + int ret; + struct kernfs_root *fs_root; + + fs_root = kernfs_create_root(NULL, KERNFS_ROOT_CREATE_DEACTIVATED | + KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, NULL); + if (IS_ERR(fs_root)) + return PTR_ERR(fs_root); + + ret = gf_add_files(fs_root->kn, top_dirtab, NULL); + if (ret) { + kernfs_destroy_root(fs_root); + return ret; + } + + ghost_kfs_root = fs_root; + + ghost_abi_init(); + + kernfs_activate(ghost_kfs_root->kn); + + return ret; +} + +#include "../../fs/kernfs/kernfs-internal.h" + +static int ghost_get_tree(struct fs_context *fc) +{ + int ret; + + /* Technically, this should be in uapi/linux/magic.h. */ + #define GHOST_SUPER_MAGIC 0xBAD1DEA2 + + VM_BUG_ON(!ghost_kfs_root); + ret = kernfs_get_tree(fc); + if (ret) + WARN(1, "Failed to mount ghostfs!"); + return ret; +} + +static void ghost_fs_context_free(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + + kernfs_free_fs_context(fc); + kfree(kfc); +} + +static const struct fs_context_operations ghost_fs_context_ops = { + .free = ghost_fs_context_free, + .get_tree = ghost_get_tree, +}; + +static int ghost_init_fs_context(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc; + + kfc = kzalloc(sizeof(struct kernfs_fs_context), GFP_KERNEL); + if (!kfc) + return -ENOMEM; + + kfc->root = ghost_kfs_root; + kfc->magic = GHOST_SUPER_MAGIC; + fc->fs_private = kfc; + fc->ops = &ghost_fs_context_ops; + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(&init_user_ns); + fc->global = true; + return 0; +} + +static void ghost_kill_sb(struct super_block *sb) +{ + kernfs_kill_sb(sb); +} + +static struct file_system_type ghost_fs_type = { + .name = "ghost", + .init_fs_context = ghost_init_fs_context, + .kill_sb = ghost_kill_sb, +}; + +static int __init ghostfs_init(void) +{ + int ret = 0; + + ret = ghost_setup_root(); + if (ret) + return ret; + + ret = sysfs_create_mount_point(fs_kobj, "ghost"); + if (ret) + goto cleanup_root; + + ret = register_filesystem(&ghost_fs_type); + if (ret) + goto cleanup_mountpoint; + + return 0; + +cleanup_mountpoint: + sysfs_remove_mount_point(fs_kobj, "ghost"); +cleanup_root: + kernfs_destroy_root(ghost_kfs_root); + ghost_kfs_root = NULL; + + return ret; +} + +static void __exit ghostfs_exit(void) +{ + unregister_filesystem(&ghost_fs_type); + sysfs_remove_mount_point(fs_kobj, "ghost"); + kernfs_destroy_root(ghost_kfs_root); +} + +late_initcall(ghostfs_init); +__exitcall(ghostfs_exit); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 28995a89542c..45c4aa934379 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2214,6 +2214,20 @@ static inline bool skip_fair_idle_balance(struct cfs_rq *cfs_rq, return false; } +typedef const struct ghost_abi *ghost_abi_ptr_t; +struct ghost_abi { + int version; + int (*abi_init)(ghost_abi_ptr_t abi); + struct ghost_enclave * + (*create_enclave)(ghost_abi_ptr_t abi, + struct kernfs_node *dir, ulong id); +}; + +#define DEFINE_GHOST_ABI(name) \ +const static struct ghost_abi __##name##_ghost_abi \ + __aligned(__alignof__(struct ghost_abi)) \ + __used __section(".rodata.ghost_abi") + #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From 014bd0a8b07a5c60c477dc6bdcb0ff6f29b56c6b Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 24 Oct 2021 22:35:19 -0700 Subject: [PATCH 077/165] sched: allow agent to select abi when creating an enclave Prior to this change userspace assumed that kernel only supported a single ABI and therefore it was implicit at enclave creation time. The kernel now advertises all supported ABIs in the "version" file at the top of the ghostfs mount. For e.g. $ cat /sys/fs/ghost/version 55 56 57 The agent is now required to provide the ABI when it creates an enclave. For e.g. $ echo "create 1 57" > /sys/fs/ghost/ctl $ cat /sys/fs/ghost/enclave_1/status version 57 active no nr_tasks 0 Submitted in conjunction with cl/405449791 Tested: all unit tests passing in virtme Effort: sched/ghost/abi Change-Id: I52abb8c5738998e221349d7771b75b0cc37af8f3 --- include/uapi/linux/ghost.h | 2 +- kernel/sched/ghost.c | 2 +- kernel/sched/ghost_core.c | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 7970fd8ab27a..66c36771fa4b 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 56 +#define GHOST_VERSION 57 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8270954925f9..a0df53c1930e 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -7167,7 +7167,7 @@ static void runtime_adjust_dirtabs(void) static int __init abi_init(ghost_abi_ptr_t abi) { if (WARN_ON_ONCE(abi->version != GHOST_VERSION)) - return -1; + return -EINVAL; runtime_adjust_dirtabs(); return 0; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index e933cb3e3eaf..573419346fa9 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1,7 +1,6 @@ #include #include "sched.h" -#include "ghost_uapi.h" static struct kernfs_root *ghost_kfs_root; @@ -89,14 +88,15 @@ static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, struct kernfs_node *ctl = of->kn; struct kernfs_node *top_dir = ctl->parent; unsigned long x; + unsigned int abi_num; int ret; gf_strip_slash_n(buf, len); /* This will ignore any extra digits or characters beyond the %u. */ - ret = sscanf(buf, "create %lu", &x); - if (ret == 1) { - ret = make_enclave(top_dir, x, GHOST_VERSION); + ret = sscanf(buf, "create %lu %u", &x, &abi_num); + if (ret == 2) { + ret = make_enclave(top_dir, x, abi_num); return ret ? ret : len; } @@ -109,10 +109,11 @@ static struct kernfs_ops gf_ops_top_ctl = { static int gf_top_version_show(struct seq_file *sf, void *v) { - /* For now we only advertise a single abi from the kernel */ - WARN_ON_ONCE(first_ghost_abi != last_ghost_abi); + ghost_abi_ptr_t abi; + + for_each_abi(abi) + seq_printf(sf, "%u\n", abi->version); - seq_printf(sf, "%u", first_ghost_abi->version); return 0; } From 3b188a4ca63276a7725747a841b01e529d06105e Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 24 Oct 2021 23:13:38 -0700 Subject: [PATCH 078/165] sched: add "version" file to retrieve abi of an enclave. Intended to be used when an agent is associated with an already existing enclave (to ensure that the ABI of the enclave matches the ABI that the agent is compiled with). Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: I4d78abf2eac6e17e9a65d5208fecd4333baf42dc --- include/uapi/linux/ghost.h | 2 +- kernel/sched/ghost.c | 23 ++++++++++++++++++++++- kernel/sched/ghost_core.c | 2 ++ kernel/sched/sched.h | 4 ++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 66c36771fa4b..e1ab08e7d9d1 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 57 +#define GHOST_VERSION 58 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index a0df53c1930e..a714c199afbe 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -7054,7 +7054,7 @@ static int gf_status_show(struct seq_file *sf, void *v) nr_tasks = e->nr_tasks; spin_unlock_irqrestore(&e->lock, fl); - seq_printf(sf, "version %u\n", GHOST_VERSION); + seq_printf(sf, "version %u\n", e->abi->version); seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); seq_printf(sf, "nr_tasks %lu\n", nr_tasks); return 0; @@ -7066,6 +7066,21 @@ static struct kernfs_ops gf_ops_e_status = { .seq_show = gf_status_show, }; +static int gf_abi_version_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + WARN_ON_ONCE(e->abi->version != GHOST_VERSION); + seq_printf(sf, "%u\n", e->abi->version); + return 0; +} + +static struct kernfs_ops gf_ops_e_abi_version = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_abi_version_show, +}; + static struct gf_dirent enclave_dirtab[] = { { .name = "sw_regions", @@ -7122,6 +7137,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0444, .ops = &gf_ops_e_status, }, + { + .name = "abi_version", + .mode = 0444, + .ops = &gf_ops_e_abi_version, + }, {0}, }; @@ -7190,6 +7210,7 @@ static struct ghost_enclave *create_enclave(ghost_abi_ptr_t abi, if (e == NULL) return ERR_PTR(-ENOMEM); + e->abi = abi; spin_lock_init(&e->lock); kref_init(&e->kref); INIT_LIST_HEAD(&e->sw_region_list); diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 573419346fa9..053b84497875 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -73,6 +73,8 @@ static int make_enclave(struct kernfs_node *parent, unsigned long id, return PTR_ERR(e); } + WARN_ON_ONCE(e->abi != abi); + /* * Once the enclave has been activated, it is available to userspace and * can be used for scheduling. After that, we must destroy it by diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 45c4aa934379..3e6c3532db1f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -145,11 +145,15 @@ struct enclave_work { bool run_task_reaper; }; +struct ghost_abi; + /* * ghost_enclave is a container for the agents, queues and sw_regions * that express the scheduling policy for a set of CPUs. */ struct ghost_enclave { + const struct ghost_abi *abi; + /* * 'lock' serializes mutation of 'sw_region_list' as well as * allocation and freeing of status words within a region. From 7bb9ba9ba2b78ab12ceebe516d6da3090987f63b Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 25 Oct 2021 11:25:19 -0700 Subject: [PATCH 079/165] sched: move enclave cpu management into abi-independent code. CPUs are a shared resource so assigning a cpu to an enclave must happen in abi-independent code. To that end we provide the following new APIs: - ghost_claim_cpus() to assign a set of cpus to an enclave. - ghost_publish_cpu() to advertise that a cpu is participating in ghost scheduling (basically set the per-cpu enclave pointer). - ghost_unpublish_cpu() to retract a cpu from participating in ghost scheduling. - ghost_return_cpu() to make a cpu available to be assigned to another enclave. No functional change. Tested: all unit tests pass in virtme (without any WARNs) Effort: sched/ghost/abi Change-Id: I34811755a7edb668f8907a3a5cb54bb819a26c5f --- kernel/sched/ghost.c | 54 +++++++++++----------------- kernel/sched/ghost_core.c | 75 +++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 15 ++++++++ 3 files changed, 111 insertions(+), 33 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index a714c199afbe..b0231d19e0b3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -35,13 +35,8 @@ /* ghost tid */ typedef int64_t gtid_t; -/* Lock ordering is enclave -> cpu_rsvp */ -static DEFINE_SPINLOCK(cpu_rsvp); -static struct cpumask cpus_in_enclave; /* protected by cpu_rsvp lock */ - /* The ghost_txn pointer equals NULL or &enclave->cpu_data[this_cpu].txn */ static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_txn *, ghost_txn); -static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); static DEFINE_PER_CPU(int64_t, sync_group_cookie); /* @@ -1777,15 +1772,15 @@ static inline void queue_incref(struct ghost_queue *q) kref_get(&q->kref); } -/* Hold e->lock and the cpu_rsvp lock */ +/* + * Caller must hold e->lock and 'cpu' must have already been claimed + * on behalf of the enclave via ghost_claim_cpus(). + */ static void __enclave_add_cpu(struct ghost_enclave *e, int cpu) { struct ghost_txn *txn; VM_BUG_ON(!spin_is_locked(&e->lock)); - VM_BUG_ON(!spin_is_locked(&cpu_rsvp)); - - VM_BUG_ON(cpumask_test_cpu(cpu, &cpus_in_enclave)); VM_BUG_ON(cpumask_test_cpu(cpu, &e->cpus)); txn = &e->cpu_data[cpu]->txn; @@ -1800,10 +1795,14 @@ static void __enclave_add_cpu(struct ghost_enclave *e, int cpu) txn->u.sync_group_owner = -1; txn->state = GHOST_TXN_COMPLETE; - cpumask_set_cpu(cpu, &cpus_in_enclave); cpumask_set_cpu(cpu, &e->cpus); - rcu_assign_pointer(per_cpu(enclave, cpu), e); rcu_assign_pointer(per_cpu(ghost_txn, cpu), txn); + + /* + * We have already claimed 'cpu' for this enclave so now publish + * it to the rest of the kernel via the per_cpu(enclave) pointer. + */ + ghost_publish_cpu(e, cpu); } /* Hold e->lock. Caller must synchronize_rcu(). */ @@ -1811,30 +1810,27 @@ static void __enclave_unpublish_cpu(struct ghost_enclave *e, int cpu) { VM_BUG_ON(!spin_is_locked(&e->lock)); - rcu_assign_pointer(per_cpu(enclave, cpu), NULL); rcu_assign_pointer(per_cpu(ghost_txn, cpu), NULL); + ghost_unpublish_cpu(e, cpu); } -/* Hold e->lock and the cpu_rsvp lock. */ +/* Caller must hold e->lock */ static void __enclave_return_cpu(struct ghost_enclave *e, int cpu) { VM_BUG_ON(!spin_is_locked(&e->lock)); - VM_BUG_ON(!spin_is_locked(&cpu_rsvp)); - - VM_BUG_ON(!cpumask_test_cpu(cpu, &cpus_in_enclave)); VM_BUG_ON(!cpumask_test_cpu(cpu, &e->cpus)); - cpumask_clear_cpu(cpu, &cpus_in_enclave); cpumask_clear_cpu(cpu, &e->cpus); + ghost_return_cpu(e, cpu); } -/* Hold e->lock and the cpu_rsvp lock. Caller must synchronize_rcu(). */ +/* Caller must hold e->lock and synchronize_rcu() on return */ static void __enclave_remove_cpu(struct ghost_enclave *e, int cpu) { __enclave_unpublish_cpu(e, cpu); - /* cpu no longer belongs to any enclave */ + /* cpu is no longer participating in ghost scheduling */ __enclave_return_cpu(e, cpu); - /* cpu can no be assigned to another enclave */ + /* cpu can now be assigned to another enclave */ } /* @@ -2013,9 +2009,7 @@ static void ghost_destroy_enclave(struct ghost_enclave *e) struct task_struct *agent; if (!rq->ghost.agent) { - spin_lock(&cpu_rsvp); __enclave_remove_cpu(e, cpu); - spin_unlock(&cpu_rsvp); continue; } __enclave_unpublish_cpu(e, cpu); @@ -2112,26 +2106,22 @@ static int ghost_enclave_set_cpus(struct ghost_enclave *e, } } - spin_lock(&cpu_rsvp); - - if (cpumask_intersects(add, &cpus_in_enclave)) { - ret = -EBUSY; - goto out_rsvp; - } for_each_cpu(cpu, add) { if (!cpu_online(cpu)) { ret = -ENODEV; - goto out_rsvp; + goto out_e; } } + ret = ghost_claim_cpus(e, add); + if (ret) + goto out_e; + for_each_cpu(cpu, add) __enclave_add_cpu(e, cpu); for_each_cpu(cpu, del) __enclave_remove_cpu(e, cpu); -out_rsvp: - spin_unlock(&cpu_rsvp); out_e: spin_unlock_irqrestore(&e->lock, flags); @@ -4397,9 +4387,7 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) /* See ghost_destroy_enclave() */ if (rq->ghost.agent_remove_enclave_cpu) { rq->ghost.agent_remove_enclave_cpu = false; - spin_lock(&cpu_rsvp); __enclave_return_cpu(e, rq->cpu); - spin_unlock(&cpu_rsvp); } } diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 053b84497875..03486c4799d4 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1,5 +1,6 @@ #include +#define _GHOST_MAYBE_CONST #include "sched.h" static struct kernfs_root *ghost_kfs_root; @@ -292,3 +293,77 @@ static void __exit ghostfs_exit(void) late_initcall(ghostfs_init); __exitcall(ghostfs_exit); + +/* per_cpu(cpu_owner) protected by 'cpu_rsvp' */ +static DEFINE_SPINLOCK(cpu_rsvp); +static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, cpu_owner); + +DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); + +int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *new_cpus) +{ + int cpu; + + /* + * 'e->lock' must be held across ghost_claim_cpus() and the + * corresponding ghost_publish_cpu(). + */ + VM_BUG_ON(!spin_is_locked(&e->lock)); + + spin_lock(&cpu_rsvp); + + for_each_cpu(cpu, new_cpus) { + /* + * Let's make sure that 'cpu' hasn't been already claimed by + * another enclave. + */ + if (per_cpu(cpu_owner, cpu)) { + spin_unlock(&cpu_rsvp); + return -EBUSY; + } + WARN_ON_ONCE(per_cpu(enclave, cpu) != NULL); + } + + for_each_cpu(cpu, new_cpus) + per_cpu(cpu_owner, cpu) = e; + + spin_unlock(&cpu_rsvp); + return 0; +} + +void ghost_publish_cpu(struct ghost_enclave *e, int cpu) +{ + VM_BUG_ON(!spin_is_locked(&e->lock)); + + spin_lock(&cpu_rsvp); + WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); + WARN_ON_ONCE(per_cpu(enclave, cpu) != NULL); + rcu_assign_pointer(per_cpu(enclave, cpu), e); + spin_unlock(&cpu_rsvp); +} + +void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu) +{ + /* + * 'e->lock' must be held across ghost_unpublish_cpu() and the + * corresponding ghost_return_cpu(). + */ + VM_BUG_ON(!spin_is_locked(&e->lock)); + + spin_lock(&cpu_rsvp); + WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); + WARN_ON_ONCE(per_cpu(enclave, cpu) != e); + rcu_assign_pointer(per_cpu(enclave, cpu), NULL); + spin_unlock(&cpu_rsvp); +} + +void ghost_return_cpu(struct ghost_enclave *e, int cpu) +{ + VM_BUG_ON(!spin_is_locked(&e->lock)); + + spin_lock(&cpu_rsvp); + WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); + WARN_ON_ONCE(per_cpu(enclave, cpu) != NULL); + per_cpu(cpu_owner, cpu) = NULL; + spin_unlock(&cpu_rsvp); +} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3e6c3532db1f..dd4fcdbf3a0b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2232,6 +2232,21 @@ const static struct ghost_abi __##name##_ghost_abi \ __aligned(__alignof__(struct ghost_abi)) \ __used __section(".rodata.ghost_abi") +/* + * We want variables like 'per_cpu(enclave, cpu)' to be immutable in an + * ABI implementation file but mutable in the ABI independent code. The + * _GHOST_MAYBE_CONST macro is an easy way to enforce it at compile time. + */ +#ifndef _GHOST_MAYBE_CONST +#define _GHOST_MAYBE_CONST const +#endif + +_GHOST_MAYBE_CONST DECLARE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); + +int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *cpus); +void ghost_publish_cpu(struct ghost_enclave *e, int cpu); +void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu); +void ghost_return_cpu(struct ghost_enclave *e, int cpu); #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From 2d20002d8be631ac509fa30fed3d3a0749547f40 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 25 Oct 2021 11:44:28 -0700 Subject: [PATCH 080/165] sched: sync_group_cookie generation is abi independent. The 'sync_group_cookie' is an implementation detail having to do with rendezvous handling for sync_group commits. The rendezvous check (ghost_need_rendezvous()) happens on every context switch so it is important to do it inline. Since we intend to keep the rendezvous check in abi-independent code it makes sense to also generate the sync_group cookies in abi-independent code. No functional change. Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: I856481c36664a4cea7da407c989c398780ee1731 --- kernel/sched/ghost.c | 39 -------------------------------------- kernel/sched/ghost_core.c | 40 +++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 42 insertions(+), 39 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b0231d19e0b3..7d09714b4bf5 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -38,25 +38,12 @@ typedef int64_t gtid_t; /* The ghost_txn pointer equals NULL or &enclave->cpu_data[this_cpu].txn */ static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_txn *, ghost_txn); -static DEFINE_PER_CPU(int64_t, sync_group_cookie); /* * Use per-cpu memory instead of stack memory to avoid memsetting. We only * send one message at a time per cpu. */ static DEFINE_PER_CPU(struct bpf_ghost_msg, bpf_ghost_msg); -/* - * We do not want to make 'SG_COOKIE_CPU_BITS' larger than necessary so that - * we can maximize the number of times a sequence counter can increment before - * overflowing. - */ -#if (CONFIG_NR_CPUS < 2048) -#define SG_COOKIE_CPU_BITS 11 -#else -#define SG_COOKIE_CPU_BITS 14 -#endif -#define SG_COOKIE_CPU_SHIFT (63 - SG_COOKIE_CPU_BITS) /* MSB=0 */ - struct ghost_sw_region { struct list_head list; /* ghost_enclave glue */ uint32_t alloc_scan_start; /* allocator starts scan here */ @@ -2133,14 +2120,6 @@ static int ghost_enclave_set_cpus(struct ghost_enclave *e, return ret; } -void __init init_sched_ghost_class(void) -{ - int64_t cpu; - - for_each_possible_cpu(cpu) - per_cpu(sync_group_cookie, cpu) = cpu << SG_COOKIE_CPU_SHIFT; -} - /* * Helper to get an fd and translate it to an enclave. Ghostfs will maintain a * kref on the enclave. That kref is increffed when the file was opened, and @@ -5711,24 +5690,6 @@ static void ghost_reached_rendezvous(int cpu, int64_t target) smp_store_release(&rq->ghost.rendezvous, target); } -/* - * A sync-group cookie uniquely identifies a sync-group commit. - * - * It is useful to identify the initiator and participants of the sync-group. - */ -static inline int64_t ghost_sync_group_cookie(void) -{ - int64_t val; - - WARN_ON_ONCE(preemptible()); - BUILD_BUG_ON(NR_CPUS > (1U << SG_COOKIE_CPU_BITS)); - - val = __this_cpu_inc_return(sync_group_cookie); - VM_BUG_ON((val <= 0) || (val & GHOST_POISONED_RENDEZVOUS)); - - return val; -} - static int ghost_sync_group(struct ghost_enclave *e, struct ghost_ioc_commit_txn __user *arg) { diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 03486c4799d4..888bacc23051 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -14,6 +14,20 @@ extern const struct ghost_abi __end_ghost_abi[]; #define for_each_abi(abi) \ for (abi = first_ghost_abi; abi <= last_ghost_abi; abi++) +/* + * We do not want to make 'SG_COOKIE_CPU_BITS' larger than necessary so that + * we can maximize the number of times a sequence counter can increment before + * overflowing. + */ +#if (CONFIG_NR_CPUS < 2048) +#define SG_COOKIE_CPU_BITS 11 +#else +#define SG_COOKIE_CPU_BITS 14 +#endif +#define SG_COOKIE_CPU_SHIFT (63 - SG_COOKIE_CPU_BITS) /* MSB=0 */ + +static DEFINE_PER_CPU(int64_t, sync_group_cookie); + struct gf_dirent { char *name; umode_t mode; @@ -367,3 +381,29 @@ void ghost_return_cpu(struct ghost_enclave *e, int cpu) per_cpu(cpu_owner, cpu) = NULL; spin_unlock(&cpu_rsvp); } + +/* + * A sync-group cookie uniquely identifies a sync-group commit. + * + * It is useful to identify the initiator and participants of the sync-group. + */ +int64_t ghost_sync_group_cookie(void) +{ + int64_t val; + + WARN_ON_ONCE(preemptible()); + BUILD_BUG_ON(NR_CPUS > (1U << SG_COOKIE_CPU_BITS)); + + val = __this_cpu_inc_return(sync_group_cookie); + VM_BUG_ON((val <= 0) || (val & GHOST_POISONED_RENDEZVOUS)); + + return val; +} + +void __init init_sched_ghost_class(void) +{ + int64_t cpu; + + for_each_possible_cpu(cpu) + per_cpu(sync_group_cookie, cpu) = cpu << SG_COOKIE_CPU_SHIFT; +} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index dd4fcdbf3a0b..9563cceab3f8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2247,6 +2247,8 @@ int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *cpus); void ghost_publish_cpu(struct ghost_enclave *e, int cpu); void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu); void ghost_return_cpu(struct ghost_enclave *e, int cpu); + +int64_t ghost_sync_group_cookie(void); #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From e75e1a0c35d9947e012b94c3118830d598b0555e Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 26 Oct 2021 20:39:41 -0700 Subject: [PATCH 081/165] sched: add 'wait_for_rendezvous' handler to ghost_abi. No functional change. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I989d158cc718f7ab8a3aa95a460d51bdfab235ee --- kernel/sched/ghost.c | 3 ++- kernel/sched/ghost_core.c | 16 ++++++++++++++++ kernel/sched/sched.h | 3 ++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 7d09714b4bf5..b31d3f66b833 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4663,7 +4663,7 @@ static void ghost_task_blocked(struct rq *rq, struct task_struct *p) task_deliver_msg_blocked(rq, p); } -void ghost_wait_for_rendezvous(struct rq *rq) +static void wait_for_rendezvous(struct rq *rq) { int64_t target; @@ -7211,5 +7211,6 @@ DEFINE_GHOST_ABI(current_abi) = { .version = GHOST_VERSION, .abi_init = abi_init, .create_enclave = create_enclave, + .wait_for_rendezvous = wait_for_rendezvous, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 888bacc23051..3931ffa79a6b 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -400,6 +400,22 @@ int64_t ghost_sync_group_cookie(void) return val; } +void ghost_wait_for_rendezvous(struct rq *rq) +{ + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + if (WARN_ON_ONCE(!e)) + return; + + e->abi->wait_for_rendezvous(rq); +} + void __init init_sched_ghost_class(void) { int64_t cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9563cceab3f8..c88f80faa4e0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -255,7 +255,6 @@ static inline bool ghost_bpf_msg_send(struct ghost_enclave *e, } #endif -extern void ghost_wait_for_rendezvous(struct rq *rq); extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); extern int64_t ghost_alloc_gtid(struct task_struct *p); @@ -2225,6 +2224,7 @@ struct ghost_abi { struct ghost_enclave * (*create_enclave)(ghost_abi_ptr_t abi, struct kernfs_node *dir, ulong id); + void (*wait_for_rendezvous)(struct rq *rq); }; #define DEFINE_GHOST_ABI(name) \ @@ -2249,6 +2249,7 @@ void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu); void ghost_return_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); +void ghost_wait_for_rendezvous(struct rq *rq); #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From 6265c27cdea27e69a1ea3b267c0b4bd0429a2ffe Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 1 Nov 2021 15:45:44 -0700 Subject: [PATCH 082/165] sched: add 'pnt_prologue' callback to ghost_abi. ghost_abi.pnt_prologue is called at the start of every pick_next_task() called via __schedule(). At the moment we use this to produce a pending msgs for the previous task. Tested: all unit tests pass in virtme on a debug kernel. Effort: sched/ghost/abi Change-Id: I44be9cf6efa402a03ed8c71daa29734d216359f3 --- kernel/sched/core.c | 7 +------ kernel/sched/ghost.c | 16 +++++++++++++++- kernel/sched/ghost_core.c | 14 ++++++++++++++ kernel/sched/sched.h | 3 ++- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f145a76efaaa..078d49961825 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5155,12 +5155,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) struct task_struct *p; #ifdef CONFIG_SCHED_CLASS_GHOST - rq->ghost.check_prev_preemption = ghost_produce_prev_msgs(rq, prev); - - /* a negative 'switchto_count' indicates end of the chain */ - rq->ghost.switchto_count = -rq->ghost.switchto_count; - WARN_ON_ONCE(rq->ghost.switchto_count > 0); - rq->ghost.pnt_bpf_once = true; + ghost_pnt_prologue(rq, prev); #endif /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b31d3f66b833..31d07e0a5561 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1066,7 +1066,7 @@ static inline void ghost_prepare_switch(struct rq *rq, struct task_struct *prev, * msg was already produced for 'prev' (for e.g. agents don't expect to see a * TASK_PREEMPTED immediately after a TASK_BLOCKED). */ -bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev) +static bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev) { if (!task_has_ghost_policy(prev)) return false; @@ -7207,10 +7207,24 @@ static struct ghost_enclave *create_enclave(ghost_abi_ptr_t abi, return e; } +/* + * Called at the start of every pick_next_task() via __schedule(). + */ +static void pnt_prologue(struct rq *rq, struct task_struct *prev) +{ + rq->ghost.check_prev_preemption = ghost_produce_prev_msgs(rq, prev); + + /* a negative 'switchto_count' indicates end of the chain */ + rq->ghost.switchto_count = -rq->ghost.switchto_count; + WARN_ON_ONCE(rq->ghost.switchto_count > 0); + rq->ghost.pnt_bpf_once = true; +} + DEFINE_GHOST_ABI(current_abi) = { .version = GHOST_VERSION, .abi_init = abi_init, .create_enclave = create_enclave, .wait_for_rendezvous = wait_for_rendezvous, + .pnt_prologue = pnt_prologue, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 3931ffa79a6b..bf25d2e3e4a5 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -416,6 +416,20 @@ void ghost_wait_for_rendezvous(struct rq *rq) e->abi->wait_for_rendezvous(rq); } +void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev) +{ + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + if (e != NULL) + e->abi->pnt_prologue(rq, prev); +} + void __init init_sched_ghost_class(void) { int64_t cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c88f80faa4e0..d05151984e3e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -205,7 +205,6 @@ struct ghost_enclave { }; /* In kernel/sched/ghost.c */ -extern bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev); extern void enclave_release(struct kref *k); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); @@ -2225,6 +2224,7 @@ struct ghost_abi { (*create_enclave)(ghost_abi_ptr_t abi, struct kernfs_node *dir, ulong id); void (*wait_for_rendezvous)(struct rq *rq); + void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); }; #define DEFINE_GHOST_ABI(name) \ @@ -2250,6 +2250,7 @@ void ghost_return_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); void ghost_wait_for_rendezvous(struct rq *rq); +void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From e21c2118a7e03499acf079b6655e1ab4ddf375e9 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 1 Nov 2021 19:05:34 -0700 Subject: [PATCH 083/165] sched: implement enclave_abi() properly. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I97cc0ede380c59a3b50b742d238aef13e86b7a4b --- kernel/sched/bpf.c | 5 ----- kernel/sched/ghost.c | 5 ----- kernel/sched/sched.h | 5 +++++ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c index 5d131dc3df9e..d7d47a127bd1 100644 --- a/kernel/sched/bpf.c +++ b/kernel/sched/bpf.c @@ -375,11 +375,6 @@ static const struct bpf_link_ops bpf_ghost_link_ops = { .dealloc = bpf_ghost_link_dealloc, }; -static inline uint enclave_abi(struct ghost_enclave *e) -{ - return GHOST_VERSION; /* for now all enclaves have the same abi */ -} - int ghost_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 31d07e0a5561..1682f1e4679e 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -117,11 +117,6 @@ static bool check_same_enclave(int cpu_x, int cpu_y) return x == y; } -static inline uint enclave_abi(struct ghost_enclave *e) -{ - return GHOST_VERSION; /* for now all enclaves have the same abi */ -} - /* enclave::is_dying */ #define ENCLAVE_IS_DYING (1U << 0) #define ENCLAVE_IS_REAPABLE (1U << 1) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d05151984e3e..4bd2abcdc27e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2251,6 +2251,11 @@ void ghost_return_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); void ghost_wait_for_rendezvous(struct rq *rq); void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); + +static inline int enclave_abi(const struct ghost_enclave *e) +{ + return e->abi->version; +} #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From cd42a16ee2c02945b1b9bf4acdb6cbd2187f41f2 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Tue, 2 Nov 2021 11:49:12 -0700 Subject: [PATCH 084/165] sched: split ghost bpf code along ABI dependent/independent lines. BPF helpers are defined in ghost_core.c and vector via the enclave pointer to the abi-specific implementation. This implies that the BPF helper signature (return type and arguments) are shared across multiple ABIs and thus immutable. If something needs to change then it should be implemented as a _new_ BPF helper. A new BPF helper is only visible to the latest ABI so technically nothing needs to change for older ABIs (either in userspace or in the kernel). Keep in mind that in this case the function pointer in 'ghost_abi' pointing to the helper implementation will be NULL so the helper must be coded defensively (alternately we can patch the older ABI with a valid function pointer but it means that the ABI can no longer be treated as immutable). The 'is_valid_access' callback in the ghost_msg_verifier_ops uses the version embedded in 'prog->expected_attach_type' to look up the 'ghost_abi' and uses that to vector to the abi-specific handler. We short circuit this for 'ghost_sched_verifier_ops' since the context is empty and thus any access is invalid (this could change in the future in which case we'll vector to abi-specific handlers). Finally we add abi-specific handlers to attach/detach BPF programs at the specified attach point. The handlers don't need to bother with adding/dropping references on the enclave since the calling function handles that. bpf.c is no more - RIP. No functional change. Tested: - Embedding an unknown ABI version in 'expected_attach_type' causes BPF verifier to fail. - Verified that trying to attach a BPF program to an enclave with a different ABI version fails with EBADF. - All unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: Ia1d5dea4bf71912ea2c31e64c8c355c6fb93902f --- kernel/sched/Makefile | 1 - kernel/sched/bpf.c | 508 -------------------------------------- kernel/sched/ghost.c | 319 +++++++++++++++++++++++- kernel/sched/ghost_core.c | 272 ++++++++++++++++++++ kernel/sched/sched.h | 37 +-- 5 files changed, 588 insertions(+), 549 deletions(-) delete mode 100644 kernel/sched/bpf.c diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index d20bf0aa6786..6ba3d3662224 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -38,4 +38,3 @@ obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost.o obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost_core.o -obj-$(CONFIG_BPF) += bpf.o diff --git a/kernel/sched/bpf.c b/kernel/sched/bpf.c deleted file mode 100644 index d7d47a127bd1..000000000000 --- a/kernel/sched/bpf.c +++ /dev/null @@ -1,508 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#include - -#include "sched.h" -#include "ghost_uapi.h" - -#ifdef CONFIG_SCHED_CLASS_GHOST - -BPF_CALL_1(bpf_ghost_wake_agent, u32, cpu) -{ - /* - * Each BPF helper has a corresponding integer in uapi/linux/bpf.h, - * similar to syscall numbers. Catch any ABI inconsistencies between - * prodkernel and open source. - */ - BUILD_BUG_ON(BPF_FUNC_ghost_wake_agent != 204); - return ghost_wake_agent_on_check(cpu); -} - -static const struct bpf_func_proto bpf_ghost_wake_agent_proto = { - .func = bpf_ghost_wake_agent, - .gpl_only = true, - .ret_type = RET_INTEGER, - .arg1_type = ARG_ANYTHING, -}; - -BPF_CALL_3(bpf_ghost_run_gtid, s64, gtid, u32, task_barrier, int, run_flags) -{ - return ghost_run_gtid_on(gtid, task_barrier, run_flags, - smp_processor_id()); -} - -static const struct bpf_func_proto bpf_ghost_run_gtid_proto = { - .func = bpf_ghost_run_gtid, - .gpl_only = true, - .ret_type = RET_INTEGER, - .arg1_type = ARG_ANYTHING, - .arg2_type = ARG_ANYTHING, - .arg3_type = ARG_ANYTHING, -}; - -bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) -{ - struct bpf_ghost_sched_kern ctx = {}; - struct bpf_prog *prog; - - lockdep_assert_held(&rq->lock); - - prog = rcu_dereference(e->bpf_tick); - if (!prog) - return false; - - /* prog returns 1 if we want a tick on this cpu. */ - return BPF_PROG_RUN(prog, &ctx) != 1; -} - -#define BPF_GHOST_PNT_DONT_IDLE 1 - -void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) -{ - struct bpf_ghost_sched_kern ctx = {}; - struct bpf_prog *prog; - int ret; - - lockdep_assert_held(&rq->lock); - - rcu_read_lock(); - prog = rcu_dereference(e->bpf_pnt); - if (!prog) { - rcu_read_unlock(); - return; - } - - /* - * BPF programs attached here may call ghost_run_gtid(), which requires - * that we not hold any RQ locks. We are called from - * pick_next_task_ghost where it is safe to unlock the RQ. - */ - rq_unpin_lock(rq, rf); - raw_spin_unlock(&rq->lock); - - ret = BPF_PROG_RUN(prog, &ctx); - - raw_spin_lock(&rq->lock); - rq_repin_lock(rq, rf); - - rcu_read_unlock(); - - if (ret == BPF_GHOST_PNT_DONT_IDLE) { - /* - * The next time this rq selects the idle task, it will bail out - * of do_idle() quickly. Since we unlocked the RQ lock, it's - * possible that this rq will pick something other than the idle - * task, which is fine. - */ - rq->ghost.dont_idle_once = true; - } -} - -/* - * Returns true (default) if the user wants us to send the message. - * Note that our caller holds an RQ lock, and we can't safely unlock it, so - * programs at the attach point can't call ghost_run_gtid(). - */ -bool ghost_bpf_msg_send(struct ghost_enclave *e, struct bpf_ghost_msg *msg) -{ - struct bpf_prog *prog; - bool send; - - rcu_read_lock(); - prog = rcu_dereference(e->bpf_msg_send); - if (!prog) { - rcu_read_unlock(); - return true; - } - /* Program returns 0 if they want us to send the message. */ - send = BPF_PROG_RUN(prog, msg) == 0; - rcu_read_unlock(); - return send; -} - -static int ghost_sched_tick_attach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - if (e->bpf_tick) { - spin_unlock_irqrestore(&e->lock, irq_fl); - return -EBUSY; - } - rcu_assign_pointer(e->bpf_tick, prog); - spin_unlock_irqrestore(&e->lock, irq_fl); - return 0; -} - -static void ghost_sched_tick_detach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - rcu_replace_pointer(e->bpf_tick, NULL, lockdep_is_held(&e->lock)); - spin_unlock_irqrestore(&e->lock, irq_fl); -} - -static int ghost_sched_pnt_attach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - if (e->bpf_pnt) { - spin_unlock_irqrestore(&e->lock, irq_fl); - return -EBUSY; - } - rcu_assign_pointer(e->bpf_pnt, prog); - spin_unlock_irqrestore(&e->lock, irq_fl); - return 0; -} - -static void ghost_sched_pnt_detach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - rcu_replace_pointer(e->bpf_pnt, NULL, lockdep_is_held(&e->lock)); - spin_unlock_irqrestore(&e->lock, irq_fl); -} - -static const struct bpf_func_proto * -ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -{ - int eat = prog->expected_attach_type & 0xFFFF; - - switch (func_id) { - case BPF_FUNC_ghost_wake_agent: - return &bpf_ghost_wake_agent_proto; - case BPF_FUNC_ghost_run_gtid: - switch (eat) { - case BPF_GHOST_SCHED_PNT: - return &bpf_ghost_run_gtid_proto; - default: - return NULL; - } - default: - return bpf_base_func_proto(func_id); - } -} - -static bool ghost_sched_is_valid_access(int off, int size, - enum bpf_access_type type, - const struct bpf_prog *prog, - struct bpf_insn_access_aux *info) -{ - /* The verifier guarantees that size > 0. */ - if (off < 0 || off + size > sizeof(struct bpf_ghost_sched) - || off % size) - return false; - - switch (off) { - default: - return false; - } -} - -#define SCHEDULER_ACCESS_FIELD(T, F) \ - T(BPF_FIELD_SIZEOF(struct bpf_ghost_sched_kern, F), \ - si->dst_reg, si->src_reg, \ - offsetof(struct bpf_ghost_sched_kern, F)) - -static u32 ghost_sched_convert_ctx_access(enum bpf_access_type type, - const struct bpf_insn *si, - struct bpf_insn *insn_buf, - struct bpf_prog *prog, - u32 *target_size) -{ - struct bpf_insn *insn = insn_buf; - - switch (si->off) { - default: - *target_size = 0; - break; - } - - *target_size = sizeof(u32); - - return insn - insn_buf; -} - -const struct bpf_verifier_ops ghost_sched_verifier_ops = { - .get_func_proto = ghost_sched_func_proto, - .is_valid_access = ghost_sched_is_valid_access, - .convert_ctx_access = ghost_sched_convert_ctx_access, -}; - -const struct bpf_prog_ops ghost_sched_prog_ops = { -}; - - -static int ghost_msg_send_attach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - if (e->bpf_msg_send) { - spin_unlock_irqrestore(&e->lock, irq_fl); - return -EBUSY; - } - rcu_assign_pointer(e->bpf_msg_send, prog); - spin_unlock_irqrestore(&e->lock, irq_fl); - return 0; -} - -static void ghost_msg_send_detach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - rcu_replace_pointer(e->bpf_msg_send, NULL, lockdep_is_held(&e->lock)); - spin_unlock_irqrestore(&e->lock, irq_fl); -} - -static const struct bpf_func_proto * -ghost_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -{ - switch (func_id) { - case BPF_FUNC_ghost_wake_agent: - return &bpf_ghost_wake_agent_proto; - default: - return bpf_base_func_proto(func_id); - } -} - -#include -static bool ghost_msg_is_valid_access(int off, int size, - enum bpf_access_type type, - const struct bpf_prog *prog, - struct bpf_insn_access_aux *info) -{ - /* The verifier guarantees that size > 0. */ - if (off < 0 || off + size > sizeof(struct bpf_ghost_msg) - || off % size) - return false; - - return true; -} - -const struct bpf_verifier_ops ghost_msg_verifier_ops = { - .get_func_proto = ghost_msg_func_proto, - .is_valid_access = ghost_msg_is_valid_access, -}; - -const struct bpf_prog_ops ghost_msg_prog_ops = { -}; - - -struct bpf_ghost_link { - struct bpf_link link; - struct ghost_enclave *e; - enum bpf_prog_type prog_type; - enum bpf_attach_type ea_type; -}; - -static void bpf_ghost_link_dealloc(struct bpf_link *link) -{ - struct bpf_ghost_link *sc_link = - container_of(link, struct bpf_ghost_link, link); - - kfree(sc_link); -} - -static void bpf_ghost_link_release(struct bpf_link *link) -{ - struct bpf_ghost_link *sc_link = - container_of(link, struct bpf_ghost_link, link); - struct ghost_enclave *e = sc_link->e; - - if (WARN_ONCE(!e, "Missing enclave for bpf link ea_type %d!", - sc_link->ea_type)) - return; - - switch (sc_link->prog_type) { - case BPF_PROG_TYPE_GHOST_SCHED: - switch (sc_link->ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - ghost_sched_tick_detach(e, link->prog); - break; - case BPF_GHOST_SCHED_PNT: - ghost_sched_pnt_detach(e, link->prog); - break; - default: - WARN_ONCE(1, "Unexpected release for ea_type %d", - sc_link->ea_type); - break; - } - break; - case BPF_PROG_TYPE_GHOST_MSG: - switch (sc_link->ea_type) { - case BPF_GHOST_MSG_SEND: - ghost_msg_send_detach(e, link->prog); - break; - default: - WARN_ONCE(1, "Unexpected release for ea_type %d", - sc_link->ea_type); - break; - } - break; - default: - WARN_ONCE(1, "Unexpected release for prog_type %d", - sc_link->prog_type); - break; - } - kref_put(&e->kref, enclave_release); - sc_link->e = NULL; -} - -static const struct bpf_link_ops bpf_ghost_link_ops = { - .release = bpf_ghost_link_release, - .dealloc = bpf_ghost_link_dealloc, -}; - -int ghost_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog) -{ - struct bpf_link_primer link_primer; - struct bpf_ghost_link *sc_link; - enum bpf_attach_type ea_type; - struct ghost_enclave *e; - struct fd f_enc = {0}; - int err, ea_abi; - - BUILD_BUG_ON(GHOST_VERSION > 0xFFFF); - - if (attr->link_create.flags) - return -EINVAL; - if (prog->expected_attach_type != attr->link_create.attach_type) - return -EINVAL; - - // Mask the ABI value encoded in the upper 16 bits. - ea_type = prog->expected_attach_type & 0xFFFF; - ea_abi = (u32)prog->expected_attach_type >> 16; - - switch (prog->type) { - case BPF_PROG_TYPE_GHOST_SCHED: - switch (ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - case BPF_GHOST_SCHED_PNT: - break; - default: - return -EINVAL; - } - break; - case BPF_PROG_TYPE_GHOST_MSG: - switch (ea_type) { - case BPF_GHOST_MSG_SEND: - break; - default: - return -EINVAL; - } - break; - default: - return -EINVAL; - } - - sc_link = kzalloc(sizeof(*sc_link), GFP_USER); - if (!sc_link) - return -ENOMEM; - bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, - &bpf_ghost_link_ops, prog); - sc_link->prog_type = prog->type; - sc_link->ea_type = ea_type; - - err = bpf_link_prime(&sc_link->link, &link_primer); - if (err) { - kfree(sc_link); - return -EINVAL; - } - - e = ghost_fdget_enclave(attr->link_create.target_fd, &f_enc); - if (!e || ea_abi != enclave_abi(e)) { - ghost_fdput_enclave(e, &f_enc); - /* bpf_link_cleanup() triggers .dealloc, but not .release. */ - bpf_link_cleanup(&link_primer); - return -EBADF; - } - - /* - * On success, sc_link will hold a kref on the enclave, which will get - * put when the link's FD is closed (and thus bpf_link_put -> - * bpf_link_free -> our release). This is similar to how ghostfs files - * hold a kref on the enclave. Release is not called on failure. - */ - kref_get(&e->kref); - sc_link->e = e; - ghost_fdput_enclave(e, &f_enc); - - switch (prog->type) { - case BPF_PROG_TYPE_GHOST_SCHED: - switch (ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - err = ghost_sched_tick_attach(e, prog); - break; - case BPF_GHOST_SCHED_PNT: - err = ghost_sched_pnt_attach(e, prog); - break; - default: - pr_warn("bad sched bpf ea_type %d, should be unreachable", - ea_type); - err = -EINVAL; - break; - } - break; - case BPF_PROG_TYPE_GHOST_MSG: - switch (ea_type) { - case BPF_GHOST_MSG_SEND: - err = ghost_msg_send_attach(e, prog); - break; - default: - pr_warn("bad msg bpf ea_type %d, should be unreachable", - ea_type); - err = -EINVAL; - break; - } - break; - default: - return -EINVAL; - } - - if (err) { - /* bpf_link_cleanup() triggers .dealloc, but not .release. */ - bpf_link_cleanup(&link_primer); - kref_put(&e->kref, enclave_release); - return err; - } - - return bpf_link_settle(&link_primer); -} - -#else /* !CONFIG_SCHED_CLASS_GHOST */ - -const struct bpf_verifier_ops ghost_sched_verifier_ops = {}; -const struct bpf_prog_ops ghost_sched_prog_ops = {}; - -const struct bpf_verifier_ops ghost_msg_verifier_ops = {}; -const struct bpf_prog_ops ghost_msg_prog_ops = {}; - -int ghost_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) -{ - return -EINVAL; -} - -#endif /* !CONFIG_SCHED_CLASS_GHOST */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 1682f1e4679e..243fa2c08ce3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -139,6 +139,109 @@ static inline bool enclave_is_reapable(struct ghost_enclave *e) return false; } +#ifdef CONFIG_BPF +#include + +static bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) +{ + struct bpf_ghost_sched_kern ctx = {}; + struct bpf_prog *prog; + + lockdep_assert_held(&rq->lock); + + prog = rcu_dereference(e->bpf_tick); + if (!prog) + return false; + + /* prog returns 1 if we want a tick on this cpu. */ + return BPF_PROG_RUN(prog, &ctx) != 1; +} + +#define BPF_GHOST_PNT_DONT_IDLE 1 + +static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, + struct rq_flags *rf) +{ + struct bpf_ghost_sched_kern ctx = {}; + struct bpf_prog *prog; + int ret; + + lockdep_assert_held(&rq->lock); + + rcu_read_lock(); + prog = rcu_dereference(e->bpf_pnt); + if (!prog) { + rcu_read_unlock(); + return; + } + + /* + * BPF programs attached here may call ghost_run_gtid(), which requires + * that we not hold any RQ locks. We are called from + * pick_next_task_ghost where it is safe to unlock the RQ. + */ + rq_unpin_lock(rq, rf); + raw_spin_unlock(&rq->lock); + + ret = BPF_PROG_RUN(prog, &ctx); + + raw_spin_lock(&rq->lock); + rq_repin_lock(rq, rf); + + rcu_read_unlock(); + + if (ret == BPF_GHOST_PNT_DONT_IDLE) { + /* + * The next time this rq selects the idle task, it will bail out + * of do_idle() quickly. Since we unlocked the RQ lock, it's + * possible that this rq will pick something other than the idle + * task, which is fine. + */ + rq->ghost.dont_idle_once = true; + } +} + +/* + * Returns true (default) if the user wants us to send the message. + * Note that our caller holds an RQ lock, and we can't safely unlock it, so + * programs at the attach point can't call ghost_run_gtid(). + */ +static bool ghost_bpf_msg_send(struct ghost_enclave *e, + struct bpf_ghost_msg *msg) +{ + struct bpf_prog *prog; + bool send; + + rcu_read_lock(); + prog = rcu_dereference(e->bpf_msg_send); + if (!prog) { + rcu_read_unlock(); + return true; + } + /* Program returns 0 if they want us to send the message. */ + send = BPF_PROG_RUN(prog, msg) == 0; + rcu_read_unlock(); + return send; +} +#else +static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) +{ + return false; +} + +static inline void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, + struct rq_flags *rf) +{ + return; +} + +static inline bool ghost_bpf_msg_send(struct ghost_enclave *e, + struct bpf_ghost_msg *msg) +{ + return true; +} +#endif /* CONFIG_BPF */ + /* * There are certain things we can't do in the scheduler while holding rq locks * (or enclave locks, which are ordered after the rq lock), such as @@ -3827,6 +3930,12 @@ static inline int __produce_for_task(struct task_struct *p, WARN(1, "unknown bpg_ghost_msg type %d!\n", msg->type); return -EINVAL; }; + /* + * XXX for a task that enters ghost into enclave_X but on a cpu + * that belongs to enclave_Y then we'll call the 'bpf_msg_send' + * program attached to enclave_X but any BPF helpers used by the + * program will callback into enclave_Y. + */ if (!ghost_bpf_msg_send(p->ghost.enclave, msg)) return -ENOMSG; return _produce(p->ghost.dst_q, barrier, msg->type, @@ -4539,7 +4648,7 @@ static void ghost_wake_agent_on(int cpu) put_cpu(); } -int ghost_wake_agent_on_check(int cpu) +static int ghost_wake_agent_on_check(int cpu) { int this_cpu = get_cpu(); int ret; @@ -5010,18 +5119,6 @@ static int __ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, return 0; } -/* - * Attempts to run gtid on cpu. Returns 0 or -error. - * - * Called from BPF helpers. The programs that can call those helpers are - * explicitly allowed in ghost_sched_func_proto. - */ -int ghost_run_gtid_on(gtid_t gtid, u32 task_barrier, int run_flags, int cpu) -{ - return __ghost_run_gtid_on(gtid, task_barrier, run_flags, cpu, - /*check_caller_enclave=*/ false); -} - static inline bool _ghost_txn_ready(int cpu, int *commit_flags) { struct ghost_txn *txn; @@ -7130,6 +7227,16 @@ static void runtime_adjust_dirtabs(void) static int __init abi_init(ghost_abi_ptr_t abi) { + /* + * ghost bpf programs encode the ABI they were compiled against + * in the upper 16 bits of 'prog->expected_attach_type' which + * only works as long as GHOST_VERSION can fit within 16 bits. + * + * Ideally this would be in ghost_bpf_link_attach() but we cannot + * include the uapi header in ghost_core.c so we check it here. + */ + BUILD_BUG_ON(GHOST_VERSION > 0xFFFF); + if (WARN_ON_ONCE(abi->version != GHOST_VERSION)) return -EINVAL; @@ -7215,11 +7322,197 @@ static void pnt_prologue(struct rq *rq, struct task_struct *prev) rq->ghost.pnt_bpf_once = true; } +static int bpf_wake_agent(int cpu) +{ + return ghost_wake_agent_on_check(cpu); +} + +/* + * Attempts to run gtid on cpu. Returns 0 or -error. + * + * Called from BPF helpers. The programs that can call those helpers are + * explicitly allowed in ghost_sched_func_proto. + */ +static int bpf_run_gtid(s64 gtid, u32 task_barrier, int run_flags, int cpu) +{ + bool check_caller_enclave = (cpu != smp_processor_id()); + + return __ghost_run_gtid_on(gtid, task_barrier, run_flags, cpu, + check_caller_enclave); +} + +static bool ghost_msg_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + /* The verifier guarantees that size > 0. */ + if (off < 0 || off + size > sizeof(struct bpf_ghost_msg) || off % size) + return false; + + return true; +} + +static int ghost_sched_tick_attach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->bpf_tick) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + rcu_assign_pointer(e->bpf_tick, prog); + spin_unlock_irqrestore(&e->lock, irq_fl); + return 0; +} + +static void ghost_sched_tick_detach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + rcu_replace_pointer(e->bpf_tick, NULL, lockdep_is_held(&e->lock)); + spin_unlock_irqrestore(&e->lock, irq_fl); +} + +static int ghost_sched_pnt_attach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->bpf_pnt) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + rcu_assign_pointer(e->bpf_pnt, prog); + spin_unlock_irqrestore(&e->lock, irq_fl); + return 0; +} + +static void ghost_sched_pnt_detach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + rcu_replace_pointer(e->bpf_pnt, NULL, lockdep_is_held(&e->lock)); + spin_unlock_irqrestore(&e->lock, irq_fl); +} + +static int ghost_msg_send_attach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + if (e->bpf_msg_send) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -EBUSY; + } + rcu_assign_pointer(e->bpf_msg_send, prog); + spin_unlock_irqrestore(&e->lock, irq_fl); + return 0; +} + +static void ghost_msg_send_detach(struct ghost_enclave *e, + struct bpf_prog *prog) +{ + unsigned long irq_fl; + + spin_lock_irqsave(&e->lock, irq_fl); + rcu_replace_pointer(e->bpf_msg_send, NULL, lockdep_is_held(&e->lock)); + spin_unlock_irqrestore(&e->lock, irq_fl); +} + +static int bpf_link_attach(struct ghost_enclave *e, struct bpf_prog *prog, + int prog_type, int attach_type) +{ + int err; + + switch (prog_type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (attach_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + err = ghost_sched_tick_attach(e, prog); + break; + case BPF_GHOST_SCHED_PNT: + err = ghost_sched_pnt_attach(e, prog); + break; + default: + pr_warn("bad sched bpf attach_type %d", attach_type); + err = -EINVAL; + break; + } + break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (attach_type) { + case BPF_GHOST_MSG_SEND: + err = ghost_msg_send_attach(e, prog); + break; + default: + pr_warn("bad msg bpf attach_type %d", attach_type); + err = -EINVAL; + break; + } + break; + default: + pr_warn("bad bpf prog_type %d", prog_type); + err = -EINVAL; + } + + return err; +} + +static void bpf_link_detach(struct ghost_enclave *e, struct bpf_prog *prog, + int prog_type, int attach_type) +{ + switch (prog_type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (attach_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + ghost_sched_tick_detach(e, prog); + break; + case BPF_GHOST_SCHED_PNT: + ghost_sched_pnt_detach(e, prog); + break; + default: + WARN_ONCE(1, "enclave %lu: unexpected bpf prog %d/%d", + e->id, prog_type, attach_type); + break; + } + break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (attach_type) { + case BPF_GHOST_MSG_SEND: + ghost_msg_send_detach(e, prog); + break; + default: + WARN_ONCE(1, "enclave %lu: unexpected bpf prog %d/%d", + e->id, prog_type, attach_type); + break; + } + break; + default: + WARN_ONCE(1, "enclave %lu: unexpected bpf prog %d/%d", + e->id, prog_type, attach_type); + break; + } +} + DEFINE_GHOST_ABI(current_abi) = { .version = GHOST_VERSION, .abi_init = abi_init, .create_enclave = create_enclave, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, + .bpf_wake_agent = bpf_wake_agent, + .bpf_run_gtid = bpf_run_gtid, + .ghost_msg_is_valid_access = ghost_msg_is_valid_access, + .bpf_link_attach = bpf_link_attach, + .bpf_link_detach = bpf_link_detach, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index bf25d2e3e4a5..94d8ea27d7f7 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -437,3 +437,275 @@ void __init init_sched_ghost_class(void) for_each_possible_cpu(cpu) per_cpu(sync_group_cookie, cpu) = cpu << SG_COOKIE_CPU_SHIFT; } + +#ifdef CONFIG_BPF +#include + +/* + * ghost bpf programs encode the ABI they were compiled against in the + * upper 16 bits of 'prog->expected_attach_type'. + */ +static inline int bpf_prog_eat_abi(const struct bpf_prog *prog) +{ + return (u32)prog->expected_attach_type >> 16; +} + +static inline int bpf_prog_eat_type(const struct bpf_prog *prog) +{ + return prog->expected_attach_type & 0xFFFF; +} + +BPF_CALL_1(bpf_ghost_wake_agent, u32, cpu) +{ + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + + /* + * Each BPF helper has a corresponding integer in uapi/linux/bpf.h, + * similar to syscall numbers. Catch any ABI inconsistencies between + * prodkernel and open source. + */ + BUILD_BUG_ON(BPF_FUNC_ghost_wake_agent != 204); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + e = rcu_dereference_sched(per_cpu(enclave, smp_processor_id())); + + /* Paranoia: this is not expected */ + if (WARN_ON_ONCE(!e)) + return -ENODEV; + + return e->abi->bpf_wake_agent(cpu); +} + +static const struct bpf_func_proto bpf_ghost_wake_agent_proto = { + .func = bpf_ghost_wake_agent, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, +}; + +BPF_CALL_3(bpf_ghost_run_gtid, s64, gtid, u32, task_barrier, int, run_flags) +{ + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + + /* + * Each BPF helper has a corresponding integer in uapi/linux/bpf.h, + * similar to syscall numbers. Catch any ABI inconsistencies between + * prodkernel and open source. + */ + BUILD_BUG_ON(BPF_FUNC_ghost_run_gtid != 205); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + e = rcu_dereference_sched(per_cpu(enclave, smp_processor_id())); + + /* Paranoia: this is not expected */ + if (WARN_ON_ONCE(!e)) + return -ENODEV; + + return e->abi->bpf_run_gtid(gtid, task_barrier, run_flags, + smp_processor_id()); +} + +static const struct bpf_func_proto bpf_ghost_run_gtid_proto = { + .func = bpf_ghost_run_gtid, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, +}; + +static const struct bpf_func_proto * +ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + int eat = bpf_prog_eat_type(prog); + + switch (func_id) { + case BPF_FUNC_ghost_wake_agent: + return &bpf_ghost_wake_agent_proto; + case BPF_FUNC_ghost_run_gtid: + switch (eat) { + case BPF_GHOST_SCHED_PNT: + return &bpf_ghost_run_gtid_proto; + default: + return NULL; + } + default: + return bpf_base_func_proto(func_id); + } +} + +static bool ghost_sched_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + /* struct bpf_ghost_sched is empty so all accesses are invalid. */ + return false; +} + + +const struct bpf_verifier_ops ghost_sched_verifier_ops = { + .get_func_proto = ghost_sched_func_proto, + .is_valid_access = ghost_sched_is_valid_access, +}; + +const struct bpf_prog_ops ghost_sched_prog_ops = {}; + +static const struct bpf_func_proto * +ghost_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_ghost_wake_agent: + return &bpf_ghost_wake_agent_proto; + default: + return bpf_base_func_proto(func_id); + } +} + +static bool ghost_msg_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + int version = bpf_prog_eat_abi(prog); + ghost_abi_ptr_t abi = ghost_abi_lookup(version); + + if (WARN_ON_ONCE(!abi)) + return false; + + return abi->ghost_msg_is_valid_access(off, size, type, prog, info); +} + +const struct bpf_verifier_ops ghost_msg_verifier_ops = { + .get_func_proto = ghost_msg_func_proto, + .is_valid_access = ghost_msg_is_valid_access, +}; + +const struct bpf_prog_ops ghost_msg_prog_ops = {}; + +struct bpf_ghost_link { + struct bpf_link link; + struct ghost_enclave *e; + enum bpf_prog_type prog_type; + enum bpf_attach_type ea_type; +}; + +static void bpf_ghost_link_dealloc(struct bpf_link *link) +{ + struct bpf_ghost_link *sc_link = + container_of(link, struct bpf_ghost_link, link); + + kfree(sc_link); +} + +static void bpf_ghost_link_release(struct bpf_link *link) +{ + struct bpf_ghost_link *sc_link = + container_of(link, struct bpf_ghost_link, link); + struct ghost_enclave *e = sc_link->e; + + if (WARN_ONCE(!e, "Missing enclave for bpf link ea_type %d!", + sc_link->ea_type)) + return; + + e->abi->bpf_link_detach(e, link->prog, + sc_link->prog_type, sc_link->ea_type); + + kref_put(&e->kref, enclave_release); + sc_link->e = NULL; +} + +static const struct bpf_link_ops bpf_ghost_link_ops = { + .release = bpf_ghost_link_release, + .dealloc = bpf_ghost_link_dealloc, +}; + +int ghost_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog) +{ + struct bpf_link_primer link_primer; + struct bpf_ghost_link *sc_link; + enum bpf_attach_type ea_type; + struct ghost_enclave *e; + struct fd f_enc = {0}; + int err, ea_abi; + + if (attr->link_create.flags) + return -EINVAL; + if (prog->expected_attach_type != attr->link_create.attach_type) + return -EINVAL; + + /* Mask the ABI value encoded in the upper 16 bits. */ + ea_type = bpf_prog_eat_type(prog); + ea_abi = bpf_prog_eat_abi(prog); + + switch (prog->type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (ea_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + case BPF_GHOST_SCHED_PNT: + break; + default: + return -EINVAL; + } + break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (ea_type) { + case BPF_GHOST_MSG_SEND: + break; + default: + return -EINVAL; + } + break; + default: + return -EINVAL; + } + + sc_link = kzalloc(sizeof(*sc_link), GFP_USER); + if (!sc_link) + return -ENOMEM; + bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, + &bpf_ghost_link_ops, prog); + sc_link->prog_type = prog->type; + sc_link->ea_type = ea_type; + + err = bpf_link_prime(&sc_link->link, &link_primer); + if (err) { + kfree(sc_link); + return -EINVAL; + } + + e = ghost_fdget_enclave(attr->link_create.target_fd, &f_enc); + if (!e || ea_abi != enclave_abi(e)) { + ghost_fdput_enclave(e, &f_enc); + /* bpf_link_cleanup() triggers .dealloc, but not .release. */ + bpf_link_cleanup(&link_primer); + return -EBADF; + } + + /* + * On success, sc_link will hold a kref on the enclave, which will get + * put when the link's FD is closed (and thus bpf_link_put -> + * bpf_link_free -> our release). This is similar to how ghostfs files + * hold a kref on the enclave. Release is not called on failure. + */ + kref_get(&e->kref); + sc_link->e = e; + ghost_fdput_enclave(e, &f_enc); + + err = e->abi->bpf_link_attach(e, prog, prog->type, ea_type); + + if (err) { + /* bpf_link_cleanup() triggers .dealloc, but not .release. */ + bpf_link_cleanup(&link_primer); + kref_put(&e->kref, enclave_release); + return err; + } + + return bpf_link_settle(&link_primer); +} +#endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4bd2abcdc27e..5832b25ffacd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -223,37 +223,10 @@ extern void ghost_latched_task_preempted(struct rq *rq); extern void ghost_task_preempted(struct rq *rq, struct task_struct *prev); extern void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p); extern unsigned long ghost_cfs_added_load(struct rq *rq); -extern int ghost_wake_agent_on_check(int cpu); extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); -extern int ghost_run_gtid_on(s64 gtid, u32 task_barrier, int run_flags, - int cpu); extern void ghost_cpu_idle(void); -struct rq_flags; -struct bpf_ghost_msg; -#ifdef CONFIG_BPF -extern bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq); -extern void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, - struct rq_flags *rf); -extern bool ghost_bpf_msg_send(struct ghost_enclave *e, - struct bpf_ghost_msg *msg); -#else -static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) -{ - return false; -} -static inline void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, - struct rq_flags *rf) -{ -} -static inline bool ghost_bpf_msg_send(struct ghost_enclave *e, - struct bpf_ghost_msg *msg) -{ - return true; -} -#endif - extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); extern int64_t ghost_alloc_gtid(struct task_struct *p); @@ -2225,6 +2198,16 @@ struct ghost_abi { struct kernfs_node *dir, ulong id); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); + int (*bpf_wake_agent)(int cpu); + int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); + bool (*ghost_msg_is_valid_access)(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info); + int (*bpf_link_attach)(struct ghost_enclave *e, struct bpf_prog *prog, + int prog_type, int attach_type); + void (*bpf_link_detach)(struct ghost_enclave *e, struct bpf_prog *prog, + int prog_type, int attach_type); }; #define DEFINE_GHOST_ABI(name) \ From 42181e41debf83de4635fae20282d23cf65b8dbd Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 3 Nov 2021 09:53:37 -0700 Subject: [PATCH 085/165] sched: add 'enclave_release' handler to ghost_abi Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I18f9985511d54c8ac5134be05d10aedc684f02ea --- kernel/sched/ghost.c | 4 +++- kernel/sched/ghost_core.c | 4 ++-- kernel/sched/sched.h | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 243fa2c08ce3..de546b45a584 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -96,6 +96,7 @@ static const struct file_operations queue_fops; static struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); static void ghostfs_put_enclave_ctl(struct file *f); static void ghost_destroy_enclave(struct ghost_enclave *e); +static void enclave_release(struct kref *k); /* True if X and Y have the same enclave, including having no enclave. */ static bool check_same_enclave(int cpu_x, int cpu_y) @@ -1946,7 +1947,7 @@ static void enclave_actual_release(struct work_struct *w) kfree(e); } -void enclave_release(struct kref *k) +static void enclave_release(struct kref *k) { struct ghost_enclave *e = container_of(k, struct ghost_enclave, kref); @@ -7507,6 +7508,7 @@ DEFINE_GHOST_ABI(current_abi) = { .version = GHOST_VERSION, .abi_init = abi_init, .create_enclave = create_enclave, + .enclave_release = enclave_release, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .bpf_wake_agent = bpf_wake_agent, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 94d8ea27d7f7..3133bcb04c09 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -615,7 +615,7 @@ static void bpf_ghost_link_release(struct bpf_link *link) e->abi->bpf_link_detach(e, link->prog, sc_link->prog_type, sc_link->ea_type); - kref_put(&e->kref, enclave_release); + kref_put(&e->kref, e->abi->enclave_release); sc_link->e = NULL; } @@ -702,7 +702,7 @@ int ghost_bpf_link_attach(const union bpf_attr *attr, if (err) { /* bpf_link_cleanup() triggers .dealloc, but not .release. */ bpf_link_cleanup(&link_primer); - kref_put(&e->kref, enclave_release); + kref_put(&e->kref, e->abi->enclave_release); return err; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 5832b25ffacd..2984de7013dc 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -205,7 +205,6 @@ struct ghost_enclave { }; /* In kernel/sched/ghost.c */ -extern void enclave_release(struct kref *k); extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); @@ -2196,6 +2195,7 @@ struct ghost_abi { struct ghost_enclave * (*create_enclave)(ghost_abi_ptr_t abi, struct kernfs_node *dir, ulong id); + void (*enclave_release)(struct kref *k); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); int (*bpf_wake_agent)(int cpu); From 115c7fbde4dc9888f3bbed4e52fe8d083f1d9d8a Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 3 Nov 2021 19:25:10 -0700 Subject: [PATCH 086/165] sched: move ghost_fdget_enclave() in abi-independent code. See "a8f4f8b: fix ghostfs refcounting bug" for the original commit. There wasn't an obvious way to split this along ABI dependent lines. For e.g. one choice would have been to provide a callback to verify whether a given 'struct file' referred to the enclave ctl node. Then assuming that the enclave pointer was stashed in 'kn->priv' we could have implemented this entirely in ghost_core.c. However this imposes restrictions on how an ABI uses 'kernfs_node::priv' which seems less than ideal. The approach in this KCL leaves the mapping of 'struct file' to an enclave entirely up to the ABI (okay but also less than ideal since it separates managing references on struct file (ghost_core.c) and struct kernfs_node (ghost.c)). There is also the linear search across all ABIs to find the right one that can claim the 'struct file' as its own. This should not be an issue in practice assuming we limit ourselves to supporting 2 or 3 ABIs. Finally a note why we need to carry 'struct fd' across the get/put APIs. Technically we could fdput in ghost_fdget_enclave() since 'abi->ctlfd_enclave_get()' must have made arrangements to ensure stability of the returned enclave pointer. However since it does this indirectly via kernfs_get_active() there needs to be a way to get to it again in 'ctlfd_enclave_put()' to put the reference. For this reason we carry the 'struct fd' to get to the 'file' and then to 'kernfs_node'. Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: I00bd32f4dfe311d1d94c023735f708f29309af16 --- kernel/sched/ghost.c | 41 +++++---------------------------------- kernel/sched/ghost_core.c | 40 ++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 14 ++++++++++--- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index de546b45a584..51dfa7b81ad2 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -93,8 +93,6 @@ static void __enclave_remove_task(struct ghost_enclave *e, static int __sw_region_free(struct ghost_sw_region *region); static const struct file_operations queue_fops; -static struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f); -static void ghostfs_put_enclave_ctl(struct file *f); static void ghost_destroy_enclave(struct ghost_enclave *e); static void enclave_release(struct kref *k); @@ -2219,37 +2217,6 @@ static int ghost_enclave_set_cpus(struct ghost_enclave *e, return ret; } -/* - * Helper to get an fd and translate it to an enclave. Ghostfs will maintain a - * kref on the enclave. That kref is increffed when the file was opened, and - * that kref is maintained by this call (a combination of fdget and - * kernfs_get_active). Once you call ghost_fdput_enclave(), the enclave kref - * can be released. - * - * Returns NULL if there is not a ghost_enclave for this FD, which could be due - * to concurrent enclave destruction. - * - * Caller must call ghost_fdput_enclave with e and &fd_to_put, even on error. - */ -struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) -{ - *fd_to_put = fdget(fd); - if (!fd_to_put->file) - return NULL; - return ghostfs_ctl_to_enclave(fd_to_put->file); -} - -/* - * Pairs with calls to ghost_fdget_enclave(). You can't call this while holding - * the rq lock. - */ -void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put) -{ - if (e) - ghostfs_put_enclave_ctl(fd_to_put->file); - fdput(*fd_to_put); -} - static struct ghost_queue *fd_to_queue(struct fd f) { if (!f.file) @@ -6919,7 +6886,7 @@ static struct kernfs_ops gf_ops_e_ctl = { * need to do it manually here, because this is is a "backdoor" function to get * the enclave pointer. That pointer is kept alive by kernfs. */ -static struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) +static struct ghost_enclave *ctlfd_enclave_get(struct file *f) { struct kernfs_node *kn; @@ -6936,8 +6903,8 @@ static struct ghost_enclave *ghostfs_ctl_to_enclave(struct file *f) return kn->priv; } -/* Pair this with a successful ghostfs_ctl_to_enclave call. */ -static void ghostfs_put_enclave_ctl(struct file *f) +/* Pair this with a successful ctlfd_enclave_get() call. */ +static void ctlfd_enclave_put(struct file *f) { struct kernfs_node *kn; @@ -7509,6 +7476,8 @@ DEFINE_GHOST_ABI(current_abi) = { .abi_init = abi_init, .create_enclave = create_enclave, .enclave_release = enclave_release, + .ctlfd_enclave_get = ctlfd_enclave_get, + .ctlfd_enclave_put = ctlfd_enclave_put, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .bpf_wake_agent = bpf_wake_agent, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 3133bcb04c09..5614ed8f2d11 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -438,6 +438,46 @@ void __init init_sched_ghost_class(void) per_cpu(sync_group_cookie, cpu) = cpu << SG_COOKIE_CPU_SHIFT; } +/* + * Helper to get an fd and translate it to an enclave. Ghostfs will maintain a + * kref on the enclave. That kref is increffed when the file was opened, and + * that kref is maintained by this call (a combination of fdget and + * kernfs_get_active). Once you call ghost_fdput_enclave(), the enclave kref + * can be released. + * + * Returns NULL if there is not a ghost_enclave for this FD, which could be due + * to concurrent enclave destruction. + * + * Caller must call ghost_fdput_enclave with e and &fd_to_put, even on error. + */ +struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) +{ + ghost_abi_ptr_t abi; + struct ghost_enclave *e = NULL; + + *fd_to_put = fdget(fd); + if (!fd_to_put->file) + return NULL; + + for_each_abi(abi) { + e = abi->ctlfd_enclave_get(fd_to_put->file); + if (e) + break; + } + return e; +} + +/* + * Pairs with calls to ghost_fdget_enclave(). You can't call this while holding + * the rq lock. + */ +void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd) +{ + if (e) + e->abi->ctlfd_enclave_put(fd->file); + fdput(*fd); +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 2984de7013dc..605cc9287beb 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -205,9 +205,6 @@ struct ghost_enclave { }; /* In kernel/sched/ghost.c */ -extern struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); -extern void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); - extern void init_sched_ghost_class(void); extern void init_ghost_rq(struct ghost_rq *ghost_rq); extern bool ghost_agent(const struct sched_attr *attr); @@ -2196,6 +2193,8 @@ struct ghost_abi { (*create_enclave)(ghost_abi_ptr_t abi, struct kernfs_node *dir, ulong id); void (*enclave_release)(struct kref *k); + struct ghost_enclave *(*ctlfd_enclave_get)(struct file *file); + void (*ctlfd_enclave_put)(struct file *file); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); int (*bpf_wake_agent)(int cpu); @@ -2235,6 +2234,15 @@ int64_t ghost_sync_group_cookie(void); void ghost_wait_for_rendezvous(struct rq *rq); void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); +/* + * Helper to map an 'fd' to a 'ghost_enclave'. Must always be followed with + * a call to ghost_fdput_enclave() even if the lookup fails. + * + * N.B. ghost_fdput_enclave() cannot be called with 'rq->lock' held. + */ +struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); +void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); + static inline int enclave_abi(const struct ghost_enclave *e) { return e->abi->version; From b3084c988bf59be32d84afcc485e961b19b27fe8 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 3 Nov 2021 20:11:14 -0700 Subject: [PATCH 087/165] sched: make setsched into ghost abi-independent Prior to this change it was possible for an ABI to redefine the magic numbers to indicate whether a task entering ghost was a regular task or an agent. We lock this down and make this independent of the ABI for a couple of reasons: - the magic numbers are probably the easiest to change but also the least likely to need to change (note also that we have tools like 'pushtosched' that don't include the ghost uapi and hard code the magic values). - we have already de-facto conventions like passing the the enclave ctl_fd in 'sched_runtime' or queue_fd in in 'sched_deadline'. The immediate benefit is that we don't need to add trivial abi callbacks to validate sched_attr etc. We can always add this if needed in the future. Tested: all unit tests passing in virtme. Effort: sched/ghost/abi Change-Id: I150a3b2eae13ae7ba68702bc72475b20afbedd70 --- include/uapi/linux/ghost.h | 6 ------ kernel/sched/ghost.c | 21 --------------------- kernel/sched/ghost_core.c | 23 +++++++++++++++++++++++ kernel/sched/sched.h | 5 +++-- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index e1ab08e7d9d1..884d80d28614 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -55,12 +55,6 @@ enum ghost_type { GHOST_TASK, }; -enum { - GHOST_SCHED_TASK_PRIO, - GHOST_SCHED_AGENT_PRIO, - GHOST_SCHED_MAX_PRIO, -}; - /* * Collateral for GHOST_CONFIG_QUEUE_WAKEUP op: * - kernel supports up to 'GHOST_MAX_WINFO' wakeup targets for a queue. diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 51dfa7b81ad2..ac0539101abe 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3015,27 +3015,6 @@ int ghost_setscheduler(struct task_struct *p, struct rq *rq, return ret; } -int ghost_validate_sched_attr(const struct sched_attr *attr) -{ - /* - * A thread can only make a task an agent if the thread has the - * CAP_SYS_NICE capability. - */ - switch (attr->sched_priority) { - case GHOST_SCHED_TASK_PRIO: - return 0; - case GHOST_SCHED_AGENT_PRIO: - return capable(CAP_SYS_NICE) ? 0 : -EPERM; - default: - return -EINVAL; - } -} - -bool ghost_agent(const struct sched_attr *attr) -{ - return attr->sched_priority == GHOST_SCHED_AGENT_PRIO; -} - /* Makes newq the default for e. */ static void enclave_set_default_queue(struct ghost_enclave *e, struct ghost_queue *newq) diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 5614ed8f2d11..ce6224066429 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -478,6 +478,29 @@ void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd) fdput(*fd); } +#define GHOST_SCHED_TASK_PRIO 0 +#define GHOST_SCHED_AGENT_PRIO 1 +int ghost_validate_sched_attr(const struct sched_attr *attr) +{ + /* + * A thread can only make a task an agent if the thread has the + * CAP_SYS_NICE capability. + */ + switch (attr->sched_priority) { + case GHOST_SCHED_TASK_PRIO: + return 0; + case GHOST_SCHED_AGENT_PRIO: + return capable(CAP_SYS_NICE) ? 0 : -EPERM; + default: + return -EINVAL; + } +} + +bool ghost_agent(const struct sched_attr *attr) +{ + return attr->sched_priority == GHOST_SCHED_AGENT_PRIO; +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 605cc9287beb..c825820ec810 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -207,8 +207,6 @@ struct ghost_enclave { /* In kernel/sched/ghost.c */ extern void init_sched_ghost_class(void); extern void init_ghost_rq(struct ghost_rq *ghost_rq); -extern bool ghost_agent(const struct sched_attr *attr); -extern int ghost_validate_sched_attr(const struct sched_attr *attr); extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, struct ghost_enclave *new_e, @@ -2243,6 +2241,9 @@ void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); +bool ghost_agent(const struct sched_attr *attr); +int ghost_validate_sched_attr(const struct sched_attr *attr); + static inline int enclave_abi(const struct ghost_enclave *e) { return e->abi->version; From 4ed0015797aef1bf3ddeceba3186dfb3803aae32 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 4 Nov 2021 19:25:28 -0700 Subject: [PATCH 088/165] sched: ghost_prepare_task_switch() is abi-dependent. We get rid of the '#include ' from core.c as part of this commit. This commit exposed a limitation of the some of the callbacks in ghost_abi in that they commingle update of cpu and task state in a single callback. For e.g. 'prepare_task_switch' callback updates the agent status_word to indicate cpu availability and also updates the outgoing task's status_word indicating it is no longer oncpu. However it is possible for a running task to enter ghost in one enclave (say X) but the cpu it is running on belongs to another enclave (say Y). Now we have a problem because there is only a single 'prepare_task_switch' callback that we invoke. We defer fixing this as part of this patchset since it would need a more intrusive fix. Thus this patchset allows selecting an ABI at runtime, and it also supports running multiple enclaves at the same time as long all enclaves are at the same version. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: If1aa046c883f93dd678107a2fa186da53a4f663c --- kernel/sched/core.c | 124 --------------------------------- kernel/sched/ghost.c | 142 ++++++++++++++++++++++++++++++++++++-- kernel/sched/ghost_core.c | 47 ++++++++++++- kernel/sched/sched.h | 13 ++-- 4 files changed, 187 insertions(+), 139 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 078d49961825..6b2d8d8f8af4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4239,130 +4239,6 @@ static inline void kmap_local_sched_in(void) #endif } -#ifdef CONFIG_SCHED_CLASS_GHOST -#include -static inline void ghost_prepare_task_switch(struct rq *rq, - struct task_struct *prev, - struct task_struct *next) -{ - struct ghost_status_word *agent_sw; - - if (rq->ghost.agent) { - /* - * XXX pick_next_task_fair() can return 'rq->idle' via - * core_tag_pick_next_matching_rendezvous(). - */ - agent_sw = rq->ghost.agent->ghost.status_word; - if (!task_has_ghost_policy(next) && next != rq->idle) { - ghost_sw_clear_flag(agent_sw, GHOST_SW_CPU_AVAIL); - if (rq->ghost.run_flags & NEED_CPU_NOT_IDLE) { - rq->ghost.run_flags &= ~NEED_CPU_NOT_IDLE; - ghost_need_cpu_not_idle(rq, next); - } - } else { - ghost_sw_set_flag(agent_sw, GHOST_SW_CPU_AVAIL); - } - } - - if (!task_has_ghost_policy(prev)) - goto done; - - /* Who watches the watchmen? */ - if (prev == rq->ghost.agent) - goto done; - - /* - * An oncpu task may switch into ghost (e.g. via a third party calling - * sched_setscheduler(2)) while the rq->lock is dropped in one of the - * pick_next_task handlers (e.g. during CFS idle load balancing). - * - * It is not possible to distinguish this in switched_to_ghost() so - * we resolve 'prev->ghost.new_task' here. Failing to do this has - * dire consequences if 'prev' is runnable since it will languish - * in the kernel forever (in contrast to a blocked task there is - * no trigger forthcoming to produce a TASK_NEW in the future). - */ - if (unlikely(prev->ghost.new_task)) { - WARN_ON_ONCE(prev->ghost.yield_task); - WARN_ON_ONCE(prev->ghost.blocked_task); - - /* - * We just produced TASK_NEW so no need to consider 'prev' for - * a preemption edge (see ghost_produce_prev_msgs for details). - */ - rq->ghost.check_prev_preemption = false; - - prev->ghost.new_task = false; - ghost_task_new(rq, prev); - ghost_wake_agent_of(prev); - } - - /* If we're on the ghost.tasks list, then we're runnable. */ - if (!list_empty(&prev->ghost.run_list)) { - /* - * Keep ghost.tasks sorted by last_runnable_at. Whenever we set - * the time, we append to the tail. - */ - list_del_init(&prev->ghost.run_list); - list_add_tail(&prev->ghost.run_list, &rq->ghost.tasks); - prev->ghost.last_runnable_at = ktime_get(); - } - - if (rq->ghost.check_prev_preemption) { - rq->ghost.check_prev_preemption = false; - ghost_task_preempted(rq, prev); - ghost_wake_agent_of(prev); - } - -done: - /* - * Clear the ONCPU bit after producing the task state change msg - * (e.g. preempted). This guarantees that when a task is offcpu - * its 'task_barrier' is stable. - */ - if (task_has_ghost_policy(prev)) { - struct ghost_status_word *prev_sw = prev->ghost.status_word; - WARN_ON_ONCE(!(prev_sw->flags & GHOST_SW_TASK_ONCPU)); - ghost_sw_clear_flag(prev_sw, GHOST_SW_TASK_ONCPU); - } - - /* - * CPU is switching to a non-ghost task while a task is latched. - * - * Treat this like latched_task preemption because we don't know when - * the CPU will be available again so no point in keeping it latched. - */ - if (rq->ghost.latched_task) { - /* - * If 'next' was returned from pick_next_task_ghost() then - * 'latched_task' must have been cleared. Conversely if - * there is 'latched_task' then 'next' could not have - * been returned from pick_next_task_ghost(). - */ - WARN_ON_ONCE(task_has_ghost_policy(next) && - !is_agent(rq, next)); - ghost_latched_task_preempted(rq); - - /* - * XXX the WARN above is susceptible to a false-negative - * when pick_next_task_ghost returns the idle task. This - * is not the common case but it highlights that what we - * really need to check is the sched_class that produced - * 'next'. - */ - } - - if (task_has_ghost_policy(next) && !is_agent(rq, next)) - ghost_task_got_oncpu(rq, next); - - /* - * The last task in the chain scheduled (blocked/yielded/preempted). - */ - if (rq->ghost.switchto_count < 0) - rq->ghost.switchto_count = 0; -} -#endif /* CONFIG_SCHED_CLASS_GHOST */ - /** * prepare_task_switch - prepare to switch tasks * @rq: the runqueue preparing to switch diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index ac0539101abe..abbc73c1fd16 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -78,9 +78,14 @@ static void ghost_commit_all_greedy_txns(void); static void ghost_commit_pending_txn(int where); static void release_from_ghost(struct rq *rq, struct task_struct *p); static void ghost_wake_agent_on(int cpu); +static void ghost_wake_agent_of(struct task_struct *p); +static void ghost_latched_task_preempted(struct rq *rq); +static void ghost_task_preempted(struct rq *rq, struct task_struct *p); static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, bool was_latched); +static void ghost_task_new(struct rq *rq, struct task_struct *p); + struct rq *context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf); @@ -1211,7 +1216,7 @@ static bool ghost_produce_prev_msgs(struct rq *rq, struct task_struct *prev) return true; } -void ghost_latched_task_preempted(struct rq *rq) +static void ghost_latched_task_preempted(struct rq *rq) { struct task_struct *latched = rq->ghost.latched_task; @@ -4606,7 +4611,7 @@ static int ghost_wake_agent_on_check(int cpu) return ret; } -void ghost_wake_agent_of(struct task_struct *p) +static void ghost_wake_agent_of(struct task_struct *p) { ghost_wake_agent_on(task_target_cpu(p)); } @@ -4650,7 +4655,7 @@ static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, schedule_agent(rq, false); } -void ghost_task_preempted(struct rq *rq, struct task_struct *p) +static void ghost_task_preempted(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); VM_BUG_ON(task_rq(p) != rq); @@ -4658,7 +4663,7 @@ void ghost_task_preempted(struct rq *rq, struct task_struct *p) _ghost_task_preempted(rq, p, false); } -void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p) +static void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); VM_BUG_ON(task_rq(p) != rq); @@ -4689,7 +4694,7 @@ static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable) task_deliver_msg_task_new(rq, p, runnable); } -void ghost_task_new(struct rq *rq, struct task_struct *p) +static void ghost_task_new(struct rq *rq, struct task_struct *p) { _ghost_task_new(rq, p, task_on_rq_queued(p)); } @@ -6283,7 +6288,7 @@ void ghost_switchto(struct rq *rq, struct task_struct *prev, } } -void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next) +static void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next) { if (cpu_deliver_msg_not_idle(rq, next)) ghost_wake_agent_on(agent_target_cpu(rq)); @@ -7269,6 +7274,130 @@ static void pnt_prologue(struct rq *rq, struct task_struct *prev) rq->ghost.pnt_bpf_once = true; } +/* + * Called in the context switch path when switching from 'prev' to 'next' + * (either via a normal schedule or switchto). + */ +static void prepare_task_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) +{ + struct ghost_status_word *agent_sw; + + if (rq->ghost.agent) { + /* + * XXX pick_next_task_fair() can return 'rq->idle' via + * core_tag_pick_next_matching_rendezvous(). + */ + agent_sw = rq->ghost.agent->ghost.status_word; + if (!task_has_ghost_policy(next) && next != rq->idle) { + ghost_sw_clear_flag(agent_sw, GHOST_SW_CPU_AVAIL); + if (rq->ghost.run_flags & NEED_CPU_NOT_IDLE) { + rq->ghost.run_flags &= ~NEED_CPU_NOT_IDLE; + ghost_need_cpu_not_idle(rq, next); + } + } else { + ghost_sw_set_flag(agent_sw, GHOST_SW_CPU_AVAIL); + } + } + + if (!task_has_ghost_policy(prev)) + goto done; + + /* Who watches the watchmen? */ + if (prev == rq->ghost.agent) + goto done; + + /* + * An oncpu task may switch into ghost (e.g. via a third party calling + * sched_setscheduler(2)) while the rq->lock is dropped in one of the + * pick_next_task handlers (e.g. during CFS idle load balancing). + * + * It is not possible to distinguish this in switched_to_ghost() so + * we resolve 'prev->ghost.new_task' here. Failing to do this has + * dire consequences if 'prev' is runnable since it will languish + * in the kernel forever (in contrast to a blocked task there is + * no trigger forthcoming to produce a TASK_NEW in the future). + */ + if (unlikely(prev->ghost.new_task)) { + WARN_ON_ONCE(prev->ghost.yield_task); + WARN_ON_ONCE(prev->ghost.blocked_task); + + /* + * We just produced TASK_NEW so no need to consider 'prev' for + * a preemption edge (see ghost_produce_prev_msgs for details). + */ + rq->ghost.check_prev_preemption = false; + + prev->ghost.new_task = false; + ghost_task_new(rq, prev); + ghost_wake_agent_of(prev); + } + + /* If we're on the ghost.tasks list, then we're runnable. */ + if (!list_empty(&prev->ghost.run_list)) { + /* + * Keep ghost.tasks sorted by last_runnable_at. Whenever we set + * the time, we append to the tail. + */ + list_del_init(&prev->ghost.run_list); + list_add_tail(&prev->ghost.run_list, &rq->ghost.tasks); + prev->ghost.last_runnable_at = ktime_get(); + } + + if (rq->ghost.check_prev_preemption) { + rq->ghost.check_prev_preemption = false; + ghost_task_preempted(rq, prev); + ghost_wake_agent_of(prev); + } + +done: + /* + * Clear the ONCPU bit after producing the task state change msg + * (e.g. preempted). This guarantees that when a task is offcpu + * its 'task_barrier' is stable. + */ + if (task_has_ghost_policy(prev)) { + struct ghost_status_word *prev_sw = prev->ghost.status_word; + WARN_ON_ONCE(!(prev_sw->flags & GHOST_SW_TASK_ONCPU)); + ghost_sw_clear_flag(prev_sw, GHOST_SW_TASK_ONCPU); + } + + /* + * CPU is switching to a non-ghost task while a task is latched. + * + * Treat this like latched_task preemption because we don't know when + * the CPU will be available again so no point in keeping it latched. + */ + if (rq->ghost.latched_task) { + /* + * If 'next' was returned from pick_next_task_ghost() then + * 'latched_task' must have been cleared. Conversely if + * there is 'latched_task' then 'next' could not have + * been returned from pick_next_task_ghost(). + */ + WARN_ON_ONCE(task_has_ghost_policy(next) && + !is_agent(rq, next)); + ghost_latched_task_preempted(rq); + + /* + * XXX the WARN above is susceptible to a false-negative + * when pick_next_task_ghost returns the idle task. This + * is not the common case but it highlights that what we + * really need to check is the sched_class that produced + * 'next'. + */ + } + + if (task_has_ghost_policy(next) && !is_agent(rq, next)) + ghost_task_got_oncpu(rq, next); + + /* + * The last task in the chain scheduled (blocked/yielded/preempted). + */ + if (rq->ghost.switchto_count < 0) + rq->ghost.switchto_count = 0; +} + static int bpf_wake_agent(int cpu) { return ghost_wake_agent_on_check(cpu); @@ -7459,6 +7588,7 @@ DEFINE_GHOST_ABI(current_abi) = { .ctlfd_enclave_put = ctlfd_enclave_put, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, + .prepare_task_switch = prepare_task_switch, .bpf_wake_agent = bpf_wake_agent, .bpf_run_gtid = bpf_run_gtid, .ghost_msg_is_valid_access = ghost_msg_is_valid_access, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index ce6224066429..51697c494f78 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -418,13 +418,27 @@ void ghost_wait_for_rendezvous(struct rq *rq) void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev) { - struct ghost_enclave *e; + struct ghost_enclave *this_enclave, *prev_enclave, *e; VM_BUG_ON(preemptible()); VM_BUG_ON(rq != this_rq()); /* rcu_read_lock_sched() not needed; preemption is disabled. */ - e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + this_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + prev_enclave = prev->ghost.enclave; + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(this_enclave && prev_enclave && this_enclave != prev_enclave); + + /* + * 'this_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = this_enclave ? this_enclave : prev_enclave; if (e != NULL) e->abi->pnt_prologue(rq, prev); @@ -501,6 +515,35 @@ bool ghost_agent(const struct sched_attr *attr) return attr->sched_priority == GHOST_SCHED_AGENT_PRIO; } +void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next) +{ + struct ghost_enclave *this_enclave, *prev_enclave, *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + this_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + prev_enclave = prev->ghost.enclave; + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(this_enclave && prev_enclave && this_enclave != prev_enclave); + + /* + * 'this_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = this_enclave ? this_enclave : prev_enclave; + + if (e != NULL) + e->abi->prepare_task_switch(rq, prev, next); +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c825820ec810..daa6e4df7146 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -205,7 +205,6 @@ struct ghost_enclave { }; /* In kernel/sched/ghost.c */ -extern void init_sched_ghost_class(void); extern void init_ghost_rq(struct ghost_rq *ghost_rq); extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, @@ -213,19 +212,13 @@ extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, int *reset_on_fork); extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); -extern void ghost_latched_task_preempted(struct rq *rq); -extern void ghost_task_preempted(struct rq *rq, struct task_struct *prev); -extern void ghost_task_got_oncpu(struct rq *rq, struct task_struct *p); extern unsigned long ghost_cfs_added_load(struct rq *rq); -extern void ghost_wake_agent_of(struct task_struct *p); extern void ghost_agent_schedule(void); extern void ghost_cpu_idle(void); -extern void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next); extern void ghost_tick(struct rq *rq); extern int64_t ghost_alloc_gtid(struct task_struct *p); extern void ghost_initialize_status_word(struct task_struct *p); -extern void ghost_task_new(struct rq *rq, struct task_struct *p); extern void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); @@ -2195,6 +2188,8 @@ struct ghost_abi { void (*ctlfd_enclave_put)(struct file *file); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); + void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, + struct task_struct *next); int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); bool (*ghost_msg_is_valid_access)(int off, int size, @@ -2223,6 +2218,7 @@ const static struct ghost_abi __##name##_ghost_abi \ _GHOST_MAYBE_CONST DECLARE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); +void init_sched_ghost_class(void); int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *cpus); void ghost_publish_cpu(struct ghost_enclave *e, int cpu); void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu); @@ -2248,6 +2244,9 @@ static inline int enclave_abi(const struct ghost_enclave *e) { return e->abi->version; } + +void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, + struct task_struct *next); #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From 45a13bc8f4d3016c53aa93086c3ae8fa865750db Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 4 Nov 2021 19:46:25 -0700 Subject: [PATCH 089/165] sched: add 'cpu_idle' callback to 'ghost_abi' Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: I4da741b322d8ac691f45d51e354f7551962a1dda --- kernel/sched/ghost.c | 6 ++---- kernel/sched/ghost_core.c | 15 +++++++++++++++ kernel/sched/sched.h | 7 +++++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index abbc73c1fd16..8f306b6ccbd7 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6294,13 +6294,10 @@ static void ghost_need_cpu_not_idle(struct rq *rq, struct task_struct *next) ghost_wake_agent_on(agent_target_cpu(rq)); } -void ghost_cpu_idle(void) +static void cpu_idle(struct rq *rq) { - struct rq *rq = this_rq(); struct rq_flags rf; - WARN_ON_ONCE(current != rq->idle); - rq_lock_irq(rq, &rf); if (rq->ghost.dont_idle_once) { set_tsk_need_resched(current); @@ -7589,6 +7586,7 @@ DEFINE_GHOST_ABI(current_abi) = { .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .prepare_task_switch = prepare_task_switch, + .cpu_idle = cpu_idle, .bpf_wake_agent = bpf_wake_agent, .bpf_run_gtid = bpf_run_gtid, .ghost_msg_is_valid_access = ghost_msg_is_valid_access, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 51697c494f78..5feea39e3bbf 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -544,6 +544,21 @@ void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, e->abi->prepare_task_switch(rq, prev, next); } +void ghost_cpu_idle(void) +{ + struct ghost_enclave *e; + struct rq *rq = this_rq(); + + WARN_ON_ONCE(preemptible()); + WARN_ON_ONCE(current != rq->idle); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + if (e) + e->abi->cpu_idle(rq); +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index daa6e4df7146..4f1fed0a0a40 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -213,8 +213,6 @@ extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); extern unsigned long ghost_cfs_added_load(struct rq *rq); -extern void ghost_agent_schedule(void); -extern void ghost_cpu_idle(void); extern void ghost_tick(struct rq *rq); extern int64_t ghost_alloc_gtid(struct task_struct *p); @@ -2190,6 +2188,7 @@ struct ghost_abi { void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); + void (*cpu_idle)(struct rq *rq); int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); bool (*ghost_msg_is_valid_access)(int off, int size, @@ -2247,6 +2246,10 @@ static inline int enclave_abi(const struct ghost_enclave *e) void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next); +void ghost_cpu_idle(void); + +/* ghost functions in core.c */ +void ghost_agent_schedule(void); #endif /* CONFIG_SCHED_CLASS_GHOST */ static inline bool sched_stop_runnable(struct rq *rq) From b36a954c357f607132e6c828c7e9f5a3b2289b55 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 4 Nov 2021 20:19:49 -0700 Subject: [PATCH 090/165] sched: pull ghost_cfs_added_load() into abi-independent code. While a case can be made that an ABI might want to change how to feed load into CFS there is nothing on the horizon to suggest it it likely. We could always introduce an indirection at the time it is needed. This does bring some restrictions on the meaning of some fields in 'rq->ghost' (viz. ghost_nr_running, agent, blocked_in_run and latched_task) but these fields are deeply ingrained and its hard to imagine an ABI doing something contrary to the most obvious interpretation of the field names. Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: I5adab6b4a5ceb145691b1d63f63d5c899499ead0 --- kernel/sched/ghost.c | 52 --------------------------------------- kernel/sched/ghost_core.c | 52 +++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 3 ++- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8f306b6ccbd7..4f834d6bcc84 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -52,9 +52,6 @@ struct ghost_sw_region { struct ghost_enclave *enclave; }; -/* The load contribution that CFS sees for a running ghOSt task */ -unsigned long sysctl_ghost_cfs_load_added = 1024; - static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable); static void ghost_task_yield(struct rq *rq, struct task_struct *p); @@ -6307,55 +6304,6 @@ static void cpu_idle(struct rq *rq) rq_unlock_irq(rq, &rf); } -unsigned long ghost_cfs_added_load(struct rq *rq) -{ - int ghost_nr_running = rq->ghost.ghost_nr_running; - struct task_struct *curr; - bool add_load = false; - - /* No ghost tasks; nothing to contribute load. */ - if (!ghost_nr_running) - return 0; - - /* - * We have a few cases where we want to add load: - * (a): We have a local agent that is not blocked_in_run. - * (b): Currently have a non-agent ghost task running. - * (c): Have a latched task that is not yet running. We - * treat this the same as case (b), since this is really - * just a race over getting through schedule() (modulo possible - * preemption by another sched_class). - */ - - if (ghost_nr_running > __ghost_extra_nr_running(rq)) { - /* (a) */ - add_load = true; - goto out; - } - - rcu_read_lock(); - curr = READ_ONCE(rq->curr); - if (task_has_ghost_policy(curr) && !is_agent(rq, curr) && - curr->state == TASK_RUNNING) { - /* (b) */ - add_load = true; - } - curr = NULL; /* don't use outside of RCU */ - rcu_read_unlock(); - if (add_load) - goto out; - - if (rq->ghost.latched_task) { - /* (c) */ - add_load = true; - } - -out: - if (add_load) - return sysctl_ghost_cfs_load_added; - return 0; -} - #include "../../fs/kernfs/kernfs-internal.h" /* Helper for when you "echo foo > ctl" without the -n. */ diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 5feea39e3bbf..6f13a96c34df 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -3,6 +3,9 @@ #define _GHOST_MAYBE_CONST #include "sched.h" +/* The load contribution that CFS sees for a running ghOSt task */ +unsigned long sysctl_ghost_cfs_load_added = 1024; + static struct kernfs_root *ghost_kfs_root; extern const struct ghost_abi __begin_ghost_abi[]; @@ -559,6 +562,55 @@ void ghost_cpu_idle(void) e->abi->cpu_idle(rq); } +unsigned long ghost_cfs_added_load(struct rq *rq) +{ + int ghost_nr_running = rq->ghost.ghost_nr_running; + struct task_struct *curr; + bool add_load = false; + + /* No ghost tasks; nothing to contribute load. */ + if (!ghost_nr_running) + return 0; + + /* + * We have a few cases where we want to add load: + * (a): We have a local agent that is not blocked_in_run. + * (b): Currently have a non-agent ghost task running. + * (c): Have a latched task that is not yet running. We + * treat this the same as case (b), since this is really + * just a race over getting through schedule() (modulo possible + * preemption by another sched_class). + */ + + if (ghost_nr_running > __ghost_extra_nr_running(rq)) { + /* (a) */ + add_load = true; + goto out; + } + + rcu_read_lock(); + curr = READ_ONCE(rq->curr); + if (task_has_ghost_policy(curr) && !is_agent(rq, curr) && + curr->state == TASK_RUNNING) { + /* (b) */ + add_load = true; + } + curr = NULL; /* don't use outside of RCU */ + rcu_read_unlock(); + if (add_load) + goto out; + + if (rq->ghost.latched_task) { + /* (c) */ + add_load = true; + } + +out: + if (add_load) + return sysctl_ghost_cfs_load_added; + return 0; +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4f1fed0a0a40..f42f613558c0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -212,7 +212,6 @@ extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, int *reset_on_fork); extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); -extern unsigned long ghost_cfs_added_load(struct rq *rq); extern void ghost_tick(struct rq *rq); extern int64_t ghost_alloc_gtid(struct task_struct *p); @@ -2248,6 +2247,8 @@ void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next); void ghost_cpu_idle(void); +unsigned long ghost_cfs_added_load(struct rq *rq); + /* ghost functions in core.c */ void ghost_agent_schedule(void); #endif /* CONFIG_SCHED_CLASS_GHOST */ From ea09fac51b4daae87430dddac8ca5d146cb0e62b Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Fri, 5 Nov 2021 17:36:04 -0700 Subject: [PATCH 091/165] sched: gtid allocation is independent of the ABI. The width of the sequence number in a 'gtid' is directly dependent on the maximum possible value of a task pid. Thus the layout cannot be specified by the ABI (for e.g. imagine a kernel rebase that bumps the value of PID_MAX_LIMIT; all existing ABIs would need to bump the GHOST_TID_PID_BITS correspondingly which doesn't work). Instead export PID_MAX_LIMIT via /proc/sys/kernel/pid_max_max to let userspace calculate how many bits to allocate for the 'tid' portion of the 'gtid' and then derive seqnum bits from that. Move ghost_alloc_gtid() into abi independent code. Submitted in conjunction with cl/407951843 Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: Id02441ed1ead0ae18eec9e67748dc84dbec2f9cc --- include/uapi/linux/ghost.h | 12 +----------- kernel/sched/ghost.c | 22 ---------------------- kernel/sched/ghost_core.c | 19 +++++++++++++++++++ kernel/sched/sched.h | 16 +++++++++++++++- kernel/sysctl.c | 7 +++++++ 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 884d80d28614..9126e3d348fc 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 58 +#define GHOST_VERSION 59 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -596,16 +596,6 @@ enum { GHOST_GTID_LOOKUP_TGID, /* return group_leader pid */ }; -/* - * ghost tids referring to normal tasks always have a positive value: - * (0 | 22 bits of actual pid_t | 41 bit non-zero seqnum) - * - * The embedded 'pid' following linux terminology is actually referring - * to the thread id (i.e. what would be returned by syscall(__NR_gettid)). - */ -#define GHOST_TID_SEQNUM_BITS 41 -#define GHOST_TID_PID_BITS 22 - // NOLINTEND #endif /* _UAPI_LINUX_GHOST_H_ */ diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 4f834d6bcc84..938fbb898674 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -32,9 +32,6 @@ #include "ghost_uapi.h" #include "ghost.h" -/* ghost tid */ -typedef int64_t gtid_t; - /* The ghost_txn pointer equals NULL or &enclave->cpu_data[this_cpu].txn */ static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_txn *, ghost_txn); @@ -2405,25 +2402,6 @@ void ghost_initialize_status_word(struct task_struct *p) ghost_sw_set_flag(sw, GHOST_SW_F_INUSE); } -int64_t ghost_alloc_gtid(struct task_struct *p) -{ - gtid_t gtid; - uint64_t seqnum; - - static atomic64_t gtid_seqnum = ATOMIC64_INIT(0); - - BUILD_BUG_ON(PID_MAX_LIMIT > (1UL << GHOST_TID_PID_BITS)); - - do { - seqnum = atomic64_add_return(1, >id_seqnum) & - ((1UL << GHOST_TID_SEQNUM_BITS) - 1); - } while (!seqnum); - - gtid = ((gtid_t)p->pid << GHOST_TID_SEQNUM_BITS) | seqnum; - WARN_ON_ONCE(gtid <= 0); - return gtid; -} - /* * Prepare task 'p' to participate in ghost scheduling. * diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 6f13a96c34df..c88f83579058 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -611,6 +611,25 @@ unsigned long ghost_cfs_added_load(struct rq *rq) return 0; } +int64_t ghost_alloc_gtid(struct task_struct *p) +{ + gtid_t gtid; + uint64_t seqnum; + + static atomic64_t gtid_seqnum = ATOMIC64_INIT(0); + + BUILD_BUG_ON(PID_MAX_LIMIT != (1UL << GHOST_TID_PID_BITS)); + + do { + seqnum = atomic64_add_return(1, >id_seqnum) & + ((1UL << GHOST_TID_SEQNUM_BITS) - 1); + } while (!seqnum); + + gtid = ((gtid_t)p->pid << GHOST_TID_SEQNUM_BITS) | seqnum; + WARN_ON_ONCE(gtid <= 0); + return gtid; +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f42f613558c0..9abdaeedb3a7 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -214,7 +214,6 @@ extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); extern void ghost_tick(struct rq *rq); -extern int64_t ghost_alloc_gtid(struct task_struct *p); extern void ghost_initialize_status_word(struct task_struct *p); extern void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); @@ -2173,7 +2172,21 @@ static inline bool skip_fair_idle_balance(struct cfs_rq *cfs_rq, return false; } +/* ghost tid */ +typedef int64_t gtid_t; + +/* + * ghost tids referring to normal tasks always have a positive value: + * (0 | 22 bits of actual pid_t | 41 bit non-zero seqnum) + * + * The embedded 'pid' following linux terminology is actually referring + * to the thread id (i.e. what would be returned by syscall(__NR_gettid)). + */ +#define GHOST_TID_SEQNUM_BITS 41 +#define GHOST_TID_PID_BITS 22 + typedef const struct ghost_abi *ghost_abi_ptr_t; + struct ghost_abi { int version; int (*abi_init)(ghost_abi_ptr_t abi); @@ -2248,6 +2261,7 @@ void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, void ghost_cpu_idle(void); unsigned long ghost_cfs_added_load(struct rq *rq); +int64_t ghost_alloc_gtid(struct task_struct *p); /* ghost functions in core.c */ void ghost_agent_schedule(void); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c20869705955..6f977c07dc84 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2180,6 +2180,13 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_ONE, }, #endif /* CONFIG_SMP */ + { + .procname = "pid_max_max", + .data = &pid_max_max, + .maxlen = sizeof(int), + .mode = 0444, + .proc_handler = proc_dointvec, + }, { .procname = "pid_max", .data = &pid_max, From 87372ca58548f0c20e197e8b451d9d26e2512da2 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sat, 6 Nov 2021 13:37:58 -0700 Subject: [PATCH 092/165] sched: cache 'abi' in ghost_fdget_enclave() Prior to this change every call to ghost_fdget_enclave() potentially had to iterate over every ABI compiled in the kernel before finding the one associated with the file descriptor. This is wasteful because in the common case we have one active enclave and a caching strategy should work well here. We cache a pointer to the ABI that successfully mapped the 'fd' to an enclave. Although we don't use locking we rely on atomic updates to the 'last_abi' such that it is always a valid pointer (since it is used as a hint the specific order in which concurrent updates are applied does not affect correctness; the only requirement is the atomicity of each update). Tested: all unit tests work in virtme Effort: sched/ghost/abi Change-Id: Id1268f0078955b4261ace1884a2d13ef631571b1 --- kernel/sched/ghost_core.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index c88f83579058..94c88457d080 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -472,14 +472,41 @@ struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) ghost_abi_ptr_t abi; struct ghost_enclave *e = NULL; + static ghost_abi_ptr_t last_abi; + *fd_to_put = fdget(fd); if (!fd_to_put->file) return NULL; - for_each_abi(abi) { + /* + * Try the ABI that we last used successfully to first do the lookup. + * + * In the common case of a single active enclave we avoid looping over + * all the ABIs compiled into the kernel. + */ + abi = READ_ONCE(last_abi); + if (abi) { e = abi->ctlfd_enclave_get(fd_to_put->file); if (e) + return e; + } + + for_each_abi(abi) { + e = abi->ctlfd_enclave_get(fd_to_put->file); + if (e) { + /* + * Update 'last_abi' if necessary. + * + * N.B. multiple CPUs may race to mutate 'last_abi' + * concurrently. This is okay as long as each update + * is atomic (i.e. last_abi always points to a valid + * abi). + */ + if (abi != READ_ONCE(last_abi)) + WRITE_ONCE(last_abi, abi); + break; + } } return e; } From 037b5ec3aed84ddbbad3172f2efd04e7fb074325 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 7 Nov 2021 18:30:57 -0800 Subject: [PATCH 093/165] sched: ghost_copy_process_epilogue() Tested: - all unit tests pass in virtme - $ fifo_agent& $ simple_exp Effort: sched/ghost/abi Change-Id: Iaf604673b97869cb0a7abe577d5845b1df709577 --- kernel/fork.c | 2 +- kernel/sched/ghost.c | 3 ++- kernel/sched/ghost_core.c | 10 ++++++++++ kernel/sched/sched.h | 3 ++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index d0210651e95f..eaa6098480f9 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2320,7 +2320,7 @@ static __latent_entropy struct task_struct *copy_process( #ifdef CONFIG_SCHED_CLASS_GHOST p->gtid = ghost_alloc_gtid(p); if (ghost_class(p->sched_class)) - ghost_initialize_status_word(p); + ghost_copy_process_epilogue(p); WARN_ON_ONCE(p->inhibit_task_msgs); WARN_ON_ONCE(!list_empty(&p->inhibited_task_list)); #endif diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 938fbb898674..377e93108c73 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -2381,7 +2381,7 @@ alloc_status_word_locked(struct ghost_enclave *e) return found; } -void ghost_initialize_status_word(struct task_struct *p) +static void ghost_initialize_status_word(struct task_struct *p) { struct ghost_status_word *sw = p->ghost.status_word; @@ -7512,6 +7512,7 @@ DEFINE_GHOST_ABI(current_abi) = { .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .prepare_task_switch = prepare_task_switch, + .copy_process_epilogue = ghost_initialize_status_word, .cpu_idle = cpu_idle, .bpf_wake_agent = bpf_wake_agent, .bpf_run_gtid = bpf_run_gtid, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 94c88457d080..d00cfb1290d6 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -657,6 +657,16 @@ int64_t ghost_alloc_gtid(struct task_struct *p) return gtid; } +void ghost_copy_process_epilogue(struct task_struct *p) +{ + struct ghost_enclave *e = p->ghost.enclave; + + if (WARN_ON_ONCE(!e)) + return; + + e->abi->copy_process_epilogue(p); +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9abdaeedb3a7..a860c0e317cf 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -214,7 +214,6 @@ extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); extern void ghost_tick(struct rq *rq); -extern void ghost_initialize_status_word(struct task_struct *p); extern void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); @@ -2200,6 +2199,7 @@ struct ghost_abi { void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); + void (*copy_process_epilogue)(struct task_struct *p); void (*cpu_idle)(struct rq *rq); int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); @@ -2250,6 +2250,7 @@ void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); bool ghost_agent(const struct sched_attr *attr); int ghost_validate_sched_attr(const struct sched_attr *attr); +void ghost_copy_process_epilogue(struct task_struct *p); static inline int enclave_abi(const struct ghost_enclave *e) { From a713c25d253b4e1b6847f5ea8a0faf49f0fbd08a Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Sun, 14 Nov 2021 20:39:30 -0800 Subject: [PATCH 094/165] sched: abi-specific ghost tick Add a 'tick' handler to the ghost_abi that is called either from scheduler_tick() or hrtick() in hardirq context with interrupts disabled with 'rq' unlocked. The handler is called in an rcu read side critical section to ensure that the per-cpu enclave pointer is stable. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I9301394da951c78c3af641cb8974efea82364ce3 --- kernel/sched/ghost.c | 34 ++++++++++++---------------------- kernel/sched/ghost_core.c | 21 +++++++++++++++++++++ kernel/sched/sched.h | 3 ++- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 377e93108c73..e0a17c6d21b3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -850,31 +850,20 @@ static bool check_runnable_timeout(struct ghost_enclave *e, struct rq *rq) return ok; } -/* - * Called from the timer tick handler after dropping rq->lock. Called - * regardless of whether a ghost task is current or not. - */ -void ghost_tick(struct rq *rq) +static void tick_handler(struct ghost_enclave *e, struct rq *rq) { - struct ghost_enclave *e; - - rcu_read_lock(); - e = rcu_dereference(*this_cpu_ptr(&enclave)); - if (e) { - if (READ_ONCE(e->commit_at_tick)) - ghost_commit_all_greedy_txns(); - if (!check_runnable_timeout(e, rq)) { - kref_get(&e->kref); - if (!schedule_work(&e->enclave_destroyer)) { - /* - * Safe since it is not the last reference, due - * to RCU protecting per_cpu(enclave). - */ - kref_put(&e->kref, enclave_release); - } + if (READ_ONCE(e->commit_at_tick)) + ghost_commit_all_greedy_txns(); + if (!check_runnable_timeout(e, rq)) { + kref_get(&e->kref); + if (!schedule_work(&e->enclave_destroyer)) { + /* + * Safe since it is not the last reference, due + * to RCU protecting per_cpu(enclave). + */ + kref_put(&e->kref, enclave_release); } } - rcu_read_unlock(); } /* @@ -7512,6 +7501,7 @@ DEFINE_GHOST_ABI(current_abi) = { .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .prepare_task_switch = prepare_task_switch, + .tick = tick_handler, .copy_process_epilogue = ghost_initialize_status_word, .cpu_idle = cpu_idle, .bpf_wake_agent = bpf_wake_agent, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index d00cfb1290d6..8fdea8136484 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -667,6 +667,27 @@ void ghost_copy_process_epilogue(struct task_struct *p) e->abi->copy_process_epilogue(p); } +/* + * Called from the timer tick handler after dropping rq->lock. Called + * regardless of whether a ghost task is current or not. + */ +void ghost_tick(struct rq *rq) +{ + struct ghost_enclave *e; + + WARN_ON_ONCE(preemptible()); + WARN_ON_ONCE(rq != this_rq()); + + /* + * Implicit read-side critical section since we are in hardirq context + * with interrupts disabled. + */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + if (e) + e->abi->tick(e, rq); +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a860c0e317cf..49eb098b4225 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -213,7 +213,6 @@ extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); -extern void ghost_tick(struct rq *rq); extern void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); @@ -2199,6 +2198,7 @@ struct ghost_abi { void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); + void (*tick)(struct ghost_enclave *e, struct rq *rq); void (*copy_process_epilogue)(struct task_struct *p); void (*cpu_idle)(struct rq *rq); int (*bpf_wake_agent)(int cpu); @@ -2238,6 +2238,7 @@ void ghost_return_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); void ghost_wait_for_rendezvous(struct rq *rq); void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); +void ghost_tick(struct rq *rq); /* * Helper to map an 'fd' to a 'ghost_enclave'. Must always be followed with From 8b71ffb3b1321d68ac7b1c205d149c514717d22a Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 17:53:33 -0800 Subject: [PATCH 095/165] sched: initialize 'rq->ghost' in abi-independent code. Ideally 'rq->ghost' would be initialized when a cpu is assigned to an enclave in ABI specific code. However because we allow tasks to enter ghost on arbitrary cpus (i.e. those not belonging to *any* enclave) it is necessary to initialize 'rq->ghost' on every cpu before tasks can setsched into ghost (e.g. enqueue_task_ghost() requires that 'rq->ghost.tasks' is properly initialized). This also highlights another deficiency that tasks belonging to an enclave_X can be enqueued on a cpu belonging to enclave_Y (either at the time of the initial setsched into ghost or a later wakeup; also if a cpu moved from enclave_X to enclave_Y while tasks from enclave_X are enqueued on its runqueue). Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: Ia93a83996a00d5dcf2f8dfac5d1712c6d85b3b26 --- kernel/sched/ghost.c | 6 ------ kernel/sched/ghost_core.c | 13 +++++++++++++ kernel/sched/sched.h | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index e0a17c6d21b3..166d43b31c69 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -373,12 +373,6 @@ static void submit_enclave_work(struct ghost_enclave *e, struct rq *rq, queue_balance_callback(rq, &rq->ghost.ew_head, __do_enclave_work); } -void init_ghost_rq(struct ghost_rq *ghost_rq) -{ - INIT_LIST_HEAD(&ghost_rq->tasks); - INIT_LIST_HEAD(&ghost_rq->enclave_work); -} - static inline gtid_t gtid(struct task_struct *p) { return p->gtid; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 8fdea8136484..a9f8f8204093 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -688,6 +688,19 @@ void ghost_tick(struct rq *rq) e->abi->tick(e, rq); } +/* + * TODO(multi-enclave): + * This should be done entirely within an ABI when a cpu is assigned + * to the corresponding enclave. However because we allow tasks to enter + * ghost on on arbitrary cpus (e.g. outside an enclave) then 'rq->ghost' + * must be properly initialized on those cpus as well. + */ +void init_ghost_rq(struct ghost_rq *ghost_rq) +{ + INIT_LIST_HEAD(&ghost_rq->tasks); + INIT_LIST_HEAD(&ghost_rq->enclave_work); +} + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 49eb098b4225..21a873a255c3 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -205,7 +205,6 @@ struct ghost_enclave { }; /* In kernel/sched/ghost.c */ -extern void init_ghost_rq(struct ghost_rq *ghost_rq); extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, struct ghost_enclave *new_e, @@ -2264,6 +2263,7 @@ void ghost_cpu_idle(void); unsigned long ghost_cfs_added_load(struct rq *rq); int64_t ghost_alloc_gtid(struct task_struct *p); +void init_ghost_rq(struct ghost_rq *ghost_rq); /* ghost functions in core.c */ void ghost_agent_schedule(void); From b954d051c4cfa7e7c91cb3bc962ae22c78adc9e5 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 18:16:38 -0800 Subject: [PATCH 096/165] sched: provide an abi-specific entrypoint for switchto Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I7aeb3c0cac145ca16895a36fb1cd61ae4fe5810b --- kernel/sched/ghost.c | 11 +++-------- kernel/sched/ghost_core.c | 20 ++++++++++++++++++++ kernel/sched/sched.h | 10 +++++++--- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 166d43b31c69..37e4100990c0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6208,15 +6208,9 @@ SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, #define SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK 0x10000 #endif -void ghost_switchto(struct rq *rq, struct task_struct *prev, - struct task_struct *next, int switchto_flags) +static void _ghost_switchto(struct rq *rq, struct task_struct *prev, + struct task_struct *next, int switchto_flags) { - lockdep_assert_held(&rq->lock); - VM_BUG_ON(prev != rq->curr); - VM_BUG_ON(prev->state == TASK_RUNNING); - VM_BUG_ON(next->state == TASK_RUNNING); - VM_BUG_ON(!ghost_class(prev->sched_class)); - VM_BUG_ON(!ghost_class(next->sched_class)); VM_BUG_ON(rq->ghost.check_prev_preemption); VM_BUG_ON(rq->ghost.switchto_count < 0); @@ -7496,6 +7490,7 @@ DEFINE_GHOST_ABI(current_abi) = { .pnt_prologue = pnt_prologue, .prepare_task_switch = prepare_task_switch, .tick = tick_handler, + .switchto = _ghost_switchto, .copy_process_epilogue = ghost_initialize_status_word, .cpu_idle = cpu_idle, .bpf_wake_agent = bpf_wake_agent, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index a9f8f8204093..3b4b13ad5351 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -701,6 +701,26 @@ void init_ghost_rq(struct ghost_rq *ghost_rq) INIT_LIST_HEAD(&ghost_rq->enclave_work); } +#ifdef CONFIG_SWITCHTO_API +void ghost_switchto(struct rq *rq, struct task_struct *prev, + struct task_struct *next, int switchto_flags) +{ + struct ghost_enclave *e; + + lockdep_assert_held(&rq->lock); + VM_BUG_ON(prev != rq->curr); + VM_BUG_ON(prev->state == TASK_RUNNING); + VM_BUG_ON(next->state == TASK_RUNNING); + VM_BUG_ON(!ghost_class(prev->sched_class)); + VM_BUG_ON(!ghost_class(next->sched_class)); + VM_BUG_ON(prev->ghost.enclave != next->ghost.enclave); + VM_BUG_ON(!next->ghost.enclave); + + e = next->ghost.enclave; + e->abi->switchto(rq, prev, next, switchto_flags); +} +#endif + #ifdef CONFIG_BPF #include diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 21a873a255c3..5975360ff0da 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -212,9 +212,6 @@ extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); -extern void ghost_switchto(struct rq *rq, struct task_struct *prev, - struct task_struct *next, int switchto_flags); - static inline void sched_ghost_entity_init(struct task_struct *p) { memset(&p->ghost, 0, sizeof(p->ghost)); @@ -2198,6 +2195,8 @@ struct ghost_abi { void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); void (*tick)(struct ghost_enclave *e, struct rq *rq); + void (*switchto)(struct rq *rq, struct task_struct *prev, + struct task_struct *next, int switchto_flags); void (*copy_process_epilogue)(struct task_struct *p); void (*cpu_idle)(struct rq *rq); int (*bpf_wake_agent)(int cpu); @@ -2265,6 +2264,11 @@ unsigned long ghost_cfs_added_load(struct rq *rq); int64_t ghost_alloc_gtid(struct task_struct *p); void init_ghost_rq(struct ghost_rq *ghost_rq); +#ifdef CONFIG_SWITCHTO_API +void ghost_switchto(struct rq *rq, struct task_struct *prev, + struct task_struct *next, int switchto_flags); +#endif + /* ghost functions in core.c */ void ghost_agent_schedule(void); #endif /* CONFIG_SCHED_CLASS_GHOST */ From fc089c01faddc918ee35daa6a441c99a1e6644d8 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 19:07:41 -0800 Subject: [PATCH 097/165] sched: abi-specific entrypoint for setscheduler Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: Ibc8cb4bd82e4597c32ee64f83ee787ecdc558d82 --- kernel/sched/ghost.c | 35 +++++++-------------------------- kernel/sched/ghost_core.c | 41 +++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 12 ++++++++---- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 37e4100990c0..4f2405ae0389 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -2904,37 +2904,15 @@ void ghost_sched_cleanup_fork(struct task_struct *p) * The most reasonable fix for all of this is to directly call * ghost_fdput_enclave() from __sched_setscheduler(). */ -int ghost_setscheduler(struct task_struct *p, struct rq *rq, - const struct sched_attr *attr, - struct ghost_enclave *new_e, - int *reset_on_fork) +static int _ghost_setscheduler(struct task_struct *p, struct rq *rq, + const struct sched_attr *attr, + struct ghost_enclave *new_e, + int *reset_on_fork) { - int oldpolicy = p->policy; - int newpolicy = attr->sched_policy; int ret; - if (WARN_ON_ONCE(!ghost_policy(oldpolicy) && !ghost_policy(newpolicy))) - return -EINVAL; - - /* - * If the process is dying, finish_task_switch will call task_dead - * *after* releasing the rq lock. We don't know if task_dead was called - * yet, and it will be called without holding any locks. This can break - * ghost for both task scheduling into ghost and out of ghost. - * - If we're entering ghost, but already ran task_dead from our old - * sched class, then we'll never run ghost_task_dead. - * - If we're leaving ghost, we need to run either ghost_task_dead xor - * setscheduler from ghost, but we have no nice way of knowing if we - * already ran ghost_task_dead. - */ - if (p->state == TASK_DEAD) - return -ESRCH; - /* Cannot change attributes for a ghost task after creation. */ - if (oldpolicy == newpolicy) - return -EPERM; - /* Task 'p' is departing the ghost sched class. */ - if (ghost_policy(oldpolicy)) { + if (ghost_policy(p->policy)) { /* * Don't allow an agent to move out of the ghost sched_class. * There is no use case for this and more importantly we don't @@ -2948,7 +2926,7 @@ int ghost_setscheduler(struct task_struct *p, struct rq *rq, return 0; } - if (!new_e) + if (WARN_ON_ONCE(!new_e)) return -EBADF; if (ghost_agent(attr)) { @@ -7486,6 +7464,7 @@ DEFINE_GHOST_ABI(current_abi) = { .enclave_release = enclave_release, .ctlfd_enclave_get = ctlfd_enclave_get, .ctlfd_enclave_put = ctlfd_enclave_put, + .setscheduler = _ghost_setscheduler, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .prepare_task_switch = prepare_task_switch, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 3b4b13ad5351..01e03c781fa7 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -545,6 +545,47 @@ bool ghost_agent(const struct sched_attr *attr) return attr->sched_priority == GHOST_SCHED_AGENT_PRIO; } +int ghost_setscheduler(struct task_struct *p, struct rq *rq, + const struct sched_attr *attr, + struct ghost_enclave *new_e, + int *reset_on_fork) +{ + struct ghost_enclave *e; + int oldpolicy = p->policy; + int newpolicy = attr->sched_policy; + + if (WARN_ON_ONCE(!ghost_policy(oldpolicy) && !ghost_policy(newpolicy))) + return -EINVAL; + + /* + * If the process is dying, finish_task_switch will call task_dead + * *after* releasing the rq lock. We don't know if task_dead was called + * yet, and it will be called without holding any locks. This can break + * ghost for both task scheduling into ghost and out of ghost. + * - If we're entering ghost, but already ran task_dead from our old + * sched class, then we'll never run ghost_task_dead. + * - If we're leaving ghost, we need to run either ghost_task_dead xor + * setscheduler from ghost, but we have no nice way of knowing if we + * already ran ghost_task_dead. + */ + if (p->state == TASK_DEAD) + return -ESRCH; + + /* Cannot change attributes for a ghost task after creation. */ + if (oldpolicy == newpolicy) + return -EPERM; + + if (ghost_policy(oldpolicy)) + e = p->ghost.enclave; + else + e = new_e; + + if (!e) + return -EBADF; + + return e->abi->setscheduler(p, rq, attr, new_e, reset_on_fork); +} + void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 5975360ff0da..b0d0fc1e5a56 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -205,10 +205,6 @@ struct ghost_enclave { }; /* In kernel/sched/ghost.c */ -extern int ghost_setscheduler(struct task_struct *p, struct rq *rq, - const struct sched_attr *attr, - struct ghost_enclave *new_e, - int *reset_on_fork); extern int ghost_sched_fork(struct task_struct *p); extern void ghost_sched_cleanup_fork(struct task_struct *p); @@ -2190,6 +2186,10 @@ struct ghost_abi { void (*enclave_release)(struct kref *k); struct ghost_enclave *(*ctlfd_enclave_get)(struct file *file); void (*ctlfd_enclave_put)(struct file *file); + int (*setscheduler)(struct task_struct *p, struct rq *rq, + const struct sched_attr *attr, + struct ghost_enclave *new_e, + int *reset_on_fork); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, @@ -2246,6 +2246,10 @@ void ghost_tick(struct rq *rq); */ struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); +int ghost_setscheduler(struct task_struct *p, struct rq *rq, + const struct sched_attr *attr, + struct ghost_enclave *new_e, + int *reset_on_fork); bool ghost_agent(const struct sched_attr *attr); int ghost_validate_sched_attr(const struct sched_attr *attr); From 0eaa8c0d2c1d29d91a694ab81440b28f543ae77c Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 23:21:54 -0800 Subject: [PATCH 098/165] sched: add abi-specific entrypoint for sched_fork() Tested: - all unit tests pass in virtme - fifo_agent; simple_exp Effort: sched/ghost/abi Change-Id: Ibb7891801681d250d1a74bb7fa0364f0f44bd4b4 --- kernel/sched/ghost.c | 50 ++++++++++++++++++--------------------- kernel/sched/ghost_core.c | 40 +++++++++++++++++++++++++++++++ kernel/sched/sched.h | 8 +++---- 3 files changed, 67 insertions(+), 31 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 4f2405ae0389..439c2020dd11 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -2817,42 +2817,22 @@ static int ghost_prep_agent(struct ghost_enclave *e, struct task_struct *p, return ret; } -int ghost_sched_fork(struct task_struct *p) +static int _ghost_sched_fork(struct ghost_enclave *e, struct task_struct *p) { - struct rq *rq; - struct rq_flags rf; - int ret; - struct ghost_enclave *parent_enclave; - - VM_BUG_ON(!task_has_ghost_policy(p)); - - /* - * Another task could be attempting to setsched current out of ghOSt. - * To keep current's enclave valid, we synchronize with the RQ lock. - */ - rq = task_rq_lock(current, &rf); - if (!ghost_policy(current->policy)) { - task_rq_unlock(rq, current, &rf); - /* It's not quite ECHILD, but it'll tell us where to look. */ - return -ECHILD; - } - parent_enclave = current->ghost.enclave; - VM_BUG_ON(!parent_enclave); - ret = ghost_prep_task(parent_enclave, p, true); - task_rq_unlock(rq, current, &rf); - - return ret; + return ghost_prep_task(e, p, true); } -void ghost_sched_cleanup_fork(struct task_struct *p) +static void _ghost_sched_cleanup_fork(struct ghost_enclave *e, + struct task_struct *p) { - struct ghost_queue *q; struct ghost_status_word *sw; - struct ghost_enclave *e = p->ghost.enclave; + struct ghost_queue *q; ulong flags; int error; VM_BUG_ON(!task_has_ghost_policy(p)); + VM_BUG_ON(e != p->ghost.enclave); + VM_BUG_ON(e == NULL); /* * Clear association between 'p' and the default queue. @@ -2861,6 +2841,15 @@ void ghost_sched_cleanup_fork(struct task_struct *p) if (q != NULL) { p->ghost.dst_q = NULL; queue_decref(q); + } else { + /* + * Although it is possible for a task to have a no 'dst_q' + * this is expected only for a task that setsched's into + * ghost. For the parent task to fork it must be running + * which means that the enclave was functional with the + * 'def_q' properly configured. + */ + WARN_ON_ONCE(true); } sw = p->ghost.status_word; @@ -2871,10 +2860,15 @@ void ghost_sched_cleanup_fork(struct task_struct *p) error = free_status_word_locked(e, sw); spin_unlock_irqrestore(&e->lock, flags); VM_BUG_ON(error); + } else { + WARN_ON_ONCE(true); /* not expected */ } + if (e) { kref_put(&e->kref, enclave_release); p->ghost.enclave = NULL; + } else { + WARN_ON_ONCE(true); /* not expected */ } } @@ -7465,6 +7459,8 @@ DEFINE_GHOST_ABI(current_abi) = { .ctlfd_enclave_get = ctlfd_enclave_get, .ctlfd_enclave_put = ctlfd_enclave_put, .setscheduler = _ghost_setscheduler, + .fork = _ghost_sched_fork, + .cleanup_fork = _ghost_sched_cleanup_fork, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, .prepare_task_switch = prepare_task_switch, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 01e03c781fa7..4da5d1351cb2 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -742,6 +742,46 @@ void init_ghost_rq(struct ghost_rq *ghost_rq) INIT_LIST_HEAD(&ghost_rq->enclave_work); } +int ghost_sched_fork(struct task_struct *p) +{ + int ret; + struct rq *rq; + struct rq_flags rf; + struct ghost_enclave *e; + + VM_BUG_ON(!task_has_ghost_policy(p)); + + /* + * Another task could be attempting to setsched current out of ghOSt. + * To keep current's enclave valid, we synchronize with the RQ lock. + */ + rq = task_rq_lock(current, &rf); + if (!ghost_policy(current->policy)) { + task_rq_unlock(rq, current, &rf); + /* It's not quite ECHILD, but it'll tell us where to look. */ + return -ECHILD; + } + e = current->ghost.enclave; + VM_BUG_ON(!e); + kref_get(&e->kref); /* get a local ref */ + task_rq_unlock(rq, current, &rf); + + ret = e->abi->fork(e, p); + kref_put(&e->kref, e->abi->enclave_release); /* release local ref */ + + return ret; +} + +void ghost_sched_cleanup_fork(struct task_struct *p) +{ + struct ghost_enclave *e = p->ghost.enclave; + + VM_BUG_ON(!task_has_ghost_policy(p)); + VM_BUG_ON(!e); + + e->abi->cleanup_fork(e, p); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b0d0fc1e5a56..fb42494f3e97 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -204,10 +204,6 @@ struct ghost_enclave { #endif }; -/* In kernel/sched/ghost.c */ -extern int ghost_sched_fork(struct task_struct *p); -extern void ghost_sched_cleanup_fork(struct task_struct *p); - static inline void sched_ghost_entity_init(struct task_struct *p) { memset(&p->ghost, 0, sizeof(p->ghost)); @@ -2190,6 +2186,8 @@ struct ghost_abi { const struct sched_attr *attr, struct ghost_enclave *new_e, int *reset_on_fork); + int (*fork)(struct ghost_enclave *e, struct task_struct *p); + void (*cleanup_fork)(struct ghost_enclave *e, struct task_struct *p); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, @@ -2250,6 +2248,8 @@ int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, struct ghost_enclave *new_e, int *reset_on_fork); +int ghost_sched_fork(struct task_struct *p); +void ghost_sched_cleanup_fork(struct task_struct *p); bool ghost_agent(const struct sched_attr *attr); int ghost_validate_sched_attr(const struct sched_attr *attr); From c95154cf7c4b41a6b878dcbe2da8ca04512a93ea Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 23:26:01 -0800 Subject: [PATCH 099/165] sched: move is_agent() into sched.h (abi independent) This change requires that all ABIs agree on the meaning of 'rq->ghost.agent' and 'p->ghost.agent' which seems like a safe bet. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I602a9a8c064f2abc76f1a83b387eed5b8e3da9b2 --- kernel/sched/ghost.c | 11 ----------- kernel/sched/sched.h | 11 ++++++++++- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 439c2020dd11..3cdad7a35bd2 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -385,17 +385,6 @@ static inline bool on_ghost_rq(struct rq *rq, struct task_struct *p) return !list_empty(&p->ghost.run_list); } -bool is_agent(struct rq *rq, struct task_struct *p) -{ - if (rq->ghost.agent == p) { - VM_BUG_ON(!p->ghost.agent); - return true; - } - - VM_BUG_ON(p->ghost.agent); - return false; -} - static inline bool ghost_can_schedule(struct rq *rq, gtid_t gtid) { const struct sched_class *class = rq->curr->sched_class; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index fb42494f3e97..f01bcc1dfc29 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2095,7 +2095,16 @@ static inline bool ghost_class(const struct sched_class *class) return class == &ghost_sched_class; } -bool is_agent(struct rq *rq, struct task_struct *p); +static inline bool is_agent(struct rq *rq, struct task_struct *p) +{ + if (rq->ghost.agent == p) { + VM_BUG_ON(!p->ghost.agent); + return true; + } + + VM_BUG_ON(p->ghost.agent); + return false; +} /* * Contents of rq->ghost.rendezvous field: From d794b83345269c164859b24267cbb2dc7d9422a6 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 23:30:17 -0800 Subject: [PATCH 100/165] sched: make some symbols in ghost.c static These functions were in a global scope unnecessarily so make them static. Tested: builds and also 'nm --defined-only -g ghost.o' Effort: sched/ghost/abi Change-Id: I3c754edfa59bf28276452ca8a46b7885ba77d140 --- kernel/sched/ghost.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 3cdad7a35bd2..3af06c3e89e7 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1652,7 +1652,7 @@ static struct rq *ghost_move_task(struct rq *rq, struct task_struct *next, return rq; } -int _ghost_mmap_common(struct vm_area_struct *vma, ulong mapsize) +static int _ghost_mmap_common(struct vm_area_struct *vma, ulong mapsize) { static const struct vm_operations_struct ghost_vm_ops = {}; @@ -1791,7 +1791,7 @@ static void __queue_free_work(struct work_struct *work) kfree(q); } -void _queue_free_rcu_callback(struct rcu_head *rhp) +static void _queue_free_rcu_callback(struct rcu_head *rhp) { struct ghost_queue *q = container_of(rhp, struct ghost_queue, rcu); @@ -2491,8 +2491,8 @@ static int __ghost_prep_task(struct ghost_enclave *e, struct task_struct *p, * task requires enclave to be locked whereas ungating msgs requires * that no locks are held). */ -struct task_struct *__enclave_remove_inhibited_task(struct ghost_enclave *e, - gtid_t gtid) +static struct task_struct * +__enclave_remove_inhibited_task(struct ghost_enclave *e, gtid_t gtid) { struct task_struct *p; From be4ba1104220aecee0823627f361bb66a9f2db68 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 15 Nov 2021 23:55:06 -0800 Subject: [PATCH 101/165] sched: abi-specific entrypoint to commit greedy transactions. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I6b7ff6c4779c2c1d75219bf02dbced3b593a74a8 --- kernel/sched/ghost.c | 9 +++++++-- kernel/sched/ghost_core.c | 15 +++++++++++++++ kernel/sched/sched.h | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 3af06c3e89e7..89299efb0798 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -5582,9 +5582,13 @@ static void ghost_commit_pending_txn(int where) put_cpu(); } -extern void ghost_commit_greedy_txn(void) +static void _commit_greedy_txn(int cpu) { - ghost_commit_pending_txn(0); /* Commit a greedy pending txn */ + WARN_ON_ONCE(preemptible()); + + rcu_read_lock(); + _ghost_commit_pending_txn(cpu, /*where=*/0); + rcu_read_unlock(); } void ghost_commit_all_greedy_txns(void) @@ -7455,6 +7459,7 @@ DEFINE_GHOST_ABI(current_abi) = { .prepare_task_switch = prepare_task_switch, .tick = tick_handler, .switchto = _ghost_switchto, + .commit_greedy_txn = _commit_greedy_txn, .copy_process_epilogue = ghost_initialize_status_word, .cpu_idle = cpu_idle, .bpf_wake_agent = bpf_wake_agent, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 4da5d1351cb2..90e93e3c1276 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -782,6 +782,21 @@ void ghost_sched_cleanup_fork(struct task_struct *p) e->abi->cleanup_fork(e, p); } +void ghost_commit_greedy_txn(void) +{ + struct ghost_enclave *e; + int cpu = get_cpu(); + + VM_BUG_ON(preemptible()); + + /* Implicit read-side critical section due to disabled preemption */ + e = rcu_dereference_sched(per_cpu(enclave, cpu)); + if (e) + e->abi->commit_greedy_txn(cpu); + + put_cpu(); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f01bcc1dfc29..b3bebc4e0817 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2204,6 +2204,7 @@ struct ghost_abi { void (*tick)(struct ghost_enclave *e, struct rq *rq); void (*switchto)(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); + void (*commit_greedy_txn)(int cpu); void (*copy_process_epilogue)(struct task_struct *p); void (*cpu_idle)(struct rq *rq); int (*bpf_wake_agent)(int cpu); From 24ecacbdaa6081b314a56f91a2a214d56aa56dae Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 09:56:36 -0800 Subject: [PATCH 102/165] sched: get rid of ghost uapi dependency in timerfd.c Prior to this change ghost and timerfd were sharing the definition of 'struct timerfd_ghost' via the ghost uapi header. This immediately breaks if a newer ABI decides to change the layout or meaning of 'struct timerfd_ghost'. Fix this by providing a stable 'struct __kernel_timerfd_ghost' in . At the moment this struct mirrors the information in 'struct timerfd_ghost'. Also add a 'timerfd_triggered' callback in the 'ghost_abi' that is invoked when a timerfd is triggered on a specific cpu. TODO: file a bug to ensure that timerfd doesn't trigger on a cpu that no longer belongs to the enclave. Tested: all unit tests pass in virtme. Effort: sched/ghost/abi Change-Id: I366b6a3d158d692b10ddc16dda5c923b7d64d5aa --- fs/timerfd.c | 30 ++++++++++-------------------- include/linux/sched.h | 9 +++++++-- kernel/sched/ghost.c | 17 +++++++++++------ kernel/sched/ghost_core.c | 27 +++++++++++++++++++++++++++ kernel/sched/sched.h | 1 + 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/fs/timerfd.c b/fs/timerfd.c index 33339793b414..657255a1117c 100644 --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -28,10 +28,6 @@ #include #include -#ifdef CONFIG_SCHED_CLASS_GHOST -#include -#endif - struct timerfd_ctx { union { struct hrtimer tmr; @@ -49,7 +45,7 @@ struct timerfd_ctx { spinlock_t cancel_lock; bool might_cancel; #ifdef CONFIG_SCHED_CLASS_GHOST - struct timerfd_ghost timerfd_ghost; + struct __kernel_timerfd_ghost timerfd_ghost; #endif }; @@ -62,13 +58,6 @@ static inline bool isalarm(struct timerfd_ctx *ctx) ctx->clockid == CLOCK_BOOTTIME_ALARM; } -#ifdef CONFIG_SCHED_CLASS_GHOST -static inline bool timerfd_ghost_enabled(struct timerfd_ghost *timerfd_ghost) -{ - return timerfd_ghost->flags & TIMERFD_GHOST_ENABLED; -} -#endif - /* * This gets called when the timer event triggers. We set the "expired" * flag, but we do not re-arm the timer (in case it's necessary, @@ -79,8 +68,8 @@ static void timerfd_triggered(struct timerfd_ctx *ctx) unsigned long flags; #ifdef CONFIG_SCHED_CLASS_GHOST - struct timerfd_ghost timerfd_ghost = { - .flags = 0, /* disabled */ + struct __kernel_timerfd_ghost timerfd_ghost = { + .enabled = false, }; #endif @@ -89,13 +78,13 @@ static void timerfd_triggered(struct timerfd_ctx *ctx) ctx->ticks++; wake_up_locked_poll(&ctx->wqh, EPOLLIN); #ifdef CONFIG_SCHED_CLASS_GHOST - if (unlikely(timerfd_ghost_enabled(&ctx->timerfd_ghost))) + if (unlikely(&ctx->timerfd_ghost.enabled)) timerfd_ghost = ctx->timerfd_ghost; #endif spin_unlock_irqrestore(&ctx->wqh.lock, flags); #ifdef CONFIG_SCHED_CLASS_GHOST - if (unlikely(timerfd_ghost_enabled(&timerfd_ghost))) + if (unlikely(timerfd_ghost.enabled)) ghost_timerfd_triggered(&timerfd_ghost); #endif } @@ -466,7 +455,8 @@ SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags) #ifdef CONFIG_SCHED_CLASS_GHOST int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, - struct itimerspec64 *old, struct timerfd_ghost *tfdl) + struct itimerspec64 *old, + struct __kernel_timerfd_ghost *ktfd) #else static int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, @@ -536,10 +526,10 @@ static int do_timerfd_settime(int ufd, int flags, */ ret = timerfd_setup(ctx, flags, new); #ifdef CONFIG_SCHED_CLASS_GHOST - if (tfdl) - memcpy(&ctx->timerfd_ghost, tfdl, sizeof(struct timerfd_ghost)); + if (ktfd) + memcpy(&ctx->timerfd_ghost, ktfd, sizeof(*ktfd)); else - ctx->timerfd_ghost.flags = 0; /* disabled */ + ctx->timerfd_ghost.enabled = false; #endif spin_unlock_irq(&ctx->wqh.lock); fdput(f); diff --git a/include/linux/sched.h b/include/linux/sched.h index 40296ba2a13d..0764aac288bc 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -588,7 +588,6 @@ struct sched_dl_entity { struct ghost_queue; struct ghost_status_word; struct ghost_enclave; -struct timerfd_ghost; struct sched_ghost_entity { struct list_head run_list; @@ -643,8 +642,14 @@ struct sched_ghost_entity { struct rcu_head rcu; }; +struct __kernel_timerfd_ghost { + bool enabled; + int cpu; + uint64_t cookie; +}; + extern void ghost_commit_greedy_txn(void); -extern void ghost_timerfd_triggered(struct timerfd_ghost *timer); +extern void ghost_timerfd_triggered(struct __kernel_timerfd_ghost *timer); #endif diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 89299efb0798..07e9f25bc11c 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6037,7 +6037,8 @@ static int ghost_timerfd_validate(struct timerfd_ghost *timerfd_ghost) /* fs/timerfd.c */ int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new, - struct itimerspec64 *old, struct timerfd_ghost *tfdl); + struct itimerspec64 *old, + struct __kernel_timerfd_ghost *ktfd); static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) { @@ -6049,6 +6050,7 @@ static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) struct __kernel_itimerspec *otmr; struct timerfd_ghost timerfd_ghost; struct ghost_ioc_timerfd_settime settime_data; + struct __kernel_timerfd_ghost ktfd_ghost; if (copy_from_user(&settime_data, arg, sizeof(struct ghost_ioc_timerfd_settime))) @@ -6067,7 +6069,11 @@ static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) if (ret) return ret; - ret = do_timerfd_settime(timerfd, flags, &new, &old, &timerfd_ghost); + ktfd_ghost.enabled = timerfd_ghost.flags & TIMERFD_GHOST_ENABLED; + ktfd_ghost.cpu = timerfd_ghost.cpu; + ktfd_ghost.cookie = timerfd_ghost.cookie; + + ret = do_timerfd_settime(timerfd, flags, &new, &old, &ktfd_ghost); if (ret) return ret; @@ -6077,12 +6083,10 @@ static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) return ret; } -/* Called from timerfd_triggered() on timer expiry */ -void ghost_timerfd_triggered(struct timerfd_ghost *timerfd_ghost) +static void _ghost_timerfd_triggered(int cpu, uint64_t cookie) { struct rq *rq; struct rq_flags rf; - int cpu = timerfd_ghost->cpu; if (WARN_ON_ONCE(cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu))) return; @@ -6090,7 +6094,7 @@ void ghost_timerfd_triggered(struct timerfd_ghost *timerfd_ghost) rq = cpu_rq(cpu); rq_lock_irqsave(rq, &rf); - if (cpu_deliver_timer_expired(rq, timerfd_ghost->cookie)) + if (cpu_deliver_timer_expired(rq, cookie)) ghost_wake_agent_on(agent_target_cpu(rq)); rq_unlock_irqrestore(rq, &rf); @@ -7462,6 +7466,7 @@ DEFINE_GHOST_ABI(current_abi) = { .commit_greedy_txn = _commit_greedy_txn, .copy_process_epilogue = ghost_initialize_status_word, .cpu_idle = cpu_idle, + .timerfd_triggered = _ghost_timerfd_triggered, .bpf_wake_agent = bpf_wake_agent, .bpf_run_gtid = bpf_run_gtid, .ghost_msg_is_valid_access = ghost_msg_is_valid_access, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 90e93e3c1276..b4f0a27d5353 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -797,6 +797,33 @@ void ghost_commit_greedy_txn(void) put_cpu(); } +/* Called from timerfd_triggered() on timer expiry */ +void ghost_timerfd_triggered(struct __kernel_timerfd_ghost *ktfd) +{ + int cpu = ktfd->cpu; + struct ghost_enclave *e; + + if (WARN_ON_ONCE(cpu < 0 || cpu >= nr_cpu_ids || !cpu_online(cpu))) + return; + + if (WARN_ON_ONCE(!ktfd->enabled)) + return; + + /* + * XXX the enclave lookup doesn't handle cpus moving out of enclaves + * properly (for e.g. a timerfd might be triggered on a cpu that is + * not associated with the enclave). + */ + rcu_read_lock(); + e = rcu_dereference(per_cpu(enclave, cpu)); + if (e) + e->abi->timerfd_triggered(cpu, ktfd->cookie); + else + WARN_ONCE(true, "cpu %d has no enclave", cpu); + rcu_read_unlock(); +} + + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b3bebc4e0817..536a97ef11e4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2207,6 +2207,7 @@ struct ghost_abi { void (*commit_greedy_txn)(int cpu); void (*copy_process_epilogue)(struct task_struct *p); void (*cpu_idle)(struct rq *rq); + void (*timerfd_triggered)(int cpu, uint64_t cookie); int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); bool (*ghost_msg_is_valid_access)(int off, int size, From 3dbe12ca138011b7a1228b119539eba89fb3afe7 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 10:31:59 -0800 Subject: [PATCH 103/165] sched: add abi-specific callback from pick_next_ghost_agent Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I1fb1f778edaf8a890e75939ec4259289da4c5c1f --- kernel/sched/ghost.c | 20 +------------------- kernel/sched/ghost_core.c | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 1 + 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 07e9f25bc11c..b25af82fdc6c 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1594,25 +1594,6 @@ DEFINE_SCHED_CLASS(ghost) = { #endif }; -static int balance_agent(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf) -{ - return 0; -} - -/* - * An interstitial sched class that operates below the stop_sched_class. It - * enables returning to an agent at the highest priority when the agent is about - * to lose its CPU to another sched class. This allows the agent to transition - * to another CPU before giving up its CPU. - */ -DEFINE_SCHED_CLASS(ghost_agent) = { - .pick_next_task = pick_next_ghost_agent, -#ifdef CONFIG_SMP - .balance = balance_agent, -#endif -}; - /* * Migrate 'next' (if necessary) in preparation to run it on 'cpu'. * @@ -7460,6 +7441,7 @@ DEFINE_GHOST_ABI(current_abi) = { .cleanup_fork = _ghost_sched_cleanup_fork, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, + .pick_next_ghost_agent = pick_next_ghost_agent, .prepare_task_switch = prepare_task_switch, .tick = tick_handler, .switchto = _ghost_switchto, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index b4f0a27d5353..e348f18bb7b9 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -823,6 +823,42 @@ void ghost_timerfd_triggered(struct __kernel_timerfd_ghost *ktfd) rcu_read_unlock(); } +static struct task_struct *pick_next_ghost_agent(struct rq *rq) +{ + struct task_struct *next; + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + + /* Implicit read-side critical section due to disabled preemption */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + if (e) + next = e->abi->pick_next_ghost_agent(rq); + else + next = NULL; + + return next; +} + +static int balance_agent(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) +{ + return 0; +} + +/* + * An interstitial sched class that operates below the stop_sched_class. It + * enables returning to an agent at the highest priority when the agent is about + * to lose its CPU to another sched class. This allows the agent to transition + * to another CPU before giving up its CPU. + */ +DEFINE_SCHED_CLASS(ghost_agent) = { + .pick_next_task = pick_next_ghost_agent, +#ifdef CONFIG_SMP + .balance = balance_agent, +#endif +}; #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 536a97ef11e4..b75b03a8adc0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2199,6 +2199,7 @@ struct ghost_abi { void (*cleanup_fork)(struct ghost_enclave *e, struct task_struct *p); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); + struct task_struct *(*pick_next_ghost_agent)(struct rq *rq); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); void (*tick)(struct ghost_enclave *e, struct rq *rq); From e4fcc3b51bd1f7413d0a8dde259f104986919a79 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 12:14:35 -0800 Subject: [PATCH 104/165] sched: s/update_curr_ghost/_update_curr_ghost No functional change. Tested: builds Effort: sched/ghost/abi Change-Id: I3f26ab5e7c01a43e7e1d7c8f6cda3a980a8d68fa --- kernel/sched/ghost.c | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b25af82fdc6c..0564c696c9c7 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -580,7 +580,7 @@ static void __update_curr_ghost(struct rq *rq, bool update_sw) WRITE_ONCE(sw->runtime, curr->se.sum_exec_runtime); } -static void update_curr_ghost(struct rq *rq) +static void _update_curr_ghost(struct rq *rq) { __update_curr_ghost(rq, true); } @@ -681,16 +681,16 @@ static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) /* * A task is accumulating cputime only when it is oncpu. Thus it is - * useless to call update_curr_ghost for a task that is 'on_rq' but + * useless to call _update_curr_ghost for a task that is 'on_rq' but * is not running (in this case we'll just update the cputime of * whatever task happens to be oncpu). * * Ordinarily we wouldn't care but we routinely dequeue_task_ghost() * when migrating a task via ghost_move_task() during txn commit so - * we call update_curr_ghost() only if 'p' is actually running. + * we call _update_curr_ghost() only if 'p' is actually running. */ if (task_current(rq, p)) - update_curr_ghost(rq); + _update_curr_ghost(rq); BUG_ON(rq->ghost.ghost_nr_running <= 0); rq->ghost.ghost_nr_running--; @@ -735,7 +735,7 @@ static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) static void put_prev_task_ghost(struct rq *rq, struct task_struct *p) { - update_curr_ghost(rq); + _update_curr_ghost(rq); } static void @@ -1573,7 +1573,7 @@ static void task_woken_ghost(struct rq *rq, struct task_struct *p) } DEFINE_SCHED_CLASS(ghost) = { - .update_curr = update_curr_ghost, + .update_curr = _update_curr_ghost, .prio_changed = prio_changed_ghost, .switched_to = switched_to_ghost, .switched_from = switched_from_ghost, @@ -4524,15 +4524,15 @@ static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, lockdep_assert_held(&rq->lock); VM_BUG_ON(task_rq(p) != rq); - /* - * When TASK_PREEMPTED is produced before returning from pick_next_task - * (e.g. via pick_next_ghost_agent) we don't have an up-to-date runtime - * since put_prev_task() hasn't been called yet. - * - * Therefore if 'p == rq->curr' we must do update_curr_ghost() by hand. - */ - if (p == rq->curr) - update_curr_ghost(rq); + /* + * When TASK_PREEMPTED is produced before returning from pick_next_task + * (e.g. via pick_next_ghost_agent) we don't have an up-to-date runtime + * since put_prev_task() hasn't been called yet. + * + * Therefore if 'p == rq->curr' we must do _update_curr_ghost() by hand. + */ + if (p == rq->curr) + _update_curr_ghost(rq); /* * If a latched task was preempted then by definition it was not @@ -4587,9 +4587,9 @@ static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable) enclave_add_task(p->ghost.enclave, p); - /* See explanation in ghost_task_preempted() */ - if (p == rq->curr) - update_curr_ghost(rq); + /* See explanation in ghost_task_preempted() */ + if (p == rq->curr) + _update_curr_ghost(rq); task_deliver_msg_task_new(rq, p, runnable); } @@ -4604,9 +4604,9 @@ static void ghost_task_yield(struct rq *rq, struct task_struct *p) lockdep_assert_held(&rq->lock); VM_BUG_ON(task_rq(p) != rq); - /* See explanation in ghost_task_preempted() */ - if (p == rq->curr) - update_curr_ghost(rq); + /* See explanation in ghost_task_preempted() */ + if (p == rq->curr) + update_curr_ghost(rq); task_deliver_msg_yield(rq, p); } @@ -6167,7 +6167,7 @@ static void _ghost_switchto(struct rq *rq, struct task_struct *prev, if (switchto_flags & SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK) { next->se.exec_start = prev->se.exec_start; } else { - update_curr_ghost(rq); + _update_curr_ghost(rq); next->se.exec_start = rq_clock_task(rq); } From aad7d156aa67d92cf5b073f34000ff81dc495092 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 12:23:26 -0800 Subject: [PATCH 105/165] sched: add abi-specific callback for sched_class.update_curr There some contortions here: 'ghost_sched_class' is still defined in ghost.c but it points at 'update_curr_ghost()' in ghost_core.c that in turn calls back into ghost.c via 'ghost_abi.update_curr' All of this helps to move the sched_class callbacks incrementally and make it easier to test and review. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I3a9d11b28cdcfc9141a08240f260e1f3f13b55bd --- kernel/sched/ghost.c | 9 +++++++-- kernel/sched/ghost_core.c | 30 ++++++++++++++++++++++++++++++ kernel/sched/sched.h | 10 +++++++++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 0564c696c9c7..0009dc80ed3c 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1573,7 +1573,7 @@ static void task_woken_ghost(struct rq *rq, struct task_struct *p) } DEFINE_SCHED_CLASS(ghost) = { - .update_curr = _update_curr_ghost, + .update_curr = update_curr_ghost, /* ghost_core.c */ .prio_changed = prio_changed_ghost, .switched_to = switched_to_ghost, .switched_from = switched_from_ghost, @@ -7441,7 +7441,6 @@ DEFINE_GHOST_ABI(current_abi) = { .cleanup_fork = _ghost_sched_cleanup_fork, .wait_for_rendezvous = wait_for_rendezvous, .pnt_prologue = pnt_prologue, - .pick_next_ghost_agent = pick_next_ghost_agent, .prepare_task_switch = prepare_task_switch, .tick = tick_handler, .switchto = _ghost_switchto, @@ -7454,5 +7453,11 @@ DEFINE_GHOST_ABI(current_abi) = { .ghost_msg_is_valid_access = ghost_msg_is_valid_access, .bpf_link_attach = bpf_link_attach, .bpf_link_detach = bpf_link_detach, + + /* ghost_agent_sched_class callbacks */ + .pick_next_ghost_agent = pick_next_ghost_agent, + + /* ghost_sched_class callbacks */ + .update_curr = _update_curr_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index e348f18bb7b9..c1b79ebbe3dd 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -860,6 +860,36 @@ DEFINE_SCHED_CLASS(ghost_agent) = { #endif }; +void update_curr_ghost(struct rq *rq) +{ + struct ghost_enclave *rq_enclave, *curr_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + if (ghost_class(rq->curr->sched_class)) + curr_enclave = rq->curr->ghost.enclave; + else + curr_enclave = NULL; + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && curr_enclave && rq_enclave != curr_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : curr_enclave; + if (e) + e->abi->update_curr(rq); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index b75b03a8adc0..f25e3f3a9f33 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2199,7 +2199,6 @@ struct ghost_abi { void (*cleanup_fork)(struct ghost_enclave *e, struct task_struct *p); void (*wait_for_rendezvous)(struct rq *rq); void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); - struct task_struct *(*pick_next_ghost_agent)(struct rq *rq); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); void (*tick)(struct ghost_enclave *e, struct rq *rq); @@ -2219,8 +2218,17 @@ struct ghost_abi { int prog_type, int attach_type); void (*bpf_link_detach)(struct ghost_enclave *e, struct bpf_prog *prog, int prog_type, int attach_type); + + /* ghost_agent_sched_class callbacks */ + struct task_struct *(*pick_next_ghost_agent)(struct rq *rq); + + /* ghost_sched_class callbacks */ + void (*update_curr)(struct rq *rq); }; +/* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ +void update_curr_ghost(struct rq *rq); + #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ __aligned(__alignof__(struct ghost_abi)) \ From 94448e46e022e4839113cd95e13d1986a3a65f35 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 12:43:02 -0800 Subject: [PATCH 106/165] sched: add abi-specific callback for sched_class.prio_changed Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I12e8b544a8b646059d3f381b408b40175f224064 --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 14 ++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 0009dc80ed3c..cc6d2107a260 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -585,7 +585,7 @@ static void _update_curr_ghost(struct rq *rq) __update_curr_ghost(rq, true); } -static void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) +static void _prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) { /* * XXX produce MSG_TASK_PRIO_CHANGE into p->ghost.dst_q. @@ -1574,7 +1574,7 @@ static void task_woken_ghost(struct rq *rq, struct task_struct *p) DEFINE_SCHED_CLASS(ghost) = { .update_curr = update_curr_ghost, /* ghost_core.c */ - .prio_changed = prio_changed_ghost, + .prio_changed = prio_changed_ghost, /* ghost_core.c */ .switched_to = switched_to_ghost, .switched_from = switched_from_ghost, .task_dead = task_dead_ghost, @@ -7459,5 +7459,6 @@ DEFINE_GHOST_ABI(current_abi) = { /* ghost_sched_class callbacks */ .update_curr = _update_curr_ghost, + .prio_changed = _prio_changed_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index c1b79ebbe3dd..dda307b57075 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -890,6 +890,20 @@ void update_curr_ghost(struct rq *rq) e->abi->update_curr(rq); } +void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) +{ + struct ghost_enclave *e; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_held(&rq->lock); + + e = p->ghost.enclave; + if (e) + e->abi->prio_changed(rq, p, old); + else + WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f25e3f3a9f33..1c26a39687f9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2224,10 +2224,12 @@ struct ghost_abi { /* ghost_sched_class callbacks */ void (*update_curr)(struct rq *rq); + void (*prio_changed)(struct rq *rq, struct task_struct *p, int old); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ void update_curr_ghost(struct rq *rq); +void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From a4080f828b330e035ad29d5c20b4c0323cfb0b6c Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 12:53:07 -0800 Subject: [PATCH 107/165] sched: add abi-specific callback for sched_class.switched_to Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I34c92adf1ade9868efc81d06bdf88e65d7a534c3 --- kernel/sched/ghost.c | 7 ++++--- kernel/sched/ghost_core.c | 14 ++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index cc6d2107a260..ec157588ed63 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -599,7 +599,7 @@ static void _prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) */ } -static void switched_to_ghost(struct rq *rq, struct task_struct *p) +static void _switched_to_ghost(struct rq *rq, struct task_struct *p) { struct ghost_status_word *status_word = p->ghost.status_word; @@ -1575,7 +1575,7 @@ static void task_woken_ghost(struct rq *rq, struct task_struct *p) DEFINE_SCHED_CLASS(ghost) = { .update_curr = update_curr_ghost, /* ghost_core.c */ .prio_changed = prio_changed_ghost, /* ghost_core.c */ - .switched_to = switched_to_ghost, + .switched_to = switched_to_ghost, /* ghost_core.c */ .switched_from = switched_from_ghost, .task_dead = task_dead_ghost, .dequeue_task = dequeue_task_ghost, @@ -7162,7 +7162,7 @@ static void prepare_task_switch(struct rq *rq, struct task_struct *prev, * sched_setscheduler(2)) while the rq->lock is dropped in one of the * pick_next_task handlers (e.g. during CFS idle load balancing). * - * It is not possible to distinguish this in switched_to_ghost() so + * It is not possible to distinguish this in _switched_to_ghost() so * we resolve 'prev->ghost.new_task' here. Failing to do this has * dire consequences if 'prev' is runnable since it will languish * in the kernel forever (in contrast to a blocked task there is @@ -7460,5 +7460,6 @@ DEFINE_GHOST_ABI(current_abi) = { /* ghost_sched_class callbacks */ .update_curr = _update_curr_ghost, .prio_changed = _prio_changed_ghost, + .switched_to = _switched_to_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index dda307b57075..b9bb4b785cf0 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -904,6 +904,20 @@ void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } +void switched_to_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_enclave *e; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_held(&rq->lock); + + e = p->ghost.enclave; + if (e) + e->abi->switched_to(rq, p); + else + WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1c26a39687f9..cae609bc1c06 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2225,11 +2225,13 @@ struct ghost_abi { /* ghost_sched_class callbacks */ void (*update_curr)(struct rq *rq); void (*prio_changed)(struct rq *rq, struct task_struct *p, int old); + void (*switched_to)(struct rq *rq, struct task_struct *p); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ void update_curr_ghost(struct rq *rq); void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old); +void switched_to_ghost(struct rq *rq, struct task_struct *p); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From dca02896d32b44a7c1770ae585b36e41ffea4d48 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 13:34:50 -0800 Subject: [PATCH 108/165] sched: add abi-specific callback for sched_class.switched_from Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I1622f569d5fbe849b1dfd08512079604c7af6089 --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 14 ++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index ec157588ed63..0a8065762b4f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -632,7 +632,7 @@ static void _switched_to_ghost(struct rq *rq, struct task_struct *p) } } -static void switched_from_ghost(struct rq *rq, struct task_struct *p) +static void _switched_from_ghost(struct rq *rq, struct task_struct *p) { /* * A running task can be switched into ghost while it is executing @@ -1576,7 +1576,7 @@ DEFINE_SCHED_CLASS(ghost) = { .update_curr = update_curr_ghost, /* ghost_core.c */ .prio_changed = prio_changed_ghost, /* ghost_core.c */ .switched_to = switched_to_ghost, /* ghost_core.c */ - .switched_from = switched_from_ghost, + .switched_from = switched_from_ghost, /* ghost_core.c */ .task_dead = task_dead_ghost, .dequeue_task = dequeue_task_ghost, .put_prev_task = put_prev_task_ghost, @@ -7461,5 +7461,6 @@ DEFINE_GHOST_ABI(current_abi) = { .update_curr = _update_curr_ghost, .prio_changed = _prio_changed_ghost, .switched_to = _switched_to_ghost, + .switched_from = _switched_from_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index b9bb4b785cf0..7b27b388f900 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -918,6 +918,20 @@ void switched_to_ghost(struct rq *rq, struct task_struct *p) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } +void switched_from_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_enclave *e; + + lockdep_assert_held(&p->pi_lock); + lockdep_assert_held(&rq->lock); + + e = p->ghost.enclave; + if (e) + e->abi->switched_from(rq, p); + else + WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index cae609bc1c06..bb78e9328482 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2226,12 +2226,14 @@ struct ghost_abi { void (*update_curr)(struct rq *rq); void (*prio_changed)(struct rq *rq, struct task_struct *p, int old); void (*switched_to)(struct rq *rq, struct task_struct *p); + void (*switched_from)(struct rq *rq, struct task_struct *p); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ void update_curr_ghost(struct rq *rq); void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old); void switched_to_ghost(struct rq *rq, struct task_struct *p); +void switched_from_ghost(struct rq *rq, struct task_struct *p); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From b56f869a561b679d993a70f8045095bf37e73b4b Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 16:04:23 -0800 Subject: [PATCH 109/165] sched: add abi-specific callback for sched_class.task_dead Tested: all unit tests passing in virtme. Effort: sched/ghost/abi Change-Id: I1e0e5b03e8ed44291825afc3d57e78e1b3622f7d --- kernel/sched/ghost.c | 8 ++++---- kernel/sched/ghost_core.c | 20 ++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 0a8065762b4f..36d9e2ade1d4 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -53,7 +53,6 @@ static void _ghost_task_new(struct rq *rq, struct task_struct *p, bool runnable); static void ghost_task_yield(struct rq *rq, struct task_struct *p); static void ghost_task_blocked(struct rq *rq, struct task_struct *p); -static void task_dead_ghost(struct task_struct *p); static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p, bool runnable); static void task_deliver_msg_affinity_changed(struct rq *rq, @@ -1577,7 +1576,7 @@ DEFINE_SCHED_CLASS(ghost) = { .prio_changed = prio_changed_ghost, /* ghost_core.c */ .switched_to = switched_to_ghost, /* ghost_core.c */ .switched_from = switched_from_ghost, /* ghost_core.c */ - .task_dead = task_dead_ghost, + .task_dead = task_dead_ghost, /* ghost_core.c */ .dequeue_task = dequeue_task_ghost, .put_prev_task = put_prev_task_ghost, .enqueue_task = enqueue_task_ghost, @@ -2881,7 +2880,7 @@ static int _ghost_setscheduler(struct task_struct *p, struct rq *rq, * Don't allow an agent to move out of the ghost sched_class. * There is no use case for this and more importantly we don't * apply the side-effects of agent leaving the CPU (compare to - * task_dead_ghost() where we do). + * _task_dead_ghost() where we do). */ if (p->ghost.agent) return -EPERM; @@ -4351,7 +4350,7 @@ static void ghost_delayed_put_task_struct(struct rcu_head *rhp) put_task_struct(tsk); } -static void task_dead_ghost(struct task_struct *p) +static void _task_dead_ghost(struct task_struct *p) { struct rq_flags rf; struct rq *rq; @@ -7462,5 +7461,6 @@ DEFINE_GHOST_ABI(current_abi) = { .prio_changed = _prio_changed_ghost, .switched_to = _switched_to_ghost, .switched_from = _switched_from_ghost, + .task_dead = _task_dead_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 7b27b388f900..d1735ebe85b6 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -932,6 +932,26 @@ void switched_from_ghost(struct rq *rq, struct task_struct *p) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } +void task_dead_ghost(struct task_struct *p) +{ + struct ghost_enclave *e; + + /* + * This looks risky since neither 'p->pi_lock' nor 'rq->lock' is held + * at this point but is safe because 'p' is no longer reachable from + * userspace at this point (it lost identity much earlier in do_exit). + * + * The only thing that can mutate 'p->ghost.enclave' is moving out of + * ghost but that cannot happen for the reason stated above. + */ + e = p->ghost.enclave; + + if (e) + e->abi->task_dead(p); + else + WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bb78e9328482..52396dc98e59 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2227,6 +2227,7 @@ struct ghost_abi { void (*prio_changed)(struct rq *rq, struct task_struct *p, int old); void (*switched_to)(struct rq *rq, struct task_struct *p); void (*switched_from)(struct rq *rq, struct task_struct *p); + void (*task_dead)(struct task_struct *p); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2234,6 +2235,7 @@ void update_curr_ghost(struct rq *rq); void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old); void switched_to_ghost(struct rq *rq, struct task_struct *p); void switched_from_ghost(struct rq *rq, struct task_struct *p); +void task_dead_ghost(struct task_struct *p); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From e36234dc164850d2ab288748416c1f93e2952ee1 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 16:22:55 -0800 Subject: [PATCH 110/165] sched: add abi-specific callback for sched_class.dequeue_task Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I6f4a34dbac7d3a6463fb19c3ae552e7df8d84f22 --- kernel/sched/ghost.c | 11 ++++++----- kernel/sched/ghost_core.c | 32 ++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 36d9e2ade1d4..2d8ccdba9683 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -557,7 +557,7 @@ static void __update_curr_ghost(struct rq *rq, bool update_sw) /* * Bail if this due to a "spurious" dequeue. * - * For e.g. dequeue_task_ghost() is called when scheduling properties + * For e.g. _dequeue_task_ghost() is called when scheduling properties * of a runnable ghost task change (e.g. nice or cpu affinity), but * if that task is not oncpu then nothing needs to be done here. */ @@ -672,7 +672,7 @@ static void _switched_from_ghost(struct rq *rq, struct task_struct *p) } } -static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +static void _dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) { const bool spurious = flags & DEQUEUE_SAVE; const bool sleeping = flags & DEQUEUE_SLEEP; @@ -684,7 +684,7 @@ static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) * is not running (in this case we'll just update the cputime of * whatever task happens to be oncpu). * - * Ordinarily we wouldn't care but we routinely dequeue_task_ghost() + * Ordinarily we wouldn't care but we routinely _dequeue_task_ghost() * when migrating a task via ghost_move_task() during txn commit so * we call _update_curr_ghost() only if 'p' is actually running. */ @@ -1577,7 +1577,7 @@ DEFINE_SCHED_CLASS(ghost) = { .switched_to = switched_to_ghost, /* ghost_core.c */ .switched_from = switched_from_ghost, /* ghost_core.c */ .task_dead = task_dead_ghost, /* ghost_core.c */ - .dequeue_task = dequeue_task_ghost, + .dequeue_task = dequeue_task_ghost, /* ghost_core.c */ .put_prev_task = put_prev_task_ghost, .enqueue_task = enqueue_task_ghost, .set_next_task = set_next_task_ghost, @@ -1609,7 +1609,7 @@ static struct rq *ghost_move_task(struct rq *rq, struct task_struct *next, /* * Cleared in invalidate_cached_tasks() via move_queued_task() - * and dequeue_task_ghost(). We cannot clear it here because + * and _dequeue_task_ghost(). We cannot clear it here because * move_queued_task() will release rq->lock (the rq returned * by move_queued_task() is different than the one passed in). */ @@ -7462,5 +7462,6 @@ DEFINE_GHOST_ABI(current_abi) = { .switched_to = _switched_to_ghost, .switched_from = _switched_from_ghost, .task_dead = _task_dead_ghost, + .dequeue_task = _dequeue_task_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index d1735ebe85b6..e5ffc49363e6 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -952,6 +952,38 @@ void task_dead_ghost(struct task_struct *p) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } +void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->dequeue_task(rq, p, flags); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 52396dc98e59..6584c5723caa 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2228,6 +2228,7 @@ struct ghost_abi { void (*switched_to)(struct rq *rq, struct task_struct *p); void (*switched_from)(struct rq *rq, struct task_struct *p); void (*task_dead)(struct task_struct *p); + void (*dequeue_task)(struct rq *rq, struct task_struct *p, int flags); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2236,6 +2237,7 @@ void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old); void switched_to_ghost(struct rq *rq, struct task_struct *p); void switched_from_ghost(struct rq *rq, struct task_struct *p); void task_dead_ghost(struct task_struct *p); +void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From b6ba7583c6dc57edc0e2090db874cb27bc3b5a29 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 18:20:09 -0800 Subject: [PATCH 111/165] sched: add abi-specific callback for sched_class.put_prev_task Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I2926318922fbb5eb6e942fc9de2429c0744f8e3d --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 32 ++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 2d8ccdba9683..097dfdf4f64b 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -732,7 +732,7 @@ static void _dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) } } -static void put_prev_task_ghost(struct rq *rq, struct task_struct *p) +static void _put_prev_task_ghost(struct rq *rq, struct task_struct *p) { _update_curr_ghost(rq); } @@ -1578,7 +1578,7 @@ DEFINE_SCHED_CLASS(ghost) = { .switched_from = switched_from_ghost, /* ghost_core.c */ .task_dead = task_dead_ghost, /* ghost_core.c */ .dequeue_task = dequeue_task_ghost, /* ghost_core.c */ - .put_prev_task = put_prev_task_ghost, + .put_prev_task = put_prev_task_ghost, /* ghost_core.c */ .enqueue_task = enqueue_task_ghost, .set_next_task = set_next_task_ghost, .task_tick = task_tick_ghost, @@ -7463,5 +7463,6 @@ DEFINE_GHOST_ABI(current_abi) = { .switched_from = _switched_from_ghost, .task_dead = _task_dead_ghost, .dequeue_task = _dequeue_task_ghost, + .put_prev_task = _put_prev_task_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index e5ffc49363e6..c916f983cd27 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -984,6 +984,38 @@ void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) p->pid, p->comm, cpu_of(rq)); } +void put_prev_task_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->put_prev_task(rq, p); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6584c5723caa..23c362cf7aa8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2229,6 +2229,7 @@ struct ghost_abi { void (*switched_from)(struct rq *rq, struct task_struct *p); void (*task_dead)(struct task_struct *p); void (*dequeue_task)(struct rq *rq, struct task_struct *p, int flags); + void (*put_prev_task)(struct rq *rq, struct task_struct *p); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2238,6 +2239,7 @@ void switched_to_ghost(struct rq *rq, struct task_struct *p); void switched_from_ghost(struct rq *rq, struct task_struct *p); void task_dead_ghost(struct task_struct *p); void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags); +void put_prev_task_ghost(struct rq *rq, struct task_struct *p); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From b64d2e55ee3453e8eef4eab8c4b02c1c061cc458 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 18:39:42 -0800 Subject: [PATCH 112/165] sched: add abi-specific callback for sched_class.enqueue_task Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I3b8b506b951c2abf8c528bd279d15a4a6eb35f2f --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 32 ++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 097dfdf4f64b..f884d69d4397 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -738,7 +738,7 @@ static void _put_prev_task_ghost(struct rq *rq, struct task_struct *p) } static void -enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +_enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) { add_nr_running(rq, 1); rq->ghost.ghost_nr_running++; @@ -1579,7 +1579,7 @@ DEFINE_SCHED_CLASS(ghost) = { .task_dead = task_dead_ghost, /* ghost_core.c */ .dequeue_task = dequeue_task_ghost, /* ghost_core.c */ .put_prev_task = put_prev_task_ghost, /* ghost_core.c */ - .enqueue_task = enqueue_task_ghost, + .enqueue_task = enqueue_task_ghost, /* ghost_core.c */ .set_next_task = set_next_task_ghost, .task_tick = task_tick_ghost, .pick_next_task = pick_next_task_ghost, @@ -7464,5 +7464,6 @@ DEFINE_GHOST_ABI(current_abi) = { .task_dead = _task_dead_ghost, .dequeue_task = _dequeue_task_ghost, .put_prev_task = _put_prev_task_ghost, + .enqueue_task = _enqueue_task_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index c916f983cd27..217d5aaff3d3 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1016,6 +1016,38 @@ void put_prev_task_ghost(struct rq *rq, struct task_struct *p) p->pid, p->comm, cpu_of(rq)); } +void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->enqueue_task(rq, p, flags); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 23c362cf7aa8..9e8062d5265e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2230,6 +2230,7 @@ struct ghost_abi { void (*task_dead)(struct task_struct *p); void (*dequeue_task)(struct rq *rq, struct task_struct *p, int flags); void (*put_prev_task)(struct rq *rq, struct task_struct *p); + void (*enqueue_task)(struct rq *rq, struct task_struct *p, int flags); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2240,6 +2241,7 @@ void switched_from_ghost(struct rq *rq, struct task_struct *p); void task_dead_ghost(struct task_struct *p); void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags); void put_prev_task_ghost(struct rq *rq, struct task_struct *p); +void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From 5e0aea8fb02e38396d04c4a328019ca8f5a616d5 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 18:50:05 -0800 Subject: [PATCH 113/165] sched: add abi-specific callback for sched_class.set_curr_task Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I4ae6d0ed858f23f974e73e9fd40e627d3fd5ef53 --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 32 ++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 3 +++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index f884d69d4397..5825803c8fd3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -752,7 +752,7 @@ _enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) } } -static void set_next_task_ghost(struct rq *rq, struct task_struct *p, +static void _set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) { WARN_ON_ONCE(first); @@ -1580,7 +1580,7 @@ DEFINE_SCHED_CLASS(ghost) = { .dequeue_task = dequeue_task_ghost, /* ghost_core.c */ .put_prev_task = put_prev_task_ghost, /* ghost_core.c */ .enqueue_task = enqueue_task_ghost, /* ghost_core.c */ - .set_next_task = set_next_task_ghost, + .set_next_task = set_next_task_ghost, /* ghost_core.c */ .task_tick = task_tick_ghost, .pick_next_task = pick_next_task_ghost, .check_preempt_curr = check_preempt_curr_ghost, @@ -7465,5 +7465,6 @@ DEFINE_GHOST_ABI(current_abi) = { .dequeue_task = _dequeue_task_ghost, .put_prev_task = _put_prev_task_ghost, .enqueue_task = _enqueue_task_ghost, + .set_next_task = _set_next_task_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 217d5aaff3d3..03c25ff2354a 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1048,6 +1048,38 @@ void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) p->pid, p->comm, cpu_of(rq)); } +void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->set_next_task(rq, p, first); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9e8062d5265e..e7021bdb0867 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2231,6 +2231,8 @@ struct ghost_abi { void (*dequeue_task)(struct rq *rq, struct task_struct *p, int flags); void (*put_prev_task)(struct rq *rq, struct task_struct *p); void (*enqueue_task)(struct rq *rq, struct task_struct *p, int flags); + void (*set_next_task)(struct rq *rq, struct task_struct *p, + bool first); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2242,6 +2244,7 @@ void task_dead_ghost(struct task_struct *p); void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags); void put_prev_task_ghost(struct rq *rq, struct task_struct *p); void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags); +void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From 3a830dc17d9b76fe504257e11fa159aa131312b1 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 19:04:15 -0800 Subject: [PATCH 114/165] sched: add abi-specific callback for sched_class.task_tick Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I8fe3f9c9c16881c296c057bd8b3906782fd555e4 --- kernel/sched/ghost.c | 9 +++------ kernel/sched/ghost_core.c | 36 ++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 5825803c8fd3..27d0526359de 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -848,11 +848,7 @@ static void tick_handler(struct ghost_enclave *e, struct rq *rq) } } -/* - * Called from the timer tick handler while holding the rq->lock. Called only - * if a ghost task is current. - */ -static void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) +static void _task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) { struct task_struct *agent = rq->ghost.agent; @@ -1581,7 +1577,7 @@ DEFINE_SCHED_CLASS(ghost) = { .put_prev_task = put_prev_task_ghost, /* ghost_core.c */ .enqueue_task = enqueue_task_ghost, /* ghost_core.c */ .set_next_task = set_next_task_ghost, /* ghost_core.c */ - .task_tick = task_tick_ghost, + .task_tick = task_tick_ghost, /* ghost_core.c */ .pick_next_task = pick_next_task_ghost, .check_preempt_curr = check_preempt_curr_ghost, .yield_task = yield_task_ghost, @@ -7466,5 +7462,6 @@ DEFINE_GHOST_ABI(current_abi) = { .put_prev_task = _put_prev_task_ghost, .enqueue_task = _enqueue_task_ghost, .set_next_task = _set_next_task_ghost, + .task_tick = _task_tick_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 03c25ff2354a..52e61ed1cf74 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1080,6 +1080,42 @@ void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) p->pid, p->comm, cpu_of(rq)); } +/* + * Called from the timer tick handler while holding the rq->lock. Called only + * if a ghost task is current. + */ +void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->task_tick(rq, p, queued); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e7021bdb0867..069b9fe860ba 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2233,6 +2233,7 @@ struct ghost_abi { void (*enqueue_task)(struct rq *rq, struct task_struct *p, int flags); void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); + void (*task_tick)(struct rq *rq, struct task_struct *p, int queued); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2245,6 +2246,7 @@ void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags); void put_prev_task_ghost(struct rq *rq, struct task_struct *p); void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags); void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first); +void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From b6f63bfbfee21ffa382aa19d8682aaa2ab782205 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 19:46:05 -0800 Subject: [PATCH 115/165] sched: add abi-specific callback for sched_class.pick_next_task Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I9cfbeb38e6ab028a862dd0b1dcf496c200208184 --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 18 ++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 27d0526359de..9103c5c0fe5c 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1300,7 +1300,7 @@ static struct task_struct *pick_next_ghost_agent(struct rq *rq) return next; } -static struct task_struct *pick_next_task_ghost(struct rq *rq) +static struct task_struct *_pick_next_task_ghost(struct rq *rq) { struct task_struct *agent = rq->ghost.agent; struct task_struct *prev = rq->curr; @@ -1578,7 +1578,7 @@ DEFINE_SCHED_CLASS(ghost) = { .enqueue_task = enqueue_task_ghost, /* ghost_core.c */ .set_next_task = set_next_task_ghost, /* ghost_core.c */ .task_tick = task_tick_ghost, /* ghost_core.c */ - .pick_next_task = pick_next_task_ghost, + .pick_next_task = pick_next_task_ghost, /* ghost_core.c */ .check_preempt_curr = check_preempt_curr_ghost, .yield_task = yield_task_ghost, #ifdef CONFIG_SMP @@ -7463,5 +7463,6 @@ DEFINE_GHOST_ABI(current_abi) = { .enqueue_task = _enqueue_task_ghost, .set_next_task = _set_next_task_ghost, .task_tick = _task_tick_ghost, + .pick_next_task = _pick_next_task_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 52e61ed1cf74..a20a7960933d 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1116,6 +1116,24 @@ void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) p->pid, p->comm, cpu_of(rq)); } +struct task_struct *pick_next_task_ghost(struct rq *rq) +{ + struct task_struct *next; + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + + /* Implicit read-side critical section due to disabled preemption */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + if (e) + next = e->abi->pick_next_task(rq); + else + next = NULL; + + return next; +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 069b9fe860ba..8ec19baf8aa5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2234,6 +2234,7 @@ struct ghost_abi { void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first); void (*task_tick)(struct rq *rq, struct task_struct *p, int queued); + struct task_struct *(*pick_next_task)(struct rq *rq); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2247,6 +2248,7 @@ void put_prev_task_ghost(struct rq *rq, struct task_struct *p); void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags); void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first); void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued); +struct task_struct *pick_next_task_ghost(struct rq *rq); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From c91e13dd7aa81ceb54f5e5521fcd172f56e97fa0 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 20:00:51 -0800 Subject: [PATCH 116/165] sched: add abi-specific callback for sched_class.check_preempt_curr Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I7368eb2741599c7307412b65a86c6a8224b2c107 --- kernel/sched/ghost.c | 7 ++++--- kernel/sched/ghost_core.c | 33 +++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 4 ++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 9103c5c0fe5c..120238991f2d 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1435,8 +1435,8 @@ static struct task_struct *_pick_next_task_ghost(struct rq *rq) return next; } -static void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, - int wake_flags) +static void _check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, + int wake_flags) { if (is_agent(rq, p)) { /* @@ -1579,7 +1579,7 @@ DEFINE_SCHED_CLASS(ghost) = { .set_next_task = set_next_task_ghost, /* ghost_core.c */ .task_tick = task_tick_ghost, /* ghost_core.c */ .pick_next_task = pick_next_task_ghost, /* ghost_core.c */ - .check_preempt_curr = check_preempt_curr_ghost, + .check_preempt_curr = check_preempt_curr_ghost, /* ghost_core.c */ .yield_task = yield_task_ghost, #ifdef CONFIG_SMP .balance = balance_ghost, @@ -7464,5 +7464,6 @@ DEFINE_GHOST_ABI(current_abi) = { .set_next_task = _set_next_task_ghost, .task_tick = _task_tick_ghost, .pick_next_task = _pick_next_task_ghost, + .check_preempt_curr = _check_preempt_curr_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index a20a7960933d..fd807d4f32da 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1134,6 +1134,39 @@ struct task_struct *pick_next_task_ghost(struct rq *rq) return next; } +void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, + int wake_flags) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->check_preempt_curr(rq, p, wake_flags); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 8ec19baf8aa5..9815f76ef202 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2235,6 +2235,8 @@ struct ghost_abi { bool first); void (*task_tick)(struct rq *rq, struct task_struct *p, int queued); struct task_struct *(*pick_next_task)(struct rq *rq); + void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, + int wake_flags); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2249,6 +2251,8 @@ void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags); void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first); void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued); struct task_struct *pick_next_task_ghost(struct rq *rq); +void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, + int wake_flags); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From 234048a6e54a41ff0ca8536ac290b885e86d1a88 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 20:30:09 -0800 Subject: [PATCH 117/165] sched: add abi-specific callback for sched_class.yield_task Tested: all unit tests passing in virtme Effort: sched/ghost/abi Change-Id: Idbbeb0a1dfee3483ea4933bd1185e622d423e9d4 --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 34 ++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 120238991f2d..8beecbfec16a 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1466,7 +1466,7 @@ static void _check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, } } -static void yield_task_ghost(struct rq *rq) +static void _yield_task_ghost(struct rq *rq) { struct task_struct *curr = rq->curr; @@ -1580,7 +1580,7 @@ DEFINE_SCHED_CLASS(ghost) = { .task_tick = task_tick_ghost, /* ghost_core.c */ .pick_next_task = pick_next_task_ghost, /* ghost_core.c */ .check_preempt_curr = check_preempt_curr_ghost, /* ghost_core.c */ - .yield_task = yield_task_ghost, + .yield_task = yield_task_ghost, /* ghost_core.c */ #ifdef CONFIG_SMP .balance = balance_ghost, .select_task_rq = select_task_rq_ghost, @@ -7465,5 +7465,6 @@ DEFINE_GHOST_ABI(current_abi) = { .task_tick = _task_tick_ghost, .pick_next_task = _pick_next_task_ghost, .check_preempt_curr = _check_preempt_curr_ghost, + .yield_task = _yield_task_ghost, }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index fd807d4f32da..f9f15d6b0eba 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1167,6 +1167,40 @@ void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, p->pid, p->comm, cpu_of(rq)); } +void yield_task_ghost(struct rq *rq) +{ + struct task_struct *p = current; + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->yield_task(rq); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9815f76ef202..92bf8fa382aa 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2237,6 +2237,7 @@ struct ghost_abi { struct task_struct *(*pick_next_task)(struct rq *rq); void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int wake_flags); + void (*yield_task)(struct rq *rq); }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2253,6 +2254,7 @@ void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued); struct task_struct *pick_next_task_ghost(struct rq *rq); void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, int wake_flags); +void yield_task_ghost(struct rq *rq); #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From ffe002a9274b0aa4c4b0b941a02d81c6542676c9 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 20:41:33 -0800 Subject: [PATCH 118/165] sched: add abi-specific callback for sched_class.select_task_rq Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: Iabe17576cbdb2f721dce4b1e09050306c6d6ce5d --- kernel/sched/ghost.c | 7 +++++-- kernel/sched/ghost_core.c | 14 ++++++++++++++ kernel/sched/sched.h | 6 ++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8beecbfec16a..8414e54850bb 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -932,7 +932,7 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, return rq->ghost.latched_task || rq_adj_nr_running(rq); } -static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) +static int _select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) { int waker_cpu = smp_processor_id(); @@ -1583,7 +1583,7 @@ DEFINE_SCHED_CLASS(ghost) = { .yield_task = yield_task_ghost, /* ghost_core.c */ #ifdef CONFIG_SMP .balance = balance_ghost, - .select_task_rq = select_task_rq_ghost, + .select_task_rq = select_task_rq_ghost, /* ghost_core.c */ .task_woken = task_woken_ghost, .set_cpus_allowed = set_cpus_allowed_ghost, #endif @@ -7466,5 +7466,8 @@ DEFINE_GHOST_ABI(current_abi) = { .pick_next_task = _pick_next_task_ghost, .check_preempt_curr = _check_preempt_curr_ghost, .yield_task = _yield_task_ghost, +#ifdef CONFIG_SMP + .select_task_rq = _select_task_rq_ghost, +#endif }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index f9f15d6b0eba..3176bbf9d76c 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1201,6 +1201,20 @@ void yield_task_ghost(struct rq *rq) p->pid, p->comm, cpu_of(rq)); } +int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) +{ + struct ghost_enclave *e; + + lockdep_assert_held(&p->pi_lock); + + e = p->ghost.enclave; + if (e) + return e->abi->select_task_rq(p, cpu, wake_flags); + + WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); + return 0; +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 92bf8fa382aa..d393cf531dd8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2238,6 +2238,9 @@ struct ghost_abi { void (*check_preempt_curr)(struct rq *rq, struct task_struct *p, int wake_flags); void (*yield_task)(struct rq *rq); +#ifdef CONFIG_SMP + int (*select_task_rq)(struct task_struct *p, int cpu, int wake_flags); +#endif }; /* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ @@ -2255,6 +2258,9 @@ struct task_struct *pick_next_task_ghost(struct rq *rq); void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, int wake_flags); void yield_task_ghost(struct rq *rq); +#ifdef CONFIG_SMP +int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags); +#endif #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ From 111c64469cb7cbd3ccb28307dfabd660640f1480 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 20:53:31 -0800 Subject: [PATCH 119/165] sched: add abi-specific callback for sched_class.task_woken Tested: all unit tests passing in virtme Effort: sched/ghost/abi Change-Id: Id532de2a106fd42c68eb6114fb431c6a304b5edc --- kernel/sched/ghost.c | 5 +++-- kernel/sched/ghost_core.c | 32 ++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 2 ++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8414e54850bb..721478803c76 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1542,7 +1542,7 @@ static void set_cpus_allowed_ghost(struct task_struct *p, set_cpus_allowed_common(p, newmask, flags); } -static void task_woken_ghost(struct rq *rq, struct task_struct *p) +static void _task_woken_ghost(struct rq *rq, struct task_struct *p) { struct ghost_status_word *sw = p->ghost.status_word; @@ -1584,7 +1584,7 @@ DEFINE_SCHED_CLASS(ghost) = { #ifdef CONFIG_SMP .balance = balance_ghost, .select_task_rq = select_task_rq_ghost, /* ghost_core.c */ - .task_woken = task_woken_ghost, + .task_woken = task_woken_ghost, /* ghost_core.c */ .set_cpus_allowed = set_cpus_allowed_ghost, #endif }; @@ -7468,6 +7468,7 @@ DEFINE_GHOST_ABI(current_abi) = { .yield_task = _yield_task_ghost, #ifdef CONFIG_SMP .select_task_rq = _select_task_rq_ghost, + .task_woken = _task_woken_ghost, #endif }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 3176bbf9d76c..cb4c6fc51eaa 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1215,6 +1215,38 @@ int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) return 0; } +void task_woken_ghost(struct rq *rq, struct task_struct *p) +{ + struct ghost_enclave *rq_enclave, *task_enclave, *e; + + VM_BUG_ON(preemptible()); + lockdep_assert_held(&rq->lock); + + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave); + + /* + * XXX it is possible for a running task to enter ghost into enclave X + * while the cpu it is running on belongs to enclave Y. This is not + * supported properly yet hence the VM_BUG_ON. + */ + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); + + /* + * 'rq_enclave' may be NULL if a task is woken up on a cpu + * that does not belong to any enclave. + */ + e = rq_enclave ? rq_enclave : task_enclave; + if (e) + e->abi->task_woken(rq, p); + else + WARN_ONCE(1, "task %d/%s without enclave on cpu %d", + p->pid, p->comm, cpu_of(rq)); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d393cf531dd8..c9c3bd68511d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2240,6 +2240,7 @@ struct ghost_abi { void (*yield_task)(struct rq *rq); #ifdef CONFIG_SMP int (*select_task_rq)(struct task_struct *p, int cpu, int wake_flags); + void (*task_woken)(struct rq *rq, struct task_struct *p); #endif }; @@ -2260,6 +2261,7 @@ void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, void yield_task_ghost(struct rq *rq); #ifdef CONFIG_SMP int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags); +void task_woken_ghost(struct rq *rq, struct task_struct *p); #endif #define DEFINE_GHOST_ABI(name) \ From 26104b1188b14f7c911dfb40c29b22b640e7030a Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 21:03:47 -0800 Subject: [PATCH 120/165] sched: add abi-specific callback for sched_class.set_cpus_allowed Tested: all unit tests passing in virtme Effort: sched/ghost/abi Change-Id: I326aa749c887cec16ea403ada5cc882676e01efa --- kernel/sched/ghost.c | 7 ++++--- kernel/sched/ghost_core.c | 14 ++++++++++++++ kernel/sched/sched.h | 4 ++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 721478803c76..9ccd932e140f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1504,8 +1504,8 @@ static void _yield_task_ghost(struct rq *rq) curr->ghost.yield_task = true; } -static void set_cpus_allowed_ghost(struct task_struct *p, - const struct cpumask *newmask, u32 flags) +static void _set_cpus_allowed_ghost(struct task_struct *p, + const struct cpumask *newmask, u32 flags) { struct rq_flags rf; struct rq *rq = task_rq(p); @@ -1585,7 +1585,7 @@ DEFINE_SCHED_CLASS(ghost) = { .balance = balance_ghost, .select_task_rq = select_task_rq_ghost, /* ghost_core.c */ .task_woken = task_woken_ghost, /* ghost_core.c */ - .set_cpus_allowed = set_cpus_allowed_ghost, + .set_cpus_allowed = set_cpus_allowed_ghost, /* ghost_core.c */ #endif }; @@ -7469,6 +7469,7 @@ DEFINE_GHOST_ABI(current_abi) = { #ifdef CONFIG_SMP .select_task_rq = _select_task_rq_ghost, .task_woken = _task_woken_ghost, + .set_cpus_allowed = _set_cpus_allowed_ghost, #endif }; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index cb4c6fc51eaa..517152e561b2 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1247,6 +1247,20 @@ void task_woken_ghost(struct rq *rq, struct task_struct *p) p->pid, p->comm, cpu_of(rq)); } +void set_cpus_allowed_ghost(struct task_struct *p, + const struct cpumask *newmask, u32 flags) +{ + struct ghost_enclave *e; + + lockdep_assert_held(&p->pi_lock); + + e = p->ghost.enclave; + if (e) + e->abi->set_cpus_allowed(p, newmask, flags); + else + WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); +} + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c9c3bd68511d..aa01387670ed 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2241,6 +2241,8 @@ struct ghost_abi { #ifdef CONFIG_SMP int (*select_task_rq)(struct task_struct *p, int cpu, int wake_flags); void (*task_woken)(struct rq *rq, struct task_struct *p); + void (*set_cpus_allowed)(struct task_struct *p, + const struct cpumask *newmask, u32 flags); #endif }; @@ -2262,6 +2264,8 @@ void yield_task_ghost(struct rq *rq); #ifdef CONFIG_SMP int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags); void task_woken_ghost(struct rq *rq, struct task_struct *p); +void set_cpus_allowed_ghost(struct task_struct *p, + const struct cpumask *newmask, u32 flags); #endif #define DEFINE_GHOST_ABI(name) \ From 5b959e7dd6bc7e9e652c6ac67b6e0ea85d2004d1 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 17 Nov 2021 21:10:15 -0800 Subject: [PATCH 121/165] sched: move 'ghost_sched_class' into abi-independent code. Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I5dc9bde0eb633377fb639d91beea17bf2d2ee808 --- kernel/sched/ghost.c | 29 +++-------------- kernel/sched/ghost_core.c | 68 ++++++++++++++++++++++++++++++--------- kernel/sched/sched.h | 24 ++------------ 3 files changed, 59 insertions(+), 62 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 9ccd932e140f..9681c709c8bc 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -892,8 +892,8 @@ static void _task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) ghost_wake_agent_on(agent_target_cpu(rq)); } -static int balance_ghost(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf) +static int _balance_ghost(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) { struct task_struct *agent = rq->ghost.agent; @@ -1567,28 +1567,6 @@ static void _task_woken_ghost(struct rq *rq, struct task_struct *p) ghost_wake_agent_of(p); } -DEFINE_SCHED_CLASS(ghost) = { - .update_curr = update_curr_ghost, /* ghost_core.c */ - .prio_changed = prio_changed_ghost, /* ghost_core.c */ - .switched_to = switched_to_ghost, /* ghost_core.c */ - .switched_from = switched_from_ghost, /* ghost_core.c */ - .task_dead = task_dead_ghost, /* ghost_core.c */ - .dequeue_task = dequeue_task_ghost, /* ghost_core.c */ - .put_prev_task = put_prev_task_ghost, /* ghost_core.c */ - .enqueue_task = enqueue_task_ghost, /* ghost_core.c */ - .set_next_task = set_next_task_ghost, /* ghost_core.c */ - .task_tick = task_tick_ghost, /* ghost_core.c */ - .pick_next_task = pick_next_task_ghost, /* ghost_core.c */ - .check_preempt_curr = check_preempt_curr_ghost, /* ghost_core.c */ - .yield_task = yield_task_ghost, /* ghost_core.c */ -#ifdef CONFIG_SMP - .balance = balance_ghost, - .select_task_rq = select_task_rq_ghost, /* ghost_core.c */ - .task_woken = task_woken_ghost, /* ghost_core.c */ - .set_cpus_allowed = set_cpus_allowed_ghost, /* ghost_core.c */ -#endif -}; - /* * Migrate 'next' (if necessary) in preparation to run it on 'cpu'. * @@ -4601,7 +4579,7 @@ static void ghost_task_yield(struct rq *rq, struct task_struct *p) /* See explanation in ghost_task_preempted() */ if (p == rq->curr) - update_curr_ghost(rq); + _update_curr_ghost(rq); task_deliver_msg_yield(rq, p); } @@ -7467,6 +7445,7 @@ DEFINE_GHOST_ABI(current_abi) = { .check_preempt_curr = _check_preempt_curr_ghost, .yield_task = _yield_task_ghost, #ifdef CONFIG_SMP + .balance = _balance_ghost, .select_task_rq = _select_task_rq_ghost, .task_woken = _task_woken_ghost, .set_cpus_allowed = _set_cpus_allowed_ghost, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 517152e561b2..7b0af3396644 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -860,7 +860,7 @@ DEFINE_SCHED_CLASS(ghost_agent) = { #endif }; -void update_curr_ghost(struct rq *rq) +static void update_curr_ghost(struct rq *rq) { struct ghost_enclave *rq_enclave, *curr_enclave, *e; @@ -890,7 +890,7 @@ void update_curr_ghost(struct rq *rq) e->abi->update_curr(rq); } -void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) +static void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) { struct ghost_enclave *e; @@ -904,7 +904,7 @@ void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } -void switched_to_ghost(struct rq *rq, struct task_struct *p) +static void switched_to_ghost(struct rq *rq, struct task_struct *p) { struct ghost_enclave *e; @@ -918,7 +918,7 @@ void switched_to_ghost(struct rq *rq, struct task_struct *p) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } -void switched_from_ghost(struct rq *rq, struct task_struct *p) +static void switched_from_ghost(struct rq *rq, struct task_struct *p) { struct ghost_enclave *e; @@ -932,7 +932,7 @@ void switched_from_ghost(struct rq *rq, struct task_struct *p) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } -void task_dead_ghost(struct task_struct *p) +static void task_dead_ghost(struct task_struct *p) { struct ghost_enclave *e; @@ -952,7 +952,7 @@ void task_dead_ghost(struct task_struct *p) WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } -void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -984,7 +984,7 @@ void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) p->pid, p->comm, cpu_of(rq)); } -void put_prev_task_ghost(struct rq *rq, struct task_struct *p) +static void put_prev_task_ghost(struct rq *rq, struct task_struct *p) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1016,7 +1016,7 @@ void put_prev_task_ghost(struct rq *rq, struct task_struct *p) p->pid, p->comm, cpu_of(rq)); } -void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) +static void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1048,7 +1048,7 @@ void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) p->pid, p->comm, cpu_of(rq)); } -void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) +static void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1084,7 +1084,7 @@ void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) * Called from the timer tick handler while holding the rq->lock. Called only * if a ghost task is current. */ -void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) +static void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1116,7 +1116,7 @@ void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) p->pid, p->comm, cpu_of(rq)); } -struct task_struct *pick_next_task_ghost(struct rq *rq) +static struct task_struct *pick_next_task_ghost(struct rq *rq) { struct task_struct *next; struct ghost_enclave *e; @@ -1134,7 +1134,7 @@ struct task_struct *pick_next_task_ghost(struct rq *rq) return next; } -void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, +static void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, int wake_flags) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1167,7 +1167,7 @@ void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, p->pid, p->comm, cpu_of(rq)); } -void yield_task_ghost(struct rq *rq) +static void yield_task_ghost(struct rq *rq) { struct task_struct *p = current; struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1201,7 +1201,7 @@ void yield_task_ghost(struct rq *rq) p->pid, p->comm, cpu_of(rq)); } -int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) +static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) { struct ghost_enclave *e; @@ -1215,6 +1215,22 @@ int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) return 0; } +static int balance_ghost(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) +{ + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + VM_BUG_ON(rq != this_rq()); + + /* Implicit read-side critical section due to disabled preemption */ + e = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + if (e) + return e->abi->balance(rq, prev, rf); + + return -1; +} + void task_woken_ghost(struct rq *rq, struct task_struct *p) { struct ghost_enclave *rq_enclave, *task_enclave, *e; @@ -1247,7 +1263,7 @@ void task_woken_ghost(struct rq *rq, struct task_struct *p) p->pid, p->comm, cpu_of(rq)); } -void set_cpus_allowed_ghost(struct task_struct *p, +static void set_cpus_allowed_ghost(struct task_struct *p, const struct cpumask *newmask, u32 flags) { struct ghost_enclave *e; @@ -1261,6 +1277,28 @@ void set_cpus_allowed_ghost(struct task_struct *p, WARN_ONCE(1, "task %d/%s without enclave", p->pid, p->comm); } +DEFINE_SCHED_CLASS(ghost) = { + .update_curr = update_curr_ghost, + .prio_changed = prio_changed_ghost, + .switched_to = switched_to_ghost, + .switched_from = switched_from_ghost, + .task_dead = task_dead_ghost, + .dequeue_task = dequeue_task_ghost, + .put_prev_task = put_prev_task_ghost, + .enqueue_task = enqueue_task_ghost, + .set_next_task = set_next_task_ghost, + .task_tick = task_tick_ghost, + .pick_next_task = pick_next_task_ghost, + .check_preempt_curr = check_preempt_curr_ghost, + .yield_task = yield_task_ghost, +#ifdef CONFIG_SMP + .balance = balance_ghost, + .select_task_rq = select_task_rq_ghost, + .task_woken = task_woken_ghost, + .set_cpus_allowed = set_cpus_allowed_ghost, +#endif +}; + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index aa01387670ed..eefad5e9b566 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2239,6 +2239,8 @@ struct ghost_abi { int wake_flags); void (*yield_task)(struct rq *rq); #ifdef CONFIG_SMP + int (*balance)(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf); int (*select_task_rq)(struct task_struct *p, int cpu, int wake_flags); void (*task_woken)(struct rq *rq, struct task_struct *p); void (*set_cpus_allowed)(struct task_struct *p, @@ -2246,28 +2248,6 @@ struct ghost_abi { #endif }; -/* temporary: remove after moving 'ghost_sched_class' into ghost_core.c */ -void update_curr_ghost(struct rq *rq); -void prio_changed_ghost(struct rq *rq, struct task_struct *p, int old); -void switched_to_ghost(struct rq *rq, struct task_struct *p); -void switched_from_ghost(struct rq *rq, struct task_struct *p); -void task_dead_ghost(struct task_struct *p); -void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags); -void put_prev_task_ghost(struct rq *rq, struct task_struct *p); -void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags); -void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first); -void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued); -struct task_struct *pick_next_task_ghost(struct rq *rq); -void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, - int wake_flags); -void yield_task_ghost(struct rq *rq); -#ifdef CONFIG_SMP -int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags); -void task_woken_ghost(struct rq *rq, struct task_struct *p); -void set_cpus_allowed_ghost(struct task_struct *p, - const struct cpumask *newmask, u32 flags); -#endif - #define DEFINE_GHOST_ABI(name) \ const static struct ghost_abi __##name##_ghost_abi \ __aligned(__alignof__(struct ghost_abi)) \ From 5d92e048322733f369eecefdce5136f5ca9374c7 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 18 Nov 2021 12:32:22 -0800 Subject: [PATCH 122/165] sched: use an ioctl for ghost_run() Closely based on go/kcl/440830 authored by hannahpan@ Submitted in conjunction with cl/410872654 Tested: all unit tests pass in virtme Effort: sched/ghost/abi Change-Id: I9d95675d1afa64f92de552d2d2b1ede8be054402 --- arch/x86/entry/syscalls/syscall_64.tbl | 1 - include/linux/syscalls.h | 2 -- include/uapi/linux/ghost.h | 14 ++++++++++---- kernel/sched/core.c | 6 ------ kernel/sched/ghost.c | 21 +++++++++++++++++++-- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index b7b24da49776..098adf13d606 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -363,7 +363,6 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 -450 64 ghost_run sys_ghost_run 451 64 ghost sys_ghost # diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index fad3ee1a703a..3f4fd20b4829 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1365,8 +1365,6 @@ int __sys_getsockopt(int fd, int level, int optname, char __user *optval, int __sys_setsockopt(int fd, int level, int optname, char __user *optval, int optlen); -asmlinkage long gsys_ghost_run(s64 gtid, u32 agent_barrier, u32 task_barrier, - int run_cpu, int run_flags); asmlinkage long gsys_ghost(u64 op, u64 arg1, u64 arg2, u64 arg3, u64 arg4, u64 arg5); diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 9126e3d348fc..af767c87b091 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 59 +#define GHOST_VERSION 60 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -149,6 +149,14 @@ struct ghost_ioc_timerfd_settime { struct timerfd_ghost timerfd_ghost; }; +struct ghost_ioc_run { + int64_t gtid; + uint32_t agent_barrier; + uint32_t task_barrier; + int run_cpu; + int run_flags; +}; + #define GHOST_IOC_NULL _IO('g', 0) #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info) @@ -160,6 +168,7 @@ struct ghost_ioc_timerfd_settime { #define GHOST_IOC_COMMIT_TXN _IOW('g', 8, struct ghost_ioc_commit_txn) #define GHOST_IOC_SYNC_GROUP_TXN _IOW('g', 9, struct ghost_ioc_commit_txn) #define GHOST_IOC_TIMERFD_SETTIME _IOWR('g', 10, struct ghost_ioc_timerfd_settime) +#define GHOST_IOC_RUN _IOW('g', 11, struct ghost_ioc_run) /* * Status word region APIs. @@ -427,9 +436,6 @@ struct ghost_ring { * Define ghOSt syscall numbers here until they can be discovered via * . */ -#ifndef __NR_ghost_run -#define __NR_ghost_run 450 -#endif #ifndef __NR_ghost #define __NR_ghost 451 #endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6b2d8d8f8af4..86035625825f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9422,12 +9422,6 @@ struct cgroup_subsys cpu_cgrp_subsys = { #endif /* CONFIG_CGROUP_SCHED */ #ifndef CONFIG_SCHED_CLASS_GHOST -SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, u32, task_barrier, - int, run_cpu, int, run_flags) -{ - return -ENOSYS; -} - SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, u64, arg3, u64, arg4, u64, arg5) { diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 9681c709c8bc..ba30734edae8 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4724,8 +4724,7 @@ static inline bool commit_flags_valid(int commit_flags, int valid_commit_flags) * * Return 0 on success and -1 on failure. */ -SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, - u32, task_barrier, int, run_cpu, int, run_flags) +static int ghost_run(struct ghost_enclave *e, struct ghost_ioc_run __user *arg) { struct task_struct *agent; struct rq_flags rf; @@ -4733,11 +4732,27 @@ SYSCALL_DEFINE5(ghost_run, s64, gtid, u32, agent_barrier, int error = 0; int this_cpu; + s64 gtid; + u32 agent_barrier; + u32 task_barrier; + int run_cpu; + int run_flags; + struct ghost_ioc_run run_params; + const int supported_flags = RTLA_ON_IDLE; if (!capable(CAP_SYS_NICE)) return -EPERM; + if (copy_from_user(&run_params, arg, sizeof(struct ghost_ioc_run))) + return -EFAULT; + + gtid = run_params.gtid; + agent_barrier = run_params.agent_barrier; + task_barrier = run_params.task_barrier; + run_cpu = run_params.run_cpu; + run_flags = run_params.run_flags; + if (run_cpu < 0 || run_cpu >= nr_cpu_ids || !cpu_online(run_cpu)) return -EINVAL; @@ -6670,6 +6685,8 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, case GHOST_IOC_TIMERFD_SETTIME: return ghost_timerfd_settime( (struct ghost_ioc_timerfd_settime __user *)arg); + case GHOST_IOC_RUN: + return ghost_run(e, (struct ghost_ioc_run __user *)arg); } return -ENOIOCTLCMD; } From a29bcf88645e7b6d8f6ebf6c70226e30512ef999 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 18 Nov 2021 13:10:22 -0800 Subject: [PATCH 123/165] sched: deprecate ghost syscall Tested: all unit tests passing in virtme Effort: sched/ghost/abi Change-Id: Ide9e85f74518435908ac1d174b881023a736729a --- arch/x86/entry/syscalls/syscall_64.tbl | 1 - include/linux/syscalls.h | 3 -- include/uapi/linux/ghost.h | 10 +--- kernel/sched/core.c | 8 --- kernel/sched/ghost.c | 73 -------------------------- 5 files changed, 1 insertion(+), 94 deletions(-) diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 098adf13d606..78672124d28b 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -363,7 +363,6 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 -451 64 ghost sys_ghost # # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 3f4fd20b4829..ba97b5480662 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1365,7 +1365,4 @@ int __sys_getsockopt(int fd, int level, int optname, char __user *optval, int __sys_setsockopt(int fd, int level, int optname, char __user *optval, int optlen); -asmlinkage long gsys_ghost(u64 op, u64 arg1, u64 arg2, u64 arg3, u64 arg4, - u64 arg5); - #endif diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index af767c87b091..90beb1631869 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 60 +#define GHOST_VERSION 61 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -432,14 +432,6 @@ struct ghost_ring { #define GHOST_MAX_QUEUE_ELEMS 65536 /* arbitrary */ -/* - * Define ghOSt syscall numbers here until they can be discovered via - * . - */ -#ifndef __NR_ghost -#define __NR_ghost 451 -#endif - /* * 'ops' supported by gsys_ghost(). */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 86035625825f..d5fd788268bb 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -9421,14 +9421,6 @@ struct cgroup_subsys cpu_cgrp_subsys = { #endif /* CONFIG_CGROUP_SCHED */ -#ifndef CONFIG_SCHED_CLASS_GHOST -SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, u64, arg3, u64, arg4, - u64, arg5) -{ - return -ENOSYS; -} -#endif - void dump_cpu_task(int cpu) { pr_info("Task dump for CPU %d:\n", cpu); diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index ba30734edae8..0f3aabfa9861 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6069,79 +6069,6 @@ static void _ghost_timerfd_triggered(int cpu, uint64_t cookie) rq_unlock_irqrestore(rq, &rf); } -static int ghost_gtid_lookup(int64_t id, int op, int flags, int64_t __user *out) -{ - int error = 0; - int64_t result; - struct task_struct *target; - - if (flags) - return -EINVAL; - - rcu_read_lock(); - target = find_task_by_gtid(id); - if (!target) { - rcu_read_unlock(); - return -ESRCH; - } - - switch (op) { - case GHOST_GTID_LOOKUP_TGID: - result = task_tgid_nr(target); - break; - default: - error = -EINVAL; - break; - } - rcu_read_unlock(); - - if (!error) { - if (copy_to_user(out, &result, sizeof(result))) - error = -EFAULT; - } - - return error; -} - -static int ghost_get_gtid(int64_t __user *out) -{ - gtid_t gtid = current->gtid; - - if (copy_to_user(out, >id, sizeof(gtid))) - return -EFAULT; - - return 0; -} - -/* - * TODO: Access to the ghost syscalls needs to be restricted, probably via a - * capability. - */ -SYSCALL_DEFINE6(ghost, u64, op, u64, arg1, u64, arg2, - u64, arg3, u64, arg4, u64, arg5) -{ - bool be_nice = true; - - if (op == GHOST_BASE_GET_GTID) - be_nice = false; - - if (be_nice && !capable(CAP_SYS_NICE)) - return -EPERM; - - switch (op) { - case GHOST_GTID_LOOKUP: - return ghost_gtid_lookup(arg1, arg2, arg3, - (int64_t __user *)arg4); - case GHOST_BASE_GET_GTID: - return ghost_get_gtid((int64_t __user *)arg1); - default: - if (op >= _GHOST_BASE_OP_FIRST) - return -EOPNOTSUPP; - else - return -EINVAL; - } -} - #ifndef SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK #define SYS_SWITCHTO_SWITCH_FLAGS_LAZY_EXEC_CLOCK 0x10000 #endif From 7e1e33c2ce34157d7927a3e7cdcea5df73fdc594 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 18 Nov 2021 17:49:50 -0800 Subject: [PATCH 124/165] sched: include "kernfs-internal.h" from ghost.c Add $(srctree)/fs/kernfs to the include path when compiling ghost.c. This allows ghost.c to include "kernfs-internal.h" without knowledge about where it is located in the source tree relative to fs/kernfs. Tested: builds Effort: sched/ghost/abi Change-Id: Iee576f3191205fbf6143271be1c08decca02e275 --- kernel/sched/Makefile | 1 + kernel/sched/ghost.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 6ba3d3662224..b7b31ec8e18b 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -36,5 +36,6 @@ obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o +CFLAGS_ghost.o += -I$(srctree)/fs/kernfs obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost.o obj-$(CONFIG_SCHED_CLASS_GHOST) += ghost_core.o diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 0f3aabfa9861..119b51408996 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -27,6 +27,7 @@ #include #include "sched.h" +#include "kernfs-internal.h" #define __INCLUDE_KERNEL_SCHED_GHOST #include "ghost_uapi.h" @@ -6124,8 +6125,6 @@ static void cpu_idle(struct rq *rq) rq_unlock_irq(rq, &rf); } -#include "../../fs/kernfs/kernfs-internal.h" - /* Helper for when you "echo foo > ctl" without the -n. */ static void strip_slash_n(char *buf, size_t count) { From 89c1edaec2be5bcbc11ee257c6de072bfa5d99b5 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 18 Nov 2021 20:14:20 -0800 Subject: [PATCH 125/165] sched: helper script to easily manage ghost abi snapshots. Taking a snapshot of the current ABI involves the following (assuming current abi is 60): - create a 'kernel/sched/ghost_abi_60' - copy kernel/sched/ghost.[hc] and include/uapi/kernel/ghost.h to this new directory. - create kernel/sched/ghost_abi_60/Makefile to compile ghost.c - update kernel/sched/Makefile to descend into 'ghost_abi_60' - bump up GHOST_VERSION in include/uapi/kernel/ghost.h as the snapshot also marks the beginning of a new ABI. Deleting a snapshot involves the following: - remove reference to 'ghost_abi_60' from kernel/sched/Makefile - delete 'kernel/sched/ghost_abi_60' The script should be executed from the top level kernel directory: $ kernel/sched/ghost_abi_helper.sh -h Usage: ghost_abi_helper.sh [-hs][-d ] -h: print this help message -s: snapshot abi (implicit if no other option is specified) -d: delete snapshot at the specified version $ kernel/sched/ghost_abi_helper.sh -s $ kernel/sched/ghost_abi_helper.sh -d 60 Tested: created a couple of snapshots and deleted them. Effort: sched/ghost/abi Change-Id: I2b541240bcf022cfdff724377dbcefd6f5ad4c05 --- kernel/sched/ghost_abi_helper.sh | 106 +++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 kernel/sched/ghost_abi_helper.sh diff --git a/kernel/sched/ghost_abi_helper.sh b/kernel/sched/ghost_abi_helper.sh new file mode 100755 index 000000000000..223038ef97d9 --- /dev/null +++ b/kernel/sched/ghost_abi_helper.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +progname=`basename $0` + +err_exit() { + local msg=$1 + echo "*** ${msg}" 1>&2 + exit 1 +} + +usage() { + local rc=$1 + echo "Usage: $progname [-hs][-d ]" + echo " -h: print this help message" + echo " -s: snapshot abi (implicit if no other option is specified)" + echo " -d: delete snapshot at the specified version" + exit $rc +} + +check_topdir() { + local topdir=$1 + + if [ -d ${topdir}/kernel/sched -a \ + -f ${topdir}/kernel/sched/ghost.c -a \ + -f ${topdir}/kernel/sched/ghost.h -a \ + -f ${topdir}/include/uapi/linux/ghost.h ]; then + return 0 + else + return -1 + fi +} + +delete_snapshot=0 +abi=0 + +while getopts "hsd:" opt; do + case $opt in + d) + abi="${OPTARG}" + delete_snapshot=1 + ;; + s) + delete_snapshot=0 + ;; + h) + usage 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + ;; + esac +done + +shift "$((OPTIND - 1))" + +if [ $# -gt 0 ]; then + usage 1 +fi + +topdir=${PWD} +check_topdir ${topdir} || err_exit "Are you in the top level kernel directory?" + +sched_dir=${topdir}/kernel/sched +uapi_dir=${topdir}/include/uapi/linux + +if [ $delete_snapshot -eq 0 ]; then + abi=`grep GHOST_VERSION ${uapi_dir}/ghost.h | awk '{print $3}'` + if [ $? -ne 0 -o -z "${abi}" ]; then + err_exit "unable to parse GHOST_VERSION from ghost.h" + fi +fi + +snapshot_dir=${sched_dir}/ghost_abi_${abi} + +if [ ${delete_snapshot} -eq 1 ]; then + git rm -r ${snapshot_dir} || err_exit "error git rm'ing ${snapshot_dir}" + sed -i "/ghost_abi_${abi}/d" ${sched_dir}/Makefile || \ + err_exit "error removing ghost_abi_${abi} from Makefile" + rm -rf ${snapshot_dir} + git commit -a -m "sched: delete snapshot of ghost abi ${abi}" + exit $? +fi + +# Take a snapshot of the ABI. +mkdir ${snapshot_dir} || err_exit "error creating ${snapshot_dir}" +cp ${sched_dir}/ghost.h ${snapshot_dir} || err_exit "error creating ghost.h" +cp ${sched_dir}/ghost.c ${snapshot_dir} || err_exit "error creating ghost.c" +cp ${uapi_dir}/ghost.h ${snapshot_dir}/ghost_uapi.h || err_exit "error creating ghost_uapi.h" + +cat > ${snapshot_dir}/Makefile << EOF +obj-\$(CONFIG_SCHED_CLASS_GHOST) += ghost.o +CFLAGS_ghost.o += -I\$(srctree)/fs/kernfs -I\$(srctree)/kernel/sched +EOF + +cat >> ${sched_dir}/Makefile << EOF +obj-\$(CONFIG_SCHED_CLASS_GHOST) += ghost_abi_${abi}/ +EOF + +sed -i -e "s/^#define\s\+GHOST_VERSION\s\+[0-9]\+$/#define GHOST_VERSION $((abi+1))/" \ + ${uapi_dir}/ghost.h || err_exit "error bumping current abi version" + +git add ${snapshot_dir} ${sched_dir}/Makefile ${uapi_dir}/ghost.h || \ + err_exit "git add" + +git commit -m "sched: snapshot ghost abi ${abi}" +exit $? From a33088314ffbb5928f427b6fa346e8bc8eac9ea9 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 2 Dec 2021 10:10:02 -0800 Subject: [PATCH 126/165] sched: simplify for_each_abi() We don't need a pointer to the 'last_ghost_abi' anymore (it was used earlier in the patchset to WARN if there more than one ABI was present). Pointed out by: brho@ Tested: root@(none):~# cat /sys/fs/ghost/version 64 61 62 63 Effort: sched/ghost/abi Change-Id: I8e1a0d6f2ca2dc824a54714826a8c826d750287d --- kernel/sched/ghost_core.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 7b0af3396644..d7b7334c0f9a 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -11,11 +11,8 @@ static struct kernfs_root *ghost_kfs_root; extern const struct ghost_abi __begin_ghost_abi[]; extern const struct ghost_abi __end_ghost_abi[]; -#define first_ghost_abi (&__begin_ghost_abi[0]) -#define last_ghost_abi (&__end_ghost_abi[-1]) - #define for_each_abi(abi) \ - for (abi = first_ghost_abi; abi <= last_ghost_abi; abi++) + for (abi = __begin_ghost_abi; abi < __end_ghost_abi; abi++) /* * We do not want to make 'SG_COOKIE_CPU_BITS' larger than necessary so that From 21800d022acc51ec5e5692fdba0829fc4d826541 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 2 Dec 2021 10:32:52 -0800 Subject: [PATCH 127/165] sched: deprecate ghost_abi_ptr_t Requested by: brho@ (also coding style frowns upon it) Tested: builds Effort: sched/ghost/abi Change-Id: I52a82fc2e3209258cf78597e78b20bcdbc9751f2 --- kernel/sched/ghost.c | 4 ++-- kernel/sched/ghost_core.c | 16 ++++++++-------- kernel/sched/sched.h | 6 ++---- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 119b51408996..d791a3649649 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6943,7 +6943,7 @@ static void runtime_adjust_dirtabs(void) enc_txn->size = GHOST_CPU_DATA_REGION_SIZE; } -static int __init abi_init(ghost_abi_ptr_t abi) +static int __init abi_init(const struct ghost_abi *abi) { /* * ghost bpf programs encode the ABI they were compiled against @@ -6962,7 +6962,7 @@ static int __init abi_init(ghost_abi_ptr_t abi) return 0; } -static struct ghost_enclave *create_enclave(ghost_abi_ptr_t abi, +static struct ghost_enclave *create_enclave(const struct ghost_abi *abi, struct kernfs_node *dir, ulong id) { diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index d7b7334c0f9a..1cf1f6a5acca 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -45,9 +45,9 @@ static void gf_strip_slash_n(char *buf, size_t count) *n = '\0'; } -static ghost_abi_ptr_t ghost_abi_lookup(uint version) +static const struct ghost_abi *ghost_abi_lookup(uint version) { - ghost_abi_ptr_t abi; + const struct ghost_abi *abi; for_each_abi(abi) { if (abi->version == version) @@ -62,7 +62,7 @@ static int make_enclave(struct kernfs_node *parent, unsigned long id, { struct kernfs_node *dir; struct ghost_enclave *e; - ghost_abi_ptr_t abi; + const struct ghost_abi *abi; char name[31]; abi = ghost_abi_lookup(version); @@ -126,7 +126,7 @@ static struct kernfs_ops gf_ops_top_ctl = { static int gf_top_version_show(struct seq_file *sf, void *v) { - ghost_abi_ptr_t abi; + const struct ghost_abi *abi; for_each_abi(abi) seq_printf(sf, "%u\n", abi->version); @@ -175,7 +175,7 @@ static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, static void __init ghost_abi_init(void) { - ghost_abi_ptr_t abi; + const struct ghost_abi *abi; int ret; for_each_abi(abi) { @@ -466,10 +466,10 @@ void __init init_sched_ghost_class(void) */ struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) { - ghost_abi_ptr_t abi; + const struct ghost_abi *abi; struct ghost_enclave *e = NULL; - static ghost_abi_ptr_t last_abi; + static const struct ghost_abi *last_abi; *fd_to_put = fdget(fd); if (!fd_to_put->file) @@ -1450,7 +1450,7 @@ static bool ghost_msg_is_valid_access(int off, int size, struct bpf_insn_access_aux *info) { int version = bpf_prog_eat_abi(prog); - ghost_abi_ptr_t abi = ghost_abi_lookup(version); + const struct ghost_abi *abi = ghost_abi_lookup(version); if (WARN_ON_ONCE(!abi)) return false; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index eefad5e9b566..67f729ac1fab 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2180,13 +2180,11 @@ typedef int64_t gtid_t; #define GHOST_TID_SEQNUM_BITS 41 #define GHOST_TID_PID_BITS 22 -typedef const struct ghost_abi *ghost_abi_ptr_t; - struct ghost_abi { int version; - int (*abi_init)(ghost_abi_ptr_t abi); + int (*abi_init)(const struct ghost_abi *abi); struct ghost_enclave * - (*create_enclave)(ghost_abi_ptr_t abi, + (*create_enclave)(const struct ghost_abi *abi, struct kernfs_node *dir, ulong id); void (*enclave_release)(struct kref *k); struct ghost_enclave *(*ctlfd_enclave_get)(struct file *file); From 110db8b5869546c9e454714dabbd0c81253eb6f7 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Wed, 15 Dec 2021 10:47:48 -0800 Subject: [PATCH 128/165] sched: add _ghost_resolve_enclave(rq, p) helper Requested by: brho@ Tested: roll up testing of the abi-selection patchset All unit tests pass in virtme kokonut presubmit: jtbjc11 http://sponge2/68d5cfb6-31f8-4f09-a24c-34a3789a5b87 jtbjc16 http://sponge2/256b491c-473e-45f1-8bd7-e7caaf6a891f qkhw13 http://sponge2/8ff6470e-1bf5-4c0e-a76d-0767ce3a4e24 otit12 http://sponge2/9e7f657b-c572-468b-8d28-441985be5aea VIT: bwaves rate/time - 4.15.0-smp-914.30.0.0 (kernel 2LCE) http://sponge2/db663210-0e73-4fdf-8fbf-7ff58d438cf1 464/935 - development/legos http://sponge2/d2b0b8c2-4464-4ca7-b5d0-66e99a748b5f 467/930 - this commit http://sponge2/d354a9fb-f697-4f18-93d0-56d0ee81ac61 466/932 SLL: - development/legos https://screenshot.googleplex.com/9bVPebNjRWESgc4 - this commit https://screenshot.googleplex.com/BWdYKVhbKhecnbM Effort: sched/ghost/abi Change-Id: Ib5a6753148294fb74cbe4793a307270e3063edd5 --- kernel/sched/ghost_core.c | 242 ++++++-------------------------------- 1 file changed, 33 insertions(+), 209 deletions(-) diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 1cf1f6a5acca..5a4dd0342958 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -416,29 +416,43 @@ void ghost_wait_for_rendezvous(struct rq *rq) e->abi->wait_for_rendezvous(rq); } -void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev) +static struct ghost_enclave *_ghost_resolve_enclave(struct rq *rq, + struct task_struct *p) { - struct ghost_enclave *this_enclave, *prev_enclave, *e; + struct ghost_enclave *rq_enclave, *task_enclave; VM_BUG_ON(preemptible()); - VM_BUG_ON(rq != this_rq()); - /* rcu_read_lock_sched() not needed; preemption is disabled. */ - this_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - prev_enclave = prev->ghost.enclave; + /* Implicit read-side critical section due to disabled preemption */ + rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); + + /* + * Caller is responsible for ensuring stability of 'p->ghost.enclave'. + * This usually happens as a side-effect of holding 'rq->lock' of the + * cpu where the task is queued or running. + */ + task_enclave = p->ghost.enclave; + WARN_ON_ONCE(!task_enclave && ghost_class(p->sched_class)); /* * XXX it is possible for a running task to enter ghost into enclave X * while the cpu it is running on belongs to enclave Y. This is not * supported properly yet hence the VM_BUG_ON. */ - VM_BUG_ON(this_enclave && prev_enclave && this_enclave != prev_enclave); + VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); /* - * 'this_enclave' may be NULL if a running task enters ghost on a cpu + * 'rq_enclave' may be NULL if a running task enters ghost on a cpu * that does not belong to any enclave. */ - e = this_enclave ? this_enclave : prev_enclave; + return rq_enclave ? rq_enclave : task_enclave; +} + +void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev) +{ + struct ghost_enclave *e = _ghost_resolve_enclave(rq, prev); + + VM_BUG_ON(rq != this_rq()); if (e != NULL) e->abi->pnt_prologue(rq, prev); @@ -586,28 +600,10 @@ int ghost_setscheduler(struct task_struct *p, struct rq *rq, void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { - struct ghost_enclave *this_enclave, *prev_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, prev); - VM_BUG_ON(preemptible()); VM_BUG_ON(rq != this_rq()); - /* rcu_read_lock_sched() not needed; preemption is disabled. */ - this_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - prev_enclave = prev->ghost.enclave; - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(this_enclave && prev_enclave && this_enclave != prev_enclave); - - /* - * 'this_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = this_enclave ? this_enclave : prev_enclave; - if (e != NULL) e->abi->prepare_task_switch(rq, prev, next); } @@ -859,30 +855,10 @@ DEFINE_SCHED_CLASS(ghost_agent) = { static void update_curr_ghost(struct rq *rq) { - struct ghost_enclave *rq_enclave, *curr_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, rq->curr); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - if (ghost_class(rq->curr->sched_class)) - curr_enclave = rq->curr->ghost.enclave; - else - curr_enclave = NULL; - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && curr_enclave && rq_enclave != curr_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : curr_enclave; if (e) e->abi->update_curr(rq); } @@ -951,29 +927,10 @@ static void task_dead_ghost(struct task_struct *p) static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->dequeue_task(rq, p, flags); else @@ -983,29 +940,10 @@ static void dequeue_task_ghost(struct rq *rq, struct task_struct *p, int flags) static void put_prev_task_ghost(struct rq *rq, struct task_struct *p) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->put_prev_task(rq, p); else @@ -1015,29 +953,10 @@ static void put_prev_task_ghost(struct rq *rq, struct task_struct *p) static void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->enqueue_task(rq, p, flags); else @@ -1047,29 +966,10 @@ static void enqueue_task_ghost(struct rq *rq, struct task_struct *p, int flags) static void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->set_next_task(rq, p, first); else @@ -1083,29 +983,10 @@ static void set_next_task_ghost(struct rq *rq, struct task_struct *p, bool first */ static void task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->task_tick(rq, p, queued); else @@ -1134,29 +1015,10 @@ static struct task_struct *pick_next_task_ghost(struct rq *rq) static void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, int wake_flags) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->check_preempt_curr(rq, p, wake_flags); else @@ -1167,30 +1029,11 @@ static void check_preempt_curr_ghost(struct rq *rq, struct task_struct *p, static void yield_task_ghost(struct rq *rq) { struct task_struct *p = current; - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); VM_BUG_ON(rq != this_rq()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a running task enters ghost on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->yield_task(rq); else @@ -1230,29 +1073,10 @@ static int balance_ghost(struct rq *rq, struct task_struct *prev, void task_woken_ghost(struct rq *rq, struct task_struct *p) { - struct ghost_enclave *rq_enclave, *task_enclave, *e; + struct ghost_enclave *e = _ghost_resolve_enclave(rq, p); - VM_BUG_ON(preemptible()); lockdep_assert_held(&rq->lock); - /* Implicit read-side critical section due to disabled preemption */ - rq_enclave = rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))); - - task_enclave = p->ghost.enclave; - WARN_ON_ONCE(!task_enclave); - - /* - * XXX it is possible for a running task to enter ghost into enclave X - * while the cpu it is running on belongs to enclave Y. This is not - * supported properly yet hence the VM_BUG_ON. - */ - VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); - - /* - * 'rq_enclave' may be NULL if a task is woken up on a cpu - * that does not belong to any enclave. - */ - e = rq_enclave ? rq_enclave : task_enclave; if (e) e->abi->task_woken(rq, p); else From 6304d49cd9e6e3f6e139381ec6a892b2366a868a Mon Sep 17 00:00:00 2001 From: Jack Humphries Date: Thu, 6 Jan 2022 10:19:04 -0800 Subject: [PATCH 129/165] sched: Add GHOST_BPF macro to include/uapi/linux/bpf.h When compiling externally on a machine running an open source ghOSt kernel, the definitions in bpf/user/agent.h cause the compilation to fail because the definitions for the same types also exist in the Linux headers installed on the machine (so the definitions in bpf/user/agent.h are *redefinitions*). Thus, to fix this, we add the GHOST_BPF macro to include/uapi/linux/bpf.h. We check for this macro in the userspace code. If the macro exists, we do not redefine the ghOSt kernel BPF types. Otherwise, we do. This KCL will be submitted concurrently with cl/419047752. Tested: Compiles. Effort: sched/ghost Change-Id: Ib9ad80fda552f1446d000a9fc8ca73e7aeaaddce --- include/uapi/linux/bpf.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 77769e679bb6..92540b713473 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5213,6 +5213,8 @@ enum { BTF_F_ZERO = (1ULL << 3), }; +#define GHOST_BPF + struct bpf_ghost_sched { }; From fe2fddfc7d455e0b3fd3a2f660a414c416592d85 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 13:49:02 -0500 Subject: [PATCH 130/165] sched/ghost: fix kernfs hang with ghost_run If you had a satellite agent blocked_in_run in ghost_run or via local txn, and then you try to destroy the enclave from ghostfs, the write(ctl, "destroy") would hang. The destroyer would be stuck on kernfs's "active protection", which essentially waits until all syscalls in-progress are complete before removing a file (e.g. ctl) from kernfs. To fix this, we just need to drop active protection. The only thing we used active protection for is to hold a kref on the enclave (open ghostfs FDs hold a kref via "active protection"), which we can do manually for ioctl. Doing this for all ioctls is cleaner than doing it for just ghost_run. Especially when you consider commit_txn also can block_in_run. Safer to do it for all ioctls. We could do this for most any ghostfs operation, and need to do it for any ghostfs operation that can block for arbitrarily long (i.e. not just a page fault). Though for files that are mmaped (cpu_data, swrs), I'd be a little careful, since active protection also keeps the mmaps around. Arguably, it doesn't matter much, since if we're removing those files, it's because the enclave is being destroyed, and the agents would get SIGKILLed anyways. Tested: agent_exp, echo destroy > enclave_1/ctl Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ie5d27bd78af124e020f37d410cfabfeab80ace3e --- kernel/sched/ghost.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index d791a3649649..ec2b803cc026 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6572,13 +6572,9 @@ static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, return len; } -static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, - unsigned long arg) +static long enclave_ioctl(struct ghost_enclave *e, unsigned int cmd, + unsigned long arg) { - struct ghost_enclave *e = of_to_e(of); - - if (!(of->file->f_mode & FMODE_WRITE)) - return -EACCES; switch (cmd) { case GHOST_IOC_NULL: return 0; @@ -6617,6 +6613,35 @@ static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, return -ENOIOCTLCMD; } +static long gf_ctl_ioctl(struct kernfs_open_file *of, unsigned int cmd, + unsigned long arg) +{ + struct ghost_enclave *e = of_to_e(of); + long ret; + + if (!(of->file->f_mode & FMODE_WRITE)) + return -EACCES; + + /* + * Active protection prevents gf_e_release from being called (and any + * mmaps from being unmapped, though that doesn't apply to ctl). Active + * protection prevents the kn from being drained/deleted while we are + * operating on it. It is meant for syscalls in progress, not held + * forever. Some of these ioctls, e.g. ghost_run, can block for + * arbitrarily long. + * + * The only thing we use the kn for is to get a stable reference to e. + * We can just up the kref on our own, and ignore the kn. + */ + kref_get(&e->kref); + kernfs_break_active_protection(of->kn); + ret = enclave_ioctl(e, cmd, arg); + kernfs_unbreak_active_protection(of->kn); + kref_put(&e->kref, enclave_release); + + return ret; +} + static struct kernfs_ops gf_ops_e_ctl = { .open = gf_e_open, .release = gf_e_release, From f891d034ca78eedf42eda492ac77d827f73b0efb Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 14:10:16 -0500 Subject: [PATCH 131/165] sched/ghost: fix never-exiting agents during destroy If you had an agent task attached to an enclave's cpu and you destroyed the enclave (e.g. via ghostfs), the agent task would never exit, and the cpu would be unusable in other enclaves. The root of the issue is that we would "unpublish" the enclave, meaning pcpu->enclave = NULL, but the cpu would still be "owned" by the enclave until the agent exited. I did this because it was a pain to synchronize_rcu() in release_from_ghost(), so it was simpler to clear pcpu->enclave and synchronize_rcu() in ghost_destroy_enclave(), which we have to do anyways, and then let the agent clean itself up later. The problem is that the ABI selection uses pcpu->enclave to know which functions to run. In this case, we'd set pcpu->enclave = NULL, but the agent hadn't run yet. It'd get SIGKILLed, but since enclave == NULL, PNT wouldn't know *which ABI's* PNT to run, and so we wouldn't run PNT at all. Thus the agent would never wake up -> _task_dead_ghost -> release_from_ghost -> return the cpu. The fix is to get rid of the split between "publishing" a cpu (mostly setting pcpu->enclave) and "claiming/returning" a cpu (the cpu_owner business). The reason for the split was because synchronize_rcu was a pain from release_task_ghost, but we can piggyback on call_rcu() from _task_dead_ghost. The main concern is that we do three things: 1) pcpu->enclave = NULL 2) sync rcu 3) decref the enclave call_rcu handles 2 and 3. I piggybacked on ghost_delayed_put_task_struct() since I stashed the enclave pointer in p->ghost. Note that __enclave_unpublish_cpu() is always immediately followed by __enclave_return_cpu(). I'll deal with that in a later commit. Tested: agent_exp, echo destroy > enclave_1/ctl Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I35c7814eb8bc6b7459897c6e7370170aacadcaf5 --- include/linux/sched.h | 2 ++ kernel/sched/ghost.c | 59 ++++++++++++++++++++++++++++++++----------- kernel/sched/sched.h | 1 - 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 0764aac288bc..d5a35bfe0680 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -597,6 +597,8 @@ struct sched_ghost_entity { struct ghost_queue *dst_q; struct ghost_status_word *status_word; struct ghost_enclave *enclave; + /* See ghost_destroy_enclave() */ + struct ghost_enclave *__agent_decref_enclave; /* * See also ghost_prepare_task_switch() and ghost_deferred_msgs() diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index ec2b803cc026..15ed5980fa24 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1994,17 +1994,24 @@ static void ghost_destroy_enclave(struct ghost_enclave *e) * give the cpu back to the system right away, we don't have to worry * about anyone reusing the rq's agent fields. * - * However, we *do* want to "unpublish" the assignment of the cpu to the - * enclave. This way, once we synchronize_rcu, no one can be using this - * cpu. We could defer that until release_from_ghost(), but that is not - * a convenient place to call synchronize_rcu() (holding RQ lock). + * Note that pcpu->enclave is still set until the agent task exits a + * given cpu. We must follow the pattern of: + * 1) pcpu->enclave = NULL + * 2) synchronize_rcu() + * 3) decref the enclave + * because pcpu->enclave is protected and kreffable during + * rcu_read_lock(). * - * In essence, __enclave_remove_cpu() is split: we unpublish the cpu - * here, then return it to the system when the agent leaves ghost. Note - * the agent could be leaving ghost of its own volition concurrently; - * the enclave lock ensures that if we see an agent, it will remove the - * cpu. And if not, we fully remove it here. We make that decision - * before we unlock in the loop below. + * All three steps are handled in release_from_ghost(). We tell the + * agent to do this via agent->ghost.__agent_decref_enclave. This needs + * to be per-agent, and not per-rq, since the moment we return the cpu + * to the system, another enclave and agent can reuse the fields in + * rq->ghost. + * + * Note the agent could be leaving ghost of its own volition + * concurrently; the enclave lock ensures that if we see an agent, it + * will remove the cpu when it exits. And if not, we fully remove it + * here. We make that decision before we unlock in the loop below. * * Why do we unlock? Because send_sig grabs the pi_lock, and the lock * ordering is pi -> rq -> e. This is safe, since the lock is no longer @@ -2022,8 +2029,9 @@ static void ghost_destroy_enclave(struct ghost_enclave *e) __enclave_remove_cpu(e, cpu); continue; } - __enclave_unpublish_cpu(e, cpu); - rq->ghost.agent_remove_enclave_cpu = true; + agent = rq->ghost.agent; + kref_get(&e->kref); + agent->ghost.__agent_decref_enclave = e; /* * We can't force_sig(). The task might be exiting and losing * its sighand_struct. send_sig_info() checks for that. We @@ -2034,7 +2042,6 @@ static void ghost_destroy_enclave(struct ghost_enclave *e) * might be dying on its own, we need to get a refcount while we * poke it. */ - agent = rq->ghost.agent; get_task_struct(agent); spin_unlock_irqrestore(&e->lock, flags); send_sig_info(SIGKILL, SEND_SIG_PRIV, agent); @@ -4294,9 +4301,23 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) ghost_claim_and_kill_txn(rq->cpu, GHOST_TXN_NO_AGENT); /* See ghost_destroy_enclave() */ - if (rq->ghost.agent_remove_enclave_cpu) { - rq->ghost.agent_remove_enclave_cpu = false; + if (p->ghost.__agent_decref_enclave) { + /* Agents should only exit, never setsched out. */ + WARN_ON_ONCE(p->state != TASK_DEAD); + VM_BUG_ON(p->ghost.__agent_decref_enclave != e); + __enclave_unpublish_cpu(e, rq->cpu); __enclave_return_cpu(e, rq->cpu); + /* + * At this moment, this cpu can be added to a new + * enclave (or our previous enclave!). + * + * Additionally, once we unlock the rq, a new agent task + * can attach to this cpu and use the rq->ghost fields. + * + * We've set pcpu->enclave = NULL (step 1). Steps 2 and + * 3 (sync rcu and decref) are handled when we return to + * _task_dead_ghost() call_rcu. + */ } } @@ -4322,6 +4343,14 @@ static void ghost_delayed_put_task_struct(struct rcu_head *rhp) { struct task_struct *tsk = container_of(rhp, struct task_struct, ghost.rcu); + /* + * Step 3 of "destroyed with agent": decref. + * See ghost_destroy_enclave(). + */ + if (tsk->ghost.__agent_decref_enclave) { + kref_put(&tsk->ghost.__agent_decref_enclave->kref, + enclave_release); + } put_task_struct(tsk); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 67f729ac1fab..d1669a8af15b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -105,7 +105,6 @@ extern void call_trace_sched_update_nr_running(struct rq *rq, int count); struct ghost_rq { struct task_struct *agent; /* protected by e->lock and rq->lock */ uint32_t agent_barrier; - bool agent_remove_enclave_cpu; /* protected by e->lock */ bool blocked_in_run; /* agent is blocked in 'ghost_run()' */ bool agent_should_wake; bool must_resched; /* rq->curr must reschedule in PNT */ From 13696d3d2ae86e3768eefbedc19b7276e5d494be Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 14:27:52 -0500 Subject: [PATCH 132/165] sched/ghost: merge unpublish_cpu and return_cpu No need to split unpublish_cpu and return_cpu anymore. Combine them into remove_cpu. Tested: edf_test, enclave_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I1cfe101afaf1a3989425c8512a8fa025ac64fae0 --- kernel/sched/ghost.c | 34 ++++++++-------------------------- kernel/sched/ghost_core.c | 16 +--------------- kernel/sched/sched.h | 3 +-- 3 files changed, 10 insertions(+), 43 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 15ed5980fa24..db103d61ce43 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1815,32 +1815,15 @@ static void __enclave_add_cpu(struct ghost_enclave *e, int cpu) ghost_publish_cpu(e, cpu); } -/* Hold e->lock. Caller must synchronize_rcu(). */ -static void __enclave_unpublish_cpu(struct ghost_enclave *e, int cpu) -{ - VM_BUG_ON(!spin_is_locked(&e->lock)); - - rcu_assign_pointer(per_cpu(ghost_txn, cpu), NULL); - ghost_unpublish_cpu(e, cpu); -} - -/* Caller must hold e->lock */ -static void __enclave_return_cpu(struct ghost_enclave *e, int cpu) +/* Caller must hold e->lock and synchronize_rcu() on return */ +static void __enclave_remove_cpu(struct ghost_enclave *e, int cpu) { VM_BUG_ON(!spin_is_locked(&e->lock)); VM_BUG_ON(!cpumask_test_cpu(cpu, &e->cpus)); + rcu_assign_pointer(per_cpu(ghost_txn, cpu), NULL); cpumask_clear_cpu(cpu, &e->cpus); - ghost_return_cpu(e, cpu); -} - -/* Caller must hold e->lock and synchronize_rcu() on return */ -static void __enclave_remove_cpu(struct ghost_enclave *e, int cpu) -{ - __enclave_unpublish_cpu(e, cpu); - /* cpu is no longer participating in ghost scheduling */ - __enclave_return_cpu(e, cpu); - /* cpu can now be assigned to another enclave */ + ghost_remove_cpu(e, cpu); } /* @@ -2050,7 +2033,7 @@ static void ghost_destroy_enclave(struct ghost_enclave *e) } spin_unlock_irqrestore(&e->lock, flags); - synchronize_rcu(); /* Required after unpublishing a cpu */ + synchronize_rcu(); /* Required after removing a cpu */ /* * It is safe to reap all tasks in the enclave only _after_ @@ -2142,7 +2125,7 @@ static int ghost_enclave_set_cpus(struct ghost_enclave *e, out_e: spin_unlock_irqrestore(&e->lock, flags); - synchronize_rcu(); /* Required after unpublishing a cpu */ + synchronize_rcu(); /* Required after removing a cpu */ free_cpumask_var(add); free_cpumask_var(del); @@ -4305,8 +4288,7 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) /* Agents should only exit, never setsched out. */ WARN_ON_ONCE(p->state != TASK_DEAD); VM_BUG_ON(p->ghost.__agent_decref_enclave != e); - __enclave_unpublish_cpu(e, rq->cpu); - __enclave_return_cpu(e, rq->cpu); + __enclave_remove_cpu(e, rq->cpu); /* * At this moment, this cpu can be added to a new * enclave (or our previous enclave!). @@ -5622,7 +5604,7 @@ void ghost_commit_all_greedy_txns(void) * Note that e's cpu mask could be changed concurrently, with cpus added * or removed. This is benign. First, any commits will look at the * rcu-protected ghost_txn pointer. That's what really matters, and - * any caller to __enclave_unpublish_cpu() will synchronize_rcu(). + * any caller to __enclave_remove_cpu() will synchronize_rcu(). * * Furthermore, if a cpu is added while we are looking (which is not * protected by RCU), it's not a big deal. This is a greedy commit, and diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 5a4dd0342958..82bf26f86831 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -356,28 +356,14 @@ void ghost_publish_cpu(struct ghost_enclave *e, int cpu) spin_unlock(&cpu_rsvp); } -void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu) +void ghost_remove_cpu(struct ghost_enclave *e, int cpu) { - /* - * 'e->lock' must be held across ghost_unpublish_cpu() and the - * corresponding ghost_return_cpu(). - */ VM_BUG_ON(!spin_is_locked(&e->lock)); spin_lock(&cpu_rsvp); WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); WARN_ON_ONCE(per_cpu(enclave, cpu) != e); rcu_assign_pointer(per_cpu(enclave, cpu), NULL); - spin_unlock(&cpu_rsvp); -} - -void ghost_return_cpu(struct ghost_enclave *e, int cpu) -{ - VM_BUG_ON(!spin_is_locked(&e->lock)); - - spin_lock(&cpu_rsvp); - WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); - WARN_ON_ONCE(per_cpu(enclave, cpu) != NULL); per_cpu(cpu_owner, cpu) = NULL; spin_unlock(&cpu_rsvp); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d1669a8af15b..228012318640 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2264,8 +2264,7 @@ _GHOST_MAYBE_CONST DECLARE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); void init_sched_ghost_class(void); int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *cpus); void ghost_publish_cpu(struct ghost_enclave *e, int cpu); -void ghost_unpublish_cpu(struct ghost_enclave *e, int cpu); -void ghost_return_cpu(struct ghost_enclave *e, int cpu); +void ghost_remove_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); void ghost_wait_for_rendezvous(struct rq *rq); From 1c5694b7e6358a47611a0f78e2448d2a2d172f1d Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 15:33:37 -0500 Subject: [PATCH 133/165] sched/ghost: merge claim_cpu and publish_cpu No need to split claim_cpu and publish_cpu anymore. ghost_add_cpus() can do them both, with a callback to the ABI's work that now runs with the cpu_rsvp lock held. Tested: edf_test, enclave_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I5b9bc8109549e2e5f9deecd1522047279defb3ef --- kernel/sched/ghost.c | 16 ++++------------ kernel/sched/ghost_core.c | 24 ++++++++---------------- kernel/sched/sched.h | 4 ++-- 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index db103d61ce43..2d14b2a936b7 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -1783,10 +1783,9 @@ static inline void queue_incref(struct ghost_queue *q) } /* - * Caller must hold e->lock and 'cpu' must have already been claimed - * on behalf of the enclave via ghost_claim_cpus(). + * Callback from ghost_add_cpus(). Caller must hold e->lock. */ -static void __enclave_add_cpu(struct ghost_enclave *e, int cpu) +static void ___enclave_add_cpu(struct ghost_enclave *e, int cpu) { struct ghost_txn *txn; @@ -1807,12 +1806,6 @@ static void __enclave_add_cpu(struct ghost_enclave *e, int cpu) cpumask_set_cpu(cpu, &e->cpus); rcu_assign_pointer(per_cpu(ghost_txn, cpu), txn); - - /* - * We have already claimed 'cpu' for this enclave so now publish - * it to the rest of the kernel via the per_cpu(enclave) pointer. - */ - ghost_publish_cpu(e, cpu); } /* Caller must hold e->lock and synchronize_rcu() on return */ @@ -2113,12 +2106,10 @@ static int ghost_enclave_set_cpus(struct ghost_enclave *e, } } - ret = ghost_claim_cpus(e, add); + ret = ghost_add_cpus(e, add); if (ret) goto out_e; - for_each_cpu(cpu, add) - __enclave_add_cpu(e, cpu); for_each_cpu(cpu, del) __enclave_remove_cpu(e, cpu); @@ -7386,6 +7377,7 @@ DEFINE_GHOST_ABI(current_abi) = { .abi_init = abi_init, .create_enclave = create_enclave, .enclave_release = enclave_release, + .enclave_add_cpu = ___enclave_add_cpu, .ctlfd_enclave_get = ctlfd_enclave_get, .ctlfd_enclave_put = ctlfd_enclave_put, .setscheduler = _ghost_setscheduler, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 82bf26f86831..7f201b74caae 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -314,14 +314,11 @@ static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, cpu_owner); DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); -int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *new_cpus) +/* Caller holds e->lock. Either all cpus are added, or none are. */ +int ghost_add_cpus(struct ghost_enclave *e, const struct cpumask *new_cpus) { int cpu; - /* - * 'e->lock' must be held across ghost_claim_cpus() and the - * corresponding ghost_publish_cpu(). - */ VM_BUG_ON(!spin_is_locked(&e->lock)); spin_lock(&cpu_rsvp); @@ -339,21 +336,16 @@ int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *new_cpus) } for_each_cpu(cpu, new_cpus) - per_cpu(cpu_owner, cpu) = e; + e->abi->enclave_add_cpu(e, cpu); - spin_unlock(&cpu_rsvp); - return 0; -} + for_each_cpu(cpu, new_cpus) + per_cpu(cpu_owner, cpu) = e; -void ghost_publish_cpu(struct ghost_enclave *e, int cpu) -{ - VM_BUG_ON(!spin_is_locked(&e->lock)); + for_each_cpu(cpu, new_cpus) + rcu_assign_pointer(per_cpu(enclave, cpu), e); - spin_lock(&cpu_rsvp); - WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); - WARN_ON_ONCE(per_cpu(enclave, cpu) != NULL); - rcu_assign_pointer(per_cpu(enclave, cpu), e); spin_unlock(&cpu_rsvp); + return 0; } void ghost_remove_cpu(struct ghost_enclave *e, int cpu) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 228012318640..d44e89189888 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2186,6 +2186,7 @@ struct ghost_abi { (*create_enclave)(const struct ghost_abi *abi, struct kernfs_node *dir, ulong id); void (*enclave_release)(struct kref *k); + void (*enclave_add_cpu)(struct ghost_enclave *e, int cpu); struct ghost_enclave *(*ctlfd_enclave_get)(struct file *file); void (*ctlfd_enclave_put)(struct file *file); int (*setscheduler)(struct task_struct *p, struct rq *rq, @@ -2262,8 +2263,7 @@ const static struct ghost_abi __##name##_ghost_abi \ _GHOST_MAYBE_CONST DECLARE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); void init_sched_ghost_class(void); -int ghost_claim_cpus(struct ghost_enclave *e, const struct cpumask *cpus); -void ghost_publish_cpu(struct ghost_enclave *e, int cpu); +int ghost_add_cpus(struct ghost_enclave *e, const struct cpumask *cpus); void ghost_remove_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); From e8c8b54af4018a80fab133d66ba2a11fe0cd04c4 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 15:37:10 -0500 Subject: [PATCH 134/165] sched/ghost: remove cpu_owner It's functionally the same as per_cpu(enclave). Tested: edf_test, enclave_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I80de86b8bcb3918b23c1ce9141a304ed55d2ea0d --- kernel/sched/ghost_core.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 7f201b74caae..1b94a67d5611 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -308,10 +308,8 @@ static void __exit ghostfs_exit(void) late_initcall(ghostfs_init); __exitcall(ghostfs_exit); -/* per_cpu(cpu_owner) protected by 'cpu_rsvp' */ +/* per_cpu(enclave) write protected by 'cpu_rsvp' */ static DEFINE_SPINLOCK(cpu_rsvp); -static DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, cpu_owner); - DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); /* Caller holds e->lock. Either all cpus are added, or none are. */ @@ -328,19 +326,15 @@ int ghost_add_cpus(struct ghost_enclave *e, const struct cpumask *new_cpus) * Let's make sure that 'cpu' hasn't been already claimed by * another enclave. */ - if (per_cpu(cpu_owner, cpu)) { + if (per_cpu(enclave, cpu)) { spin_unlock(&cpu_rsvp); return -EBUSY; } - WARN_ON_ONCE(per_cpu(enclave, cpu) != NULL); } for_each_cpu(cpu, new_cpus) e->abi->enclave_add_cpu(e, cpu); - for_each_cpu(cpu, new_cpus) - per_cpu(cpu_owner, cpu) = e; - for_each_cpu(cpu, new_cpus) rcu_assign_pointer(per_cpu(enclave, cpu), e); @@ -353,10 +347,8 @@ void ghost_remove_cpu(struct ghost_enclave *e, int cpu) VM_BUG_ON(!spin_is_locked(&e->lock)); spin_lock(&cpu_rsvp); - WARN_ON_ONCE(per_cpu(cpu_owner, cpu) != e); WARN_ON_ONCE(per_cpu(enclave, cpu) != e); rcu_assign_pointer(per_cpu(enclave, cpu), NULL); - per_cpu(cpu_owner, cpu) = NULL; spin_unlock(&cpu_rsvp); } From f0a062085b4c9ed09eaf8915c26e6cdabeaca532 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 15:50:53 -0500 Subject: [PATCH 135/165] sched/ghost: check txn cpu is in an enclave Since we just checked for an agent task, we should have an enclave. Eventually, we might drop the need for an enclave to have an agent task on all cpus. This will cover us if we ever remove that agent task check. Tested: edf_test, enclave_test, api_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I3eee6303d126404e25fa079be1cc28a4cd12affb --- kernel/sched/ghost.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 2d14b2a936b7..58f49b1c9bca 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -5248,6 +5248,11 @@ static bool _ghost_commit_txn(int run_cpu, bool sync, int64_t rendezvous, goto out; } + if (unlikely(!rcu_dereference_sched(per_cpu(enclave, cpu_of(rq))))) { + state = GHOST_TXN_CPU_UNAVAIL; + goto out; + } + if (unlikely(!txn_commit_allowed(rq, gtid, sync))) { state = GHOST_TXN_NOT_PERMITTED; goto out; From 50abbf3f2b43d61f9093c81ac46f30f679dd9af2 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 11 Jan 2022 16:02:06 -0500 Subject: [PATCH 136/165] sched/ghost: add warnings to catch kernfs_mutex deadlocks There's no way to prevent ghost client tasks from grabbing the kernfs_mutex. It's used in sysfs and other places. If an agent task, responsible for scheduling tasks, ever attempts to grab the kernfs_mutex, there's a chance we'll deadlock. Specifically, the client can hold the mutex and be descheduled. If the agent blocks on acquiring the mutex, it will never schedule the client. In general, ghostfs operations on open FDs don't touch the kernfs_mutex - there might be corner cases I didn't see. But we definitely grab it when opening paths (walking or lookups). The agent doesn't currently open any files from ghostfs from agent tasks, so this warning hasn't fired yet. But at least we'll catch it if it does and avoid a potential source of deadlock. Tested: edf_test, enclave_test, api_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I38422580ccc6b098fd2ee556d83e09f919a467cb --- kernel/sched/ghost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 58f49b1c9bca..26d294980ed3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6206,6 +6206,12 @@ static int gf_e_open(struct kernfs_open_file *of) { struct ghost_enclave *e = of_to_e(of); + /* + * kernfs open can grab kernfs_mutex, which is a potential deadlock + * scenario. agent's should open files from CFS. + */ + WARN_ON_ONCE(is_agent(task_rq(current), current)); + kref_get(&e->kref); return 0; } @@ -6478,6 +6484,12 @@ static int gf_swr_open(struct kernfs_open_file *of) { struct ghost_sw_region *swr = of_to_swr(of); + /* + * kernfs open can grab kernfs_mutex, which is a potential deadlock + * scenario. agent's should open files from CFS. + */ + WARN_ON_ONCE(is_agent(task_rq(current), current)); + kref_get(&swr->enclave->kref); return 0; } From 978181495431d7df850ae189bfefa8a249b05b94 Mon Sep 17 00:00:00 2001 From: Josh Don Date: Thu, 18 Nov 2021 16:47:18 -0800 Subject: [PATCH 137/165] ghost: add extra parameter to ghost_timer_fd This adds an extra field, 'type' to the ghost_timer_fd structure. This allows the agent to distinguish between differnt types of timers that it may queue. The cookie field in the timer_fd then acts as an argument to be interpreted based on the type of timer. Paired with cl/414055464. Tested: In combo with the userspace changes. Effort: sched/ghost Change-Id: I80bbded859b338123b0873fdb91bf554034113f4 --- include/linux/sched.h | 1 + include/uapi/linux/ghost.h | 4 +++- kernel/sched/ghost.c | 9 ++++++--- kernel/sched/ghost_core.c | 2 +- kernel/sched/sched.h | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index d5a35bfe0680..5bc75459cdb5 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -647,6 +647,7 @@ struct sched_ghost_entity { struct __kernel_timerfd_ghost { bool enabled; int cpu; + uint64_t type; uint64_t cookie; }; diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 90beb1631869..8f3c88451136 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 61 +#define GHOST_VERSION 62 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -81,6 +81,7 @@ struct ghost_msg_src { struct timerfd_ghost { int cpu; int flags; + uint64_t type; uint64_t cookie; }; #define TIMERFD_GHOST_ENABLED (1 << 0) @@ -386,6 +387,7 @@ struct ghost_msg_payload_cpu_tick { struct ghost_msg_payload_timer { int cpu; + uint64_t type; uint64_t cookie; }; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 26d294980ed3..fa7df695a18f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3812,7 +3812,8 @@ static inline bool cpu_deliver_msg_tick(struct rq *rq) } /* Returns true if MSG_CPU_TIMER_EXPIRED was produced and false otherwise */ -static inline bool cpu_deliver_timer_expired(struct rq *rq, uint64_t cookie) +static inline bool cpu_deliver_timer_expired(struct rq *rq, uint64_t type, + uint64_t cookie) { struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); struct ghost_msg_payload_timer *payload = &msg->timer; @@ -3822,6 +3823,7 @@ static inline bool cpu_deliver_timer_expired(struct rq *rq, uint64_t cookie) msg->type = MSG_CPU_TIMER_EXPIRED; payload->cpu = cpu_of(rq); + payload->type = type; payload->cookie = cookie; return !produce_for_agent(rq, msg); @@ -6048,6 +6050,7 @@ static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) ktfd_ghost.enabled = timerfd_ghost.flags & TIMERFD_GHOST_ENABLED; ktfd_ghost.cpu = timerfd_ghost.cpu; + ktfd_ghost.type = timerfd_ghost.type; ktfd_ghost.cookie = timerfd_ghost.cookie; ret = do_timerfd_settime(timerfd, flags, &new, &old, &ktfd_ghost); @@ -6060,7 +6063,7 @@ static int ghost_timerfd_settime(struct ghost_ioc_timerfd_settime __user *arg) return ret; } -static void _ghost_timerfd_triggered(int cpu, uint64_t cookie) +static void _ghost_timerfd_triggered(int cpu, uint64_t type, uint64_t cookie) { struct rq *rq; struct rq_flags rf; @@ -6071,7 +6074,7 @@ static void _ghost_timerfd_triggered(int cpu, uint64_t cookie) rq = cpu_rq(cpu); rq_lock_irqsave(rq, &rf); - if (cpu_deliver_timer_expired(rq, cookie)) + if (cpu_deliver_timer_expired(rq, type, cookie)) ghost_wake_agent_on(agent_target_cpu(rq)); rq_unlock_irqrestore(rq, &rf); diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 1b94a67d5611..ea98e16f8d03 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -780,7 +780,7 @@ void ghost_timerfd_triggered(struct __kernel_timerfd_ghost *ktfd) rcu_read_lock(); e = rcu_dereference(per_cpu(enclave, cpu)); if (e) - e->abi->timerfd_triggered(cpu, ktfd->cookie); + e->abi->timerfd_triggered(cpu, ktfd->type, ktfd->cookie); else WARN_ONCE(true, "cpu %d has no enclave", cpu); rcu_read_unlock(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d44e89189888..e62784cf589b 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2205,7 +2205,7 @@ struct ghost_abi { void (*commit_greedy_txn)(int cpu); void (*copy_process_epilogue)(struct task_struct *p); void (*cpu_idle)(struct rq *rq); - void (*timerfd_triggered)(int cpu, uint64_t cookie); + void (*timerfd_triggered)(int cpu, uint64_t type, uint64_t cookie); int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); bool (*ghost_msg_is_valid_access)(int off, int size, From 127e3a8f74da8b7fd9fd1c8ca9ead9cddfd5058d Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 13 Jan 2022 21:57:23 -0800 Subject: [PATCH 138/165] sched: eliminate gratuitous diffs In preparation for merging into sched/next get rid of some gratuitous diffs. Tested: all unit tests pass in virtme. Effort: sched/ghost Change-Id: I87885570714ee0709984ce657c96a0a1655e20e3 --- kernel/sched/core.c | 2 +- kernel/sched/ghost.c | 4 ---- kernel/sched/sched.h | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d5fd788268bb..c3979d21cbe5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4416,7 +4416,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) /* * context_switch - switch to the new MM and the new thread's register state. */ -noinline struct rq * +static noinline struct rq * context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf) { diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index fa7df695a18f..619f9c6405e7 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -80,10 +80,6 @@ static void _ghost_task_preempted(struct rq *rq, struct task_struct *p, static void ghost_task_new(struct rq *rq, struct task_struct *p); -struct rq *context_switch(struct rq *rq, struct task_struct *prev, - struct task_struct *next, struct rq_flags *rf); - -void schedule_callback(struct rq *rq); struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int new_cpu); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e62784cf589b..c208db74ad0e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -888,7 +888,6 @@ static inline long se_weight(struct sched_entity *se) return scale_load_down(se->load.weight); } - static inline bool sched_asym_prefer(int a, int b) { return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b); From 1bfe4ab9b083788e7c918ed17f876154d1f24188 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Fri, 14 Jan 2022 14:05:39 -0800 Subject: [PATCH 139/165] sched: fix arm64 build gbuild -s ARCH=arm64 Tested: amd64 builds Effort: sched/ghost Change-Id: I66b1f122a89fddb41c1065f7e7b16b3357b8028d --- kernel/bpf/syscall.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 73907a4f5c36..ff69d7fc77bd 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -4090,10 +4090,12 @@ static int link_create(union bpf_attr *attr) ret = bpf_xdp_link_attach(attr, prog); break; #endif +#ifdef CONFIG_SCHED_CLASS_GHOST case BPF_PROG_TYPE_GHOST_SCHED: case BPF_PROG_TYPE_GHOST_MSG: ret = ghost_bpf_link_attach(attr, prog); break; +#endif default: ret = -EINVAL; } From 5da05ec77890217e85947ff3573e1480579687d2 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Thu, 27 Jan 2022 11:10:11 -0800 Subject: [PATCH 140/165] sched: relocate 'ghost_rq' to the end of 'struct rq' Adding the 112 byte struct in the middle of 'struct rq' perturbed the cache locality of the following members which exhibited as increased cputime in the SwitchFutex benchmark (especially when the threads are not in the same LLC domain on AMD cpus). Effort: sched/ghost Change-Id: If17714649c9d351e85aa0e759f57b64d29b6adc4 --- kernel/sched/sched.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c208db74ad0e..4109b88c7443 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1076,10 +1076,6 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; struct dl_rq dl; -#ifdef CONFIG_SCHED_CLASS_GHOST - struct ghost_rq ghost; -#endif - #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this CPU: */ struct list_head leaf_cfs_rq_list; @@ -1211,6 +1207,10 @@ struct rq { #endif unsigned int push_busy; struct cpu_stop_work push_work; + +#ifdef CONFIG_SCHED_CLASS_GHOST + struct ghost_rq ghost; +#endif }; #ifdef CONFIG_FAIR_GROUP_SCHED From cee36dbc6d9cb04f7ae84985b15c4223b03cb9f9 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 18 Jan 2022 15:43:14 -0500 Subject: [PATCH 141/165] sched/ghost: allow lockless requests to resched prev Currently, whether or not prev is allowed to be selected by PNT is controlled by rq->ghost.must_resched, which is protected by the RQ lock. Setting must_resched means that the next time PNT runs, prev will be preempted by something - even if it is the idle task. (When prev is not an agent). An agent can request that prev be unscheduled by issuing a NULL transaction. This grabs the RQ lock. We will need to preempt tasks from BPF-MSG, which already holds an RQ lock, so we need to set must_resched, or something similar, locklessly. The lockless nature of this is tricky - if we're in PNT, already ran BPF-PNT, and then the agent tries to resched the cpu, do we want the resched to take effect or not? Was the agent referring to prev (who is different than the latched task)? Or referring to the task they just latched? This sounds a lot like why we have barriers. The agent wants to change state, based on its understanding of the system. Only apply that change if the agent's understanding matches reality. We already have the cpu_seqnum: the history/state of a given cpu, exposed in all relevant messages. Even better, unlike the barriers, userspace has no assumptions about being told about every increment to cpu_seqnum. We can increment it whenever any cpu_state changes and we don't have to send a message. So when the agent (via bpf, in an upcoming commit), asks us to resched, they can tell us the cpu_seqnum they got from their last TASK_LATCHED. As a side note, this means that they should *not* use the return value of bpf_ghost_run_gtid() to mean "it's ok to resched right now!", since they don't know the cpu_seqnum. Even if we returned that value, as soon as we send TASK_LATCHED, we increment again (it's another change in cpu state, perhaps better renamed as TASK_ON_CPU, but that's a separate issue). The agent should only resched based on the latched->cpu_seqnum from when prev got on cpu. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ie9edaab22ae2f77078ca2b887fe72d4d528b2e80 --- kernel/sched/ghost.c | 19 +++++++++++++++++++ kernel/sched/ghost_core.c | 1 + kernel/sched/sched.h | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 619f9c6405e7..48ea1c4fffbc 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -4263,6 +4263,11 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) WRITE_ONCE(rq->ghost.agent_barrier, 0); p->ghost.agent = false; VM_BUG_ON(rq->ghost.blocked_in_run); + /* + * In case the user left some value that the next enclave could + * hit. + */ + WRITE_ONCE(rq->ghost.prev_resched_seq, ~0ULL); /* * Clean up any pending transactions. We need to do this here, @@ -4393,6 +4398,12 @@ static void ghost_set_pnt_state(struct rq *rq, struct task_struct *p, rq->ghost.latched_task = p; rq->ghost.must_resched = false; rq->ghost.run_flags = run_flags; + + /* + * Even though we don't send a message, this is a change in cpu state. + * When we later send TASK_LATCHED, that will increment again. + */ + rq->ghost.cpu_seqnum++; } /* @@ -7081,6 +7092,14 @@ static void pnt_prologue(struct rq *rq, struct task_struct *prev) rq->ghost.switchto_count = -rq->ghost.switchto_count; WARN_ON_ONCE(rq->ghost.switchto_count > 0); rq->ghost.pnt_bpf_once = true; + + /* + * Lockless way to set must_resched, which kicks prev off cpu. The + * agent knows the cpu_seqnum from the last message it received, e.g. + * the TASK_LATCHED when prev started to run. + */ + if (READ_ONCE(rq->ghost.prev_resched_seq) == rq->ghost.cpu_seqnum) + rq->ghost.must_resched = true; } /* diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index ea98e16f8d03..ea62673a8a7e 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -703,6 +703,7 @@ void init_ghost_rq(struct ghost_rq *ghost_rq) { INIT_LIST_HEAD(&ghost_rq->tasks); INIT_LIST_HEAD(&ghost_rq->enclave_work); + WRITE_ONCE(ghost_rq->prev_resched_seq, ~0ULL); } int ghost_sched_fork(struct task_struct *p) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4109b88c7443..ac8b55320d3a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -106,7 +106,8 @@ struct ghost_rq { struct task_struct *agent; /* protected by e->lock and rq->lock */ uint32_t agent_barrier; bool blocked_in_run; /* agent is blocked in 'ghost_run()' */ - bool agent_should_wake; + bool agent_should_wake; /* racy reads and writes */ + uint64_t prev_resched_seq; /* racy, cpu_seqnum to resched */ bool must_resched; /* rq->curr must reschedule in PNT */ bool check_prev_preemption; /* see 'ghost_prepare_task_switch()' */ bool skip_latched_preemption; From 29cf8ee8c0fcd6cecccf0b901a5038017ebe18fb Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 19 Jan 2022 12:48:52 -0500 Subject: [PATCH 142/165] sched/ghost: add bpf_ghost_resched_cpu This is a bpf helper, callable from any enclave BPF program (i.e. BPF-PNT, BPF-MSG, or skip-tick). It will force the cpu to reschedule, such that "prev" will not be picked again. Agents can use this to preempt a task. When the cpu reschedules, assuming there is no latched task, BPF-PNT will run so that the agent can pick another task to latch. A few notes: - You can call this from within BPF-PNT. If you call it on your own cpu, it will have no effect, mostly, similar to setting need_resched during pick_next_task(). The technical reason it won't work is that the kernel calls clear_preempt_need_resched() right after pick_next_task(), so any "need_resched" set during PNT will be a noop. "Mostly", since you could write "cpu_seqnum + N" and have must_resched set at some arbitrary point in the future. (The kernel cleans this up during cpu/agent teardown). - check_same_enclave() works so long as you are in an rcu read critical section. We're a bpf helper, and our bpf programs are run under an rcu_read_lock(). Someone could remove the cpu from the enclave, but that cpu cannot be handed out to another enclave until after a grace period. This is the case for all of the ghost bpf helpers. - I think you don't need an smp_wmb() between the WRITE_ONCE and the resched_cpu_unlocked() (which is a prodkernel function). That has a CAS in it, and in general, I'd expect sane behavior between the ordering of writes and IPIs in Linux. Though that might not be sane. To be submitted with cl/422883910. Tested: edf_test, using the helper in bpf-msg. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I2746e6c789010c3a1b0014483d3d2b788723149f --- include/uapi/linux/bpf.h | 11 +++++++++++ include/uapi/linux/ghost.h | 2 +- kernel/sched/ghost.c | 25 +++++++++++++++++++++++++ kernel/sched/ghost_core.c | 30 ++++++++++++++++++++++++++++++ kernel/sched/sched.h | 1 + 5 files changed, 68 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 92540b713473..9090dcd43571 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3849,6 +3849,16 @@ union bpf_attr { * * Return * 0 on success, < 0 on error. + * + * long bpf_ghost_resched_cpu(u32 cpu, u64 cpu_seqnum) + * Description + * Reschedules **cpu** if its state is still **cpu_seqnum**, such + * that it calls pick_next_task and will not pick prev again. + * Typically, it will run BPF-PNT. + * + * Return + * 0 on success, < 0 on error. + * */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4057,6 +4067,7 @@ union bpf_attr { FN(placeholder_039), \ FN(ghost_wake_agent), \ FN(ghost_run_gtid), \ + FN(ghost_resched_cpu), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 8f3c88451136..ca6680d05442 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 62 +#define GHOST_VERSION 63 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 48ea1c4fffbc..b4a66d5d120a 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -7245,6 +7245,30 @@ static int bpf_run_gtid(s64 gtid, u32 task_barrier, int run_flags, int cpu) check_caller_enclave); } +static int bpf_resched_cpu(int cpu, u64 cpu_seqnum) +{ + int this_cpu = smp_processor_id(); + struct rq *rq; + + if (cpu < 0) + return -EINVAL; + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -ERANGE; + if (!check_same_enclave(this_cpu, cpu)) + return -EXDEV; + + rq = cpu_rq(cpu); + WRITE_ONCE(rq->ghost.prev_resched_seq, cpu_seqnum); + if (cpu == this_cpu) { + set_tsk_need_resched(current); + set_preempt_need_resched(); + } else { + resched_cpu_unlocked(cpu); + } + + return 0; +} + static bool ghost_msg_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, @@ -7429,6 +7453,7 @@ DEFINE_GHOST_ABI(current_abi) = { .timerfd_triggered = _ghost_timerfd_triggered, .bpf_wake_agent = bpf_wake_agent, .bpf_run_gtid = bpf_run_gtid, + .bpf_resched_cpu = bpf_resched_cpu, .ghost_msg_is_valid_access = ghost_msg_is_valid_access, .bpf_link_attach = bpf_link_attach, .bpf_link_detach = bpf_link_detach, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index ea62673a8a7e..67f61519c3d5 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1191,6 +1191,32 @@ static const struct bpf_func_proto bpf_ghost_run_gtid_proto = { .arg3_type = ARG_ANYTHING, }; +BPF_CALL_2(bpf_ghost_resched_cpu, u32, cpu, u64, cpu_seqnum) +{ + struct ghost_enclave *e; + + VM_BUG_ON(preemptible()); + + BUILD_BUG_ON(BPF_FUNC_ghost_resched_cpu != 206); + + /* rcu_read_lock_sched() not needed; preemption is disabled. */ + e = rcu_dereference_sched(per_cpu(enclave, smp_processor_id())); + + /* Paranoia: this is not expected */ + if (WARN_ON_ONCE(!e)) + return -ENODEV; + + return e->abi->bpf_resched_cpu(cpu, cpu_seqnum); +} + +static const struct bpf_func_proto bpf_ghost_resched_cpu_proto = { + .func = bpf_ghost_resched_cpu, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_ANYTHING, + .arg2_type = ARG_ANYTHING, +}; + static const struct bpf_func_proto * ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -1206,6 +1232,8 @@ ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) default: return NULL; } + case BPF_FUNC_ghost_resched_cpu: + return &bpf_ghost_resched_cpu_proto; default: return bpf_base_func_proto(func_id); } @@ -1234,6 +1262,8 @@ ghost_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) switch (func_id) { case BPF_FUNC_ghost_wake_agent: return &bpf_ghost_wake_agent_proto; + case BPF_FUNC_ghost_resched_cpu: + return &bpf_ghost_resched_cpu_proto; default: return bpf_base_func_proto(func_id); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ac8b55320d3a..aaf779122976 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2208,6 +2208,7 @@ struct ghost_abi { void (*timerfd_triggered)(int cpu, uint64_t type, uint64_t cookie); int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); + int (*bpf_resched_cpu)(int cpu, u64 cpu_seqnum); bool (*ghost_msg_is_valid_access)(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, From 4c4e627e4fc873a417acc76e4261f66733a6a644 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Fri, 21 Jan 2022 15:31:52 -0500 Subject: [PATCH 143/165] sched/ghost: fix bpf_helper enclave discovery Certain operations, particularly ghost bpf helpers, are operating on some enclave, but it might not be the enclave that owns the cpu. Consider an enclave on a set of cpus, and a task on another cpu enters the enclave. That generates a TASK_NEW, which triggers BPF-MSG. That all runs on a cpu that is not in the enclave. Add per-task *__target_enclave to track which enclave we are targeting with our functions. Users of the target_enclave can nest, such that if we are in the middle of some operation and get interrupted, the IRQ handler can set its own target_enclave so long as it restores it. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I1cf7bc70bf2bda6d133155b9a8877ca531074587 --- include/linux/sched.h | 3 +++ kernel/sched/ghost.c | 13 ++++++++++++- kernel/sched/ghost_core.c | 27 +++++++++++++++++++++------ kernel/sched/sched.h | 14 ++++++++++++++ 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 5bc75459cdb5..dfb429bbda89 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -600,6 +600,9 @@ struct sched_ghost_entity { /* See ghost_destroy_enclave() */ struct ghost_enclave *__agent_decref_enclave; + /* For operations with an "implicit" enclave parameter. */ + struct ghost_enclave *__target_enclave; + /* * See also ghost_prepare_task_switch() and ghost_deferred_msgs() * for flags that are used to defer messages. diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b4a66d5d120a..b683f624d5cf 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -140,6 +140,8 @@ static bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) { struct bpf_ghost_sched_kern ctx = {}; struct bpf_prog *prog; + struct ghost_enclave *old_target; + bool ret; lockdep_assert_held(&rq->lock); @@ -147,8 +149,11 @@ static bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) if (!prog) return false; + old_target = set_target_enclave(e); /* prog returns 1 if we want a tick on this cpu. */ - return BPF_PROG_RUN(prog, &ctx) != 1; + ret = BPF_PROG_RUN(prog, &ctx) != 1; + restore_target_enclave(old_target); + return ret; } #define BPF_GHOST_PNT_DONT_IDLE 1 @@ -158,6 +163,7 @@ static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, { struct bpf_ghost_sched_kern ctx = {}; struct bpf_prog *prog; + struct ghost_enclave *old_target; int ret; lockdep_assert_held(&rq->lock); @@ -177,7 +183,9 @@ static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, rq_unpin_lock(rq, rf); raw_spin_unlock(&rq->lock); + old_target = set_target_enclave(e); ret = BPF_PROG_RUN(prog, &ctx); + restore_target_enclave(old_target); raw_spin_lock(&rq->lock); rq_repin_lock(rq, rf); @@ -204,6 +212,7 @@ static bool ghost_bpf_msg_send(struct ghost_enclave *e, struct bpf_ghost_msg *msg) { struct bpf_prog *prog; + struct ghost_enclave *old_target; bool send; rcu_read_lock(); @@ -213,7 +222,9 @@ static bool ghost_bpf_msg_send(struct ghost_enclave *e, return true; } /* Program returns 0 if they want us to send the message. */ + old_target = set_target_enclave(e); send = BPF_PROG_RUN(prog, msg) == 0; + restore_target_enclave(old_target); rcu_read_unlock(); return send; } diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 67f61519c3d5..ca8b5364f4be 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -312,6 +312,24 @@ __exitcall(ghostfs_exit); static DEFINE_SPINLOCK(cpu_rsvp); DEFINE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); +struct ghost_enclave *get_target_enclave(void) +{ + return current->ghost.__target_enclave; +} + +struct ghost_enclave *set_target_enclave(struct ghost_enclave *e) +{ + struct ghost_enclave *old = current->ghost.__target_enclave; + + current->ghost.__target_enclave = e; + return old; +} + +void restore_target_enclave(struct ghost_enclave *old) +{ + current->ghost.__target_enclave = old; +} + /* Caller holds e->lock. Either all cpus are added, or none are. */ int ghost_add_cpus(struct ghost_enclave *e, const struct cpumask *new_cpus) { @@ -1141,8 +1159,7 @@ BPF_CALL_1(bpf_ghost_wake_agent, u32, cpu) */ BUILD_BUG_ON(BPF_FUNC_ghost_wake_agent != 204); - /* rcu_read_lock_sched() not needed; preemption is disabled. */ - e = rcu_dereference_sched(per_cpu(enclave, smp_processor_id())); + e = get_target_enclave(); /* Paranoia: this is not expected */ if (WARN_ON_ONCE(!e)) @@ -1171,8 +1188,7 @@ BPF_CALL_3(bpf_ghost_run_gtid, s64, gtid, u32, task_barrier, int, run_flags) */ BUILD_BUG_ON(BPF_FUNC_ghost_run_gtid != 205); - /* rcu_read_lock_sched() not needed; preemption is disabled. */ - e = rcu_dereference_sched(per_cpu(enclave, smp_processor_id())); + e = get_target_enclave(); /* Paranoia: this is not expected */ if (WARN_ON_ONCE(!e)) @@ -1199,8 +1215,7 @@ BPF_CALL_2(bpf_ghost_resched_cpu, u32, cpu, u64, cpu_seqnum) BUILD_BUG_ON(BPF_FUNC_ghost_resched_cpu != 206); - /* rcu_read_lock_sched() not needed; preemption is disabled. */ - e = rcu_dereference_sched(per_cpu(enclave, smp_processor_id())); + e = get_target_enclave(); /* Paranoia: this is not expected */ if (WARN_ON_ONCE(!e)) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index aaf779122976..a669186f495d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2263,6 +2263,20 @@ const static struct ghost_abi __##name##_ghost_abi \ _GHOST_MAYBE_CONST DECLARE_PER_CPU_READ_MOSTLY(struct ghost_enclave *, enclave); +/* + * Some functions operate on an enclave, but we are unable to easily pass the + * enclave parameter. This enclave is the `target` for an operation. + * + * Rules: + * - set_target_enclave returns the old target. Restore it when you are + * done. + * - may be called from IRQ context. + * - the target_enclave is bound to the current task + */ +struct ghost_enclave *get_target_enclave(void); +struct ghost_enclave *set_target_enclave(struct ghost_enclave *e); +void restore_target_enclave(struct ghost_enclave *old); + void init_sched_ghost_class(void); int ghost_add_cpus(struct ghost_enclave *e, const struct cpumask *cpus); void ghost_remove_cpu(struct ghost_enclave *e, int cpu); From 18cf57f6350e1f39ed6dd443912e44bd6596dc30 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 25 Jan 2022 12:07:11 -0500 Subject: [PATCH 144/165] sched/ghost: alphabetize ghostfs enclave dirtab Directories before files, then alphabetical sorting. No capital letters. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ib9267e091342ea8a84c4e0e639fe3fae5a673fba --- kernel/sched/ghost.c | 188 +++++++++++++++++++++---------------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b683f624d5cf..110732dc84ae 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6245,6 +6245,21 @@ static void gf_e_release(struct kernfs_open_file *of) kref_put(&e->kref, enclave_release); } +static int gf_abi_version_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + WARN_ON_ONCE(e->abi->version != GHOST_VERSION); + seq_printf(sf, "%u\n", e->abi->version); + return 0; +} + +static struct kernfs_ops gf_ops_e_abi_version = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_abi_version_show, +}; + /* * There can be at most one writable open, and there can be any number of * read-only opens. @@ -6327,6 +6342,37 @@ static struct kernfs_ops gf_ops_e_agent_online = { .write = gf_agent_online_write, }; +static int gf_commit_at_tick_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->commit_at_tick)); + return 0; +} + +static ssize_t gf_commit_at_tick_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->commit_at_tick, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_commit_at_tick = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_commit_at_tick_show, + .write = gf_commit_at_tick_write, +}; + static int gf_cpu_data_mmap(struct kernfs_open_file *of, struct vm_area_struct *vma) { @@ -6764,6 +6810,39 @@ static struct kernfs_ops gf_ops_e_runnable_timeout = { .write = gf_runnable_timeout_write, }; +static int gf_status_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + unsigned long fl; + bool is_active; + unsigned long nr_tasks; + + /* + * Userspace uses this to find any active enclave, since they don't have + * any other methods yet to know which enclave to use. + */ + spin_lock_irqsave(&e->lock, fl); + /* + * We don't need to lock to read agent_online, but eventually we'll + * check for the presence of an interstitial scheduler too. This status + * is for the *enclave*, not the *agent*. + */ + is_active = e->agent_online; + nr_tasks = e->nr_tasks; + spin_unlock_irqrestore(&e->lock, fl); + + seq_printf(sf, "version %u\n", e->abi->version); + seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); + seq_printf(sf, "nr_tasks %lu\n", nr_tasks); + return 0; +} + +static struct kernfs_ops gf_ops_e_status = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_status_show, +}; + static int gf_switchto_disabled_show(struct seq_file *sf, void *v) { struct ghost_enclave *e = seq_to_e(sf); @@ -6826,96 +6905,27 @@ static struct kernfs_ops gf_ops_e_wake_on_waker_cpu = { .write = gf_wake_on_waker_cpu_write, }; -static int gf_commit_at_tick_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - seq_printf(sf, "%d", READ_ONCE(e->commit_at_tick)); - return 0; -} - -static ssize_t gf_commit_at_tick_write(struct kernfs_open_file *of, char *buf, - size_t len, loff_t off) -{ - struct ghost_enclave *e = of_to_e(of); - int err; - int tunable; - - err = kstrtoint(buf, 0, &tunable); - if (err) - return -EINVAL; - - WRITE_ONCE(e->commit_at_tick, !!tunable); - - return len; -} - -static struct kernfs_ops gf_ops_e_commit_at_tick = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_commit_at_tick_show, - .write = gf_commit_at_tick_write, -}; - -static int gf_status_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - unsigned long fl; - bool is_active; - unsigned long nr_tasks; - - /* - * Userspace uses this to find any active enclave, since they don't have - * any other methods yet to know which enclave to use. - */ - spin_lock_irqsave(&e->lock, fl); - /* - * We don't need to lock to read agent_online, but eventually we'll - * check for the presence of an interstitial scheduler too. This status - * is for the *enclave*, not the *agent*. - */ - is_active = e->agent_online; - nr_tasks = e->nr_tasks; - spin_unlock_irqrestore(&e->lock, fl); - - seq_printf(sf, "version %u\n", e->abi->version); - seq_printf(sf, "active %s\n", is_active ? "yes" : "no"); - seq_printf(sf, "nr_tasks %lu\n", nr_tasks); - return 0; -} - -static struct kernfs_ops gf_ops_e_status = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_status_show, -}; - -static int gf_abi_version_show(struct seq_file *sf, void *v) -{ - struct ghost_enclave *e = seq_to_e(sf); - - WARN_ON_ONCE(e->abi->version != GHOST_VERSION); - seq_printf(sf, "%u\n", e->abi->version); - return 0; -} - -static struct kernfs_ops gf_ops_e_abi_version = { - .open = gf_e_open, - .release = gf_e_release, - .seq_show = gf_abi_version_show, -}; - static struct gf_dirent enclave_dirtab[] = { { .name = "sw_regions", .mode = 0555, .is_dir = true, }, + { + .name = "abi_version", + .mode = 0444, + .ops = &gf_ops_e_abi_version, + }, { .name = "agent_online", .mode = 0664, .ops = &gf_ops_e_agent_online, }, + { + .name = "commit_at_tick", + .mode = 0664, + .ops = &gf_ops_e_commit_at_tick, + }, { .name = "cpu_data", .mode = 0660, @@ -6941,6 +6951,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_runnable_timeout, }, + { + .name = "status", + .mode = 0444, + .ops = &gf_ops_e_status, + }, { .name = "switchto_disabled", .mode = 0664, @@ -6951,21 +6966,6 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_wake_on_waker_cpu, }, - { - .name = "commit_at_tick", - .mode = 0664, - .ops = &gf_ops_e_commit_at_tick, - }, - { - .name = "status", - .mode = 0444, - .ops = &gf_ops_e_status, - }, - { - .name = "abi_version", - .mode = 0444, - .ops = &gf_ops_e_abi_version, - }, {0}, }; From b0cfdcdda320cde816b8e36cb403009d68d9f403 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Thu, 10 Feb 2022 11:03:57 -0800 Subject: [PATCH 145/165] sched: return status directly in ghost_associate_queue() put_user() may fail at the end of ghost_associate_queue() while dst_q was set successfully. Return the status directly instead of writing it to the ioctl parameter. To be submitted with cl/427797430. Effort: sched/ghost Google-Bug-Id: 202070945 Change-Id: I9a72bde02afa556bf5c644be4bb80c98274c1b3a --- include/uapi/linux/ghost.h | 5 ++--- kernel/sched/ghost.c | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index ca6680d05442..6e80f1760614 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 63 +#define GHOST_VERSION 64 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined @@ -108,7 +108,6 @@ struct ghost_ioc_assoc_queue { struct ghost_msg_src src; int barrier; int flags; - int status; }; struct ghost_ioc_set_default_queue { @@ -162,7 +161,7 @@ struct ghost_ioc_run { #define GHOST_IOC_SW_GET_INFO _IOWR('g', 1, struct ghost_ioc_sw_get_info) #define GHOST_IOC_SW_FREE _IOW('g', 2, struct ghost_sw_info) #define GHOST_IOC_CREATE_QUEUE _IOWR('g', 3, struct ghost_ioc_create_queue) -#define GHOST_IOC_ASSOC_QUEUE _IOWR('g', 4, struct ghost_ioc_assoc_queue) +#define GHOST_IOC_ASSOC_QUEUE _IOW('g', 4, struct ghost_ioc_assoc_queue) #define GHOST_IOC_SET_DEFAULT_QUEUE _IOW('g', 5, struct ghost_ioc_set_default_queue) #define GHOST_IOC_CONFIG_QUEUE_WAKEUP _IOW('g', 6, struct ghost_ioc_config_queue_wakeup) #define GHOST_IOC_GET_CPU_TIME _IOWR('g', 7, struct ghost_ioc_get_cpu_time) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 110732dc84ae..4c98e206b760 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3341,9 +3341,10 @@ static int ghost_associate_queue(struct ghost_ioc_assoc_queue __user *arg) if (p) task_rq_unlock(rq, p, &rf); fput(file); - /* TODO(b/202070945) */ - if (!error) - error = put_user(status, &arg->status); + if (!error) { + WARN_ON_ONCE(status < 0); + return status; + } return error; } From 2f3ddbf768bf40bc15327030cbe800dcdbc754cf Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 14 Feb 2022 15:52:46 +0000 Subject: [PATCH 146/165] sched: poisoned rendezvous should not produce TASK_PREEMPT. Prior to this change a task with a poisoned rendezvous could produce a MSG_TASK_PREEMPT (e.g. if the next task picked to run on the cpu belongs to CFS). This is not intuitive since the overall sync_group has failed. It is also easy to miss since producing MSG_TASK_PREEMPT is the exception. For e.g. see cl/427576696 that fixes flaky ApiTest.LocalAgentWakeup Effort: sched/ghost Google-Bug-Id: 214648944 Change-Id: Ib13c38f60b8d1acbd292d4de7b8d7ff528f44c4f --- kernel/sched/ghost.c | 22 ++++++++++++++++------ kernel/sched/sched.h | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 4c98e206b760..b150b706e21c 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -549,13 +549,21 @@ static inline void schedule_agent(struct rq *rq, bool resched) schedule_next(rq, GHOST_AGENT_GTID, resched); } -static inline void force_offcpu(struct rq *rq, bool resched) +static inline void _force_offcpu(struct rq *rq, bool resched, + bool ignore_prev_preemption) { VM_BUG_ON(!ghost_class(rq->curr->sched_class)); + rq->ghost.ignore_prev_preemption = ignore_prev_preemption; + schedule_next(rq, GHOST_NULL_GTID, resched); } +static inline void force_offcpu(struct rq *rq, bool resched) +{ + _force_offcpu(rq, resched, /*ignore_prev_preemption=*/false); +} + static void __update_curr_ghost(struct rq *rq, bool update_sw) { struct task_struct *curr = rq->curr; @@ -4636,9 +4644,6 @@ static void wait_for_rendezvous(struct rq *rq) /* * Reschedule immediately if rendezvous is poisoned * (some other txn in the sync_group failed to commit). - * - * N.B. rescheduling here could produce TASK_PREEMPTED - * msg from 'rq->curr'. */ if (unlikely(rendezvous_poisoned(target))) { struct rq_flags rf; @@ -4682,8 +4687,9 @@ static void wait_for_rendezvous(struct rq *rq) rq_unlock_irqrestore(rq, &rf); continue; } - - force_offcpu(rq, /*resched=*/true); + _force_offcpu(rq, + /*resched=*/true, + /*ignore_prev_preemption=*/true); rq_unlock_irqrestore(rq, &rf); VM_BUG_ON(!need_resched()); } @@ -7099,6 +7105,10 @@ static struct ghost_enclave *create_enclave(const struct ghost_abi *abi, static void pnt_prologue(struct rq *rq, struct task_struct *prev) { rq->ghost.check_prev_preemption = ghost_produce_prev_msgs(rq, prev); + if (unlikely(rq->ghost.ignore_prev_preemption)) { + rq->ghost.check_prev_preemption = false; + rq->ghost.ignore_prev_preemption = false; + } /* a negative 'switchto_count' indicates end of the chain */ rq->ghost.switchto_count = -rq->ghost.switchto_count; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a669186f495d..24773e8fc02f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -109,6 +109,7 @@ struct ghost_rq { bool agent_should_wake; /* racy reads and writes */ uint64_t prev_resched_seq; /* racy, cpu_seqnum to resched */ bool must_resched; /* rq->curr must reschedule in PNT */ + bool ignore_prev_preemption; bool check_prev_preemption; /* see 'ghost_prepare_task_switch()' */ bool skip_latched_preemption; bool pnt_bpf_once; /* BPF runs at most once in PNT */ From ba3d6cf2a8b05ccc948bbb3124822c5bbb1c8a87 Mon Sep 17 00:00:00 2001 From: Neel Natu Date: Mon, 14 Feb 2022 20:56:38 +0000 Subject: [PATCH 147/165] sched: relax VM_BUG_ON check in _ghost_resolve_enclave() At the moment ghOSt supports more than one enclave running concurrently on the host as long as they all have the same ABI. Relax the check in _ghost_resolve_enclave() to reflect that since it is possible for the task_reaper (which is a CFS task) to run on any cpu (including one that belongs to an enclave different than the one it is reaping). Effort: sched/ghost Google-Bug-Id: 218869667 Change-Id: I1b79d4468c343a5a837349ba6984f79bf1f1ec40 --- kernel/sched/ghost_core.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index ca8b5364f4be..eedad1a7b5ff 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -427,7 +427,12 @@ static struct ghost_enclave *_ghost_resolve_enclave(struct rq *rq, * while the cpu it is running on belongs to enclave Y. This is not * supported properly yet hence the VM_BUG_ON. */ +#if 0 // FIXME(b/218869667) VM_BUG_ON(rq_enclave && task_enclave && rq_enclave != task_enclave); +#else + VM_BUG_ON(rq_enclave && task_enclave && + rq_enclave->abi != task_enclave->abi); +#endif /* * 'rq_enclave' may be NULL if a running task enters ghost on a cpu From 6f78ab77e5da4f009ef36d96a1e4211d002c721d Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 26 Jan 2022 14:52:49 -0500 Subject: [PATCH 148/165] sched/ghost: use ghostfs to move tasks to an enclave Instead of using syscalls, use ghostfs to move tasks to an enclave. This cleans up core.c's setsched code. We had a weird dance of passing along a ctl_fd, then converting that to an enclave, but we had to put the enclave after unlocking the RQ. It wasn't pretty. Instead, if we enter the kernel through ghostfs, the enclave is already known, and its refcounts are managed for us. This is analogous to using ghostfs ioctls instead of raw syscalls: we do not need any magic to figure out which enclave we're using. To that end, there are two mechanisms to enter ghost, one for regular tasks and another for agents. To add a regular task to an enclave, write its pid into enclave_x/tasks. Write 0 for 'current'. The tasks file is world writable, so anyone can join an enclave, if they can find it. However, to move another task (pid != 0), you need CAP_SYS_NICE, just like with sys_sched_setscheduler(). You can read the tasks file to get a list of tasks in the enclave, though beware that if you have too many tasks, the file might be truncated (seq file overflow). To add an agent to an enclave, write "become agent " into ctl. You need write access to ctl, which an agent will have. The QFD is the FD of the queue you want for this agent, -1 for "default". qfd is the same parameter we used to pass in the sched_attr. To be submitted with cl/424409182. Tested: edf_test, enclave_test, api_test, transaction_test, manually moving tasks into an enclave from the shell. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ib4ead6536354259ca1a464e62d72220c0465eef4 --- kernel/sched/core.c | 31 +------ kernel/sched/ghost.c | 164 ++++++++++++++++++++++++++++++-------- kernel/sched/ghost_core.c | 39 +++------ kernel/sched/sched.h | 26 +----- 4 files changed, 144 insertions(+), 116 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index c3979d21cbe5..a973c505c6f1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5903,13 +5903,6 @@ static void __setscheduler_params(struct task_struct *p, #ifdef CONFIG_SCHED_CLASS_GHOST if (ghost_policy(policy)) { - if (ghost_agent(attr)) { - /* - * This might catch an unlock in between - * ghost_setscheduler() and here. - */ - WARN_ON_ONCE(task_rq(p)->ghost.agent != p); - } p->rt_priority = 0; p->normal_prio = normal_prio(p); set_load_weight(p, true); @@ -5998,10 +5991,6 @@ static int __sched_setscheduler(struct task_struct *p, int reset_on_fork; int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; struct rq *rq; -#ifdef CONFIG_SCHED_CLASS_GHOST - struct ghost_enclave *new_e; - struct fd f_enc; -#endif /* The pi code expects interrupts enabled */ BUG_ON(pi && in_interrupt()); @@ -6022,9 +6011,7 @@ static int __sched_setscheduler(struct task_struct *p, #ifdef CONFIG_SCHED_CLASS_GHOST if (ghost_policy(policy)) { - retval = ghost_validate_sched_attr(attr); - if (retval) - return retval; + /* ghost_setscheduler() can fail, so we do all checks there. */ } else #endif { @@ -6210,22 +6197,13 @@ static int __sched_setscheduler(struct task_struct *p, } #ifdef CONFIG_SCHED_CLASS_GHOST - if (ghost_policy(policy)) { - new_e = ghost_fdget_enclave(ghost_schedattr_to_enclave_fd(attr), - &f_enc); - } else { - new_e = NULL; - } if (ghost_policy(policy) || ghost_policy(p->policy)) { - int error = ghost_setscheduler(p, rq, attr, new_e, - &reset_on_fork); + int error = ghost_setscheduler(p, rq, attr, &reset_on_fork); if (error) { task_rq_unlock(rq, p, &rf); if (pi) cpuset_read_unlock(); - if (ghost_policy(policy)) - ghost_fdput_enclave(new_e, &f_enc); return error; } } @@ -6293,11 +6271,6 @@ static int __sched_setscheduler(struct task_struct *p, balance_callbacks(rq, head); preempt_enable(); -#ifdef CONFIG_SCHED_CLASS_GHOST - if (ghost_policy(policy)) - ghost_fdput_enclave(new_e, &f_enc); -#endif - return 0; unlock: diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index b150b706e21c..fcf05e989cb9 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -2813,37 +2813,27 @@ static void _ghost_sched_cleanup_fork(struct ghost_enclave *e, } /* - * For tasks attempting to join ghost, __sched_setscheduler() needs to pass us - * the enclave, and it needs to manage the reference counting on the fd and its - * underlying kn. This is because we can't call ghost_fdput_enclave directly - * while holding the rq lock, because it eventually calls kernfs_put_active, - * which grabs the rq lock. + * attr->sched_priority is *not* set by the user. It is set by the ghostfs + * commands that call setscheduler elsewhere in this file. They aren't part of + * any ABI. * - * You'd think we could use a balance_callback, and pass it the fd_to_put - * (f_enc) and whether or not it had an enclave. However, there's no good place - * to put those arguments. We could stash them in struct ghost_rq, but for one - * small problem: the rq lock is released before balance_callback() runs. That - * means someone else could grab the lock, then setsched another task, thereby - * clobbering the args stored in ghost_rq. + * sched_priority is an argument from ghostfs to tell us if we're dealing with + * an agent or not, and if so, which qfd it wants. + * < -1 : a regular task + * == -1 : an agent, use the default queue + * >= 0 : an agent, sched_priority is the fd for the queue. * - * We can't dynamically allocate memory either, since we're holding the rq lock. - * I even considered reusing the sched_attr struct: cast it to some other struct - * and hang it off a linked list on the rq. The problem there is that although - * we will call balance_callback when we return to __sched_setscheduler(), - * balance_callback makes no guarantees about when the callback will run. If - * two threads call balance_callback(), one of them will run the callbacks and - * the other will return immediately. If we return immediately, then we can't - * use the schedattr. - * - * The most reasonable fix for all of this is to directly call - * ghost_fdput_enclave() from __sched_setscheduler(). + * Anyone who calls sys_sched_setscheduler (or setattr) will not have a + * target_enclave() set. */ -static int _ghost_setscheduler(struct task_struct *p, struct rq *rq, - const struct sched_attr *attr, - struct ghost_enclave *new_e, +static int _ghost_setscheduler(struct ghost_enclave *e, struct task_struct *p, + struct rq *rq, const struct sched_attr *attr, int *reset_on_fork) { - int ret; + int ret, qfd; + + if (WARN_ON_ONCE(!e)) + return -ENODEV; /* Task 'p' is departing the ghost sched class. */ if (ghost_policy(p->policy)) { @@ -2860,14 +2850,22 @@ static int _ghost_setscheduler(struct task_struct *p, struct rq *rq, return 0; } - if (WARN_ON_ONCE(!new_e)) - return -EBADF; + /* Any setsched *into* ghost must have come from ghostfs. */ + if (WARN_ON_ONCE(!get_target_enclave())) + return -ENODEV; - if (ghost_agent(attr)) { - int qfd = ghost_schedattr_to_queue_fd(attr); + qfd = (int)attr->sched_priority; + if (qfd >= -1) { struct fd f_que = {0}; struct ghost_queue *q = NULL; + /* + * A thread can only make a task an agent if the thread has the + * CAP_SYS_NICE capability. + */ + if (!capable(CAP_SYS_NICE)) + return -EPERM; + if (qfd != -1) { f_que = fdget(qfd); q = fd_to_queue(f_que); @@ -2879,12 +2877,12 @@ static int _ghost_setscheduler(struct task_struct *p, struct rq *rq, /* It's OK to set reset_on_fork even if we fail. */ *reset_on_fork = 1; - ret = ghost_prep_agent(new_e, p, rq, q); + ret = ghost_prep_agent(e, p, rq, q); if (qfd != -1) fdput(f_que); } else { - ret = ghost_prep_task(new_e, p, false); + ret = ghost_prep_task(e, p, false); } return ret; @@ -6545,6 +6543,29 @@ static void destroy_enclave(struct kernfs_open_file *ctl_of) ghost_destroy_enclave(e); } +static int ctl_become_agent(struct kernfs_open_file *of, int qfd) +{ + struct ghost_enclave *e = of_to_e(of); + struct ghost_enclave *old_target; + int ret; + struct sched_param param = { + .sched_priority = qfd, + }; + + /* + * See _ghost_setscheduler() for the meaning of sched_priority. + * -1 is valid. It means "give me the default queue". + */ + if (qfd < -1) + return -EBADF; + + old_target = set_target_enclave(e); + ret = sched_setscheduler(current, SCHED_GHOST | SCHED_RESET_ON_FORK, + ¶m); + restore_target_enclave(old_target); + return ret; +} + static struct ghost_sw_region *of_to_swr(struct kernfs_open_file *of) { return of->kn->priv; @@ -6640,7 +6661,7 @@ static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, size_t len, loff_t off) { unsigned int arg1, arg2; - int err; + int err, qfd; /* * Ignore the offset for ctl commands, so userspace doesn't have to @@ -6657,6 +6678,10 @@ static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, err = create_sw_region(of, arg1, arg2); if (err) return err; + } else if (sscanf(buf, "become agent %d", &qfd) == 1) { + err = ctl_become_agent(of, qfd); + if (err) + return err; } else { pr_err("%s: bad cmd :%s:", __func__, buf); return -EINVAL; @@ -6881,6 +6906,74 @@ static struct kernfs_ops gf_ops_e_switchto_disabled = { .write = gf_switchto_disabled_write, }; +static int gf_tasks_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + unsigned long irq_fl; + struct task_struct *t; + + /* TODO: limited to PAGE_SIZE, fixable with seq_{start,next,stop.} */ + spin_lock_irqsave(&e->lock, irq_fl); + list_for_each_entry(t, &e->task_list, ghost.task_list) + seq_printf(sf, "%d\n", t->pid); + spin_unlock_irqrestore(&e->lock, irq_fl); + + return 0; +} + +/* + * This file is world writable so that any task can join an enclave. However, + * you must have CAP_SYS_NICE to change the scheduling policy of another thread. + * This is checked in the core sched_setscheduler code. + */ +static ssize_t gf_tasks_write(struct kernfs_open_file *of, char *buf, + size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + struct ghost_enclave *old_target; + ssize_t ret; + pid_t pid; + struct task_struct *t; + + /* See _ghost_setscheduler() for the meaning of sched_priority. */ + struct sched_param param = { + .sched_priority = -2, + }; + + ret = kstrtoint(buf, 0, &pid); + if (ret) + return ret; + + rcu_read_lock(); + if (pid) { + t = find_task_by_vpid(pid); + if (!t) { + ret = -ESRCH; + goto out; + } + } else { + t = current; + } + + old_target = set_target_enclave(e); + ret = sched_setscheduler(t, SCHED_GHOST, ¶m); + if (ret == 0) + ret = len; + restore_target_enclave(old_target); + +out: + rcu_read_unlock(); + + return ret; +} + +static struct kernfs_ops gf_ops_e_tasks = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_tasks_show, + .write = gf_tasks_write, +}; + static int gf_wake_on_waker_cpu_show(struct seq_file *sf, void *v) { struct ghost_enclave *e = seq_to_e(sf); @@ -6968,6 +7061,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_switchto_disabled, }, + { + .name = "tasks", + .mode = 0666, + .ops = &gf_ops_e_tasks, + }, { .name = "wake_on_waker_cpu", .mode = 0664, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index eedad1a7b5ff..25be66a4c3d0 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -471,7 +471,7 @@ void __init init_sched_ghost_class(void) * * Caller must call ghost_fdput_enclave with e and &fd_to_put, even on error. */ -struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) +static struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) { const struct ghost_abi *abi; struct ghost_enclave *e = NULL; @@ -519,39 +519,15 @@ struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) * Pairs with calls to ghost_fdget_enclave(). You can't call this while holding * the rq lock. */ -void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd) +static void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd) { if (e) e->abi->ctlfd_enclave_put(fd->file); fdput(*fd); } -#define GHOST_SCHED_TASK_PRIO 0 -#define GHOST_SCHED_AGENT_PRIO 1 -int ghost_validate_sched_attr(const struct sched_attr *attr) -{ - /* - * A thread can only make a task an agent if the thread has the - * CAP_SYS_NICE capability. - */ - switch (attr->sched_priority) { - case GHOST_SCHED_TASK_PRIO: - return 0; - case GHOST_SCHED_AGENT_PRIO: - return capable(CAP_SYS_NICE) ? 0 : -EPERM; - default: - return -EINVAL; - } -} - -bool ghost_agent(const struct sched_attr *attr) -{ - return attr->sched_priority == GHOST_SCHED_AGENT_PRIO; -} - int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, - struct ghost_enclave *new_e, int *reset_on_fork) { struct ghost_enclave *e; @@ -579,15 +555,18 @@ int ghost_setscheduler(struct task_struct *p, struct rq *rq, if (oldpolicy == newpolicy) return -EPERM; - if (ghost_policy(oldpolicy)) + if (ghost_policy(oldpolicy)) { e = p->ghost.enclave; - else - e = new_e; + WARN_ON_ONCE(!e); + } else { + e = get_target_enclave(); + } + /* Callers from sys_sched_setscheduler will not have e set. */ if (!e) return -EBADF; - return e->abi->setscheduler(p, rq, attr, new_e, reset_on_fork); + return e->abi->setscheduler(e, p, rq, attr, reset_on_fork); } void ghost_prepare_task_switch(struct rq *rq, struct task_struct *prev, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 24773e8fc02f..288dd20b5312 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -212,16 +212,6 @@ static inline void sched_ghost_entity_init(struct task_struct *p) INIT_LIST_HEAD(&p->ghost.task_list); } -static inline int ghost_schedattr_to_enclave_fd(const struct sched_attr *attr) -{ - return attr->sched_runtime; -} - -static inline int ghost_schedattr_to_queue_fd(const struct sched_attr *attr) -{ - return attr->sched_deadline; -} - #else static inline unsigned long ghost_cfs_added_load(struct rq *rq) { return 0; } #endif /* CONFIG_SCHED_CLASS_GHOST */ @@ -2190,9 +2180,8 @@ struct ghost_abi { void (*enclave_add_cpu)(struct ghost_enclave *e, int cpu); struct ghost_enclave *(*ctlfd_enclave_get)(struct file *file); void (*ctlfd_enclave_put)(struct file *file); - int (*setscheduler)(struct task_struct *p, struct rq *rq, - const struct sched_attr *attr, - struct ghost_enclave *new_e, + int (*setscheduler)(struct ghost_enclave *e, struct task_struct *p, + struct rq *rq, const struct sched_attr *attr, int *reset_on_fork); int (*fork)(struct ghost_enclave *e, struct task_struct *p); void (*cleanup_fork)(struct ghost_enclave *e, struct task_struct *p); @@ -2287,23 +2276,12 @@ void ghost_wait_for_rendezvous(struct rq *rq); void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); void ghost_tick(struct rq *rq); -/* - * Helper to map an 'fd' to a 'ghost_enclave'. Must always be followed with - * a call to ghost_fdput_enclave() even if the lookup fails. - * - * N.B. ghost_fdput_enclave() cannot be called with 'rq->lock' held. - */ -struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put); -void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd_to_put); int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, - struct ghost_enclave *new_e, int *reset_on_fork); int ghost_sched_fork(struct task_struct *p); void ghost_sched_cleanup_fork(struct task_struct *p); -bool ghost_agent(const struct sched_attr *attr); -int ghost_validate_sched_attr(const struct sched_attr *attr); void ghost_copy_process_epilogue(struct task_struct *p); static inline int enclave_abi(const struct ghost_enclave *e) From 39c4b87aedea72dbcd7d4667aa874d186a2895d4 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 27 Jan 2022 13:30:53 -0500 Subject: [PATCH 149/165] sched/ghost: move bpf_link guts into ABI-dependent code Now that there are no other users of ctlfd_to_enclave, we can clean up how bpf programs are linked. - We know the ABI from the expected_attach_type, so we don't need to do the for_each_abi() check. - We only need the struct fd and the kernfs kn until we incref the enclave. Simply returning a kreffed pointer simplifies the code. - ghost_bpf_link_attach just needs to figure out the ABI. The rest of the guts can be in ghost.c. This means abi->bpf_attach and detach can be in ghost.c directly. Further, the ctlfd_to_enclave can be in ghost.c too. Future ABIs don't even need to use the ctlfd (though the old ghost_core.c didn't actually know what an enclave ctl is). - We no longer do any PROG_TYPE checking in ghost_core.c. That makes it easier to change our types and attach points in the future. e.g. dropping skip_tick. We still have PROG_TYPE_GHOST in other places in the kernel, so it's not doable yet. Tested: api_test, edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Iea57c6061047dcb0bf0c4bb2e031a86bbc62b689 --- kernel/sched/ghost.c | 172 ++++++++++++++++++++++++++++------- kernel/sched/ghost_core.c | 186 ++------------------------------------ kernel/sched/sched.h | 9 +- 3 files changed, 152 insertions(+), 215 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index fcf05e989cb9..65c1ce1938cc 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6769,46 +6769,46 @@ static struct kernfs_ops gf_ops_e_ctl = { }; /* - * Returns the enclave for f, if f is a ghostfs ctl file. We could support - * other file types, but since this is a backdoor into the FS, we only need to - * support ctl. - * - * Successful callers must call ghostfs_put_ctl_enclave(f). + * Returns a kreffed pointer for the enclave for f, if f is a ghostfs ctl file. + * NULL otherwise. * * The kernfs_ops don't need this helper. kernfs manages the the refcounts. We * need to do it manually here, because this is is a "backdoor" function to get * the enclave pointer. That pointer is kept alive by kernfs. */ -static struct ghost_enclave *ctlfd_enclave_get(struct file *f) +static struct ghost_enclave *ctlfd_enclave_get_kref(int ctl_fd) { + struct fd fd; struct kernfs_node *kn; + struct ghost_enclave *e = NULL; - kn = kernfs_node_from_file(f); + fd = fdget(ctl_fd); + /* + * You can fdput even if there is no fd->file. + * You must fdput if there is a file. + */ + if (!fd.file) + goto out_fdput; + kn = kernfs_node_from_file(fd.file); if (!kn) - return NULL; + goto out_fdput; if (kernfs_type(kn) != KERNFS_FILE) - return NULL; + goto out_fdput; if (kn->attr.ops != &gf_ops_e_ctl) - return NULL; + goto out_fdput; if (!kernfs_get_active(kn)) - return NULL; - WARN_ON(!kn->priv); - return kn->priv; -} - -/* Pair this with a successful ctlfd_enclave_get() call. */ -static void ctlfd_enclave_put(struct file *f) -{ - struct kernfs_node *kn; - - kn = kernfs_node_from_file(f); - if (WARN_ON(!kn)) - return; - if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) - return; - if (WARN_ON(kn->attr.ops != &gf_ops_e_ctl)) - return; + goto out_fdput; + e = kn->priv; + if (e) + kref_get(&e->kref); + else + WARN_ONCE(1, "ctlfd had no kn->priv!"); + /* We only need the active protection until we up the kref */ kernfs_put_active(kn); + +out_fdput: + fdput(fd); + return e; } static int gf_runnable_timeout_show(struct seq_file *sf, void *v) @@ -7551,14 +7551,125 @@ static void bpf_link_detach(struct ghost_enclave *e, struct bpf_prog *prog, } } +struct bpf_ghost_link { + struct bpf_link link; + struct ghost_enclave *e; + enum bpf_prog_type prog_type; + enum bpf_attach_type ea_type; +}; + +static void bpf_ghost_link_dealloc(struct bpf_link *link) +{ + struct bpf_ghost_link *sc_link = + container_of(link, struct bpf_ghost_link, link); + + kfree(sc_link); +} + +static void bpf_ghost_link_release(struct bpf_link *link) +{ + struct bpf_ghost_link *sc_link = + container_of(link, struct bpf_ghost_link, link); + struct ghost_enclave *e = sc_link->e; + + if (WARN_ONCE(!e, "Missing enclave for bpf link ea_type %d!", + sc_link->ea_type)) + return; + + bpf_link_detach(e, link->prog, sc_link->prog_type, sc_link->ea_type); + + kref_put(&e->kref, enclave_release); + sc_link->e = NULL; +} + +static const struct bpf_link_ops bpf_ghost_link_ops = { + .release = bpf_ghost_link_release, + .dealloc = bpf_ghost_link_dealloc, +}; + +static int _ghost_bpf_link_attach(const union bpf_attr *attr, + struct bpf_prog *prog, + int ea_type, int ea_abi) +{ + struct bpf_link_primer link_primer; + struct bpf_ghost_link *sc_link; + struct ghost_enclave *e; + int err; + + switch (prog->type) { + case BPF_PROG_TYPE_GHOST_SCHED: + switch (ea_type) { + case BPF_GHOST_SCHED_SKIP_TICK: + case BPF_GHOST_SCHED_PNT: + break; + default: + return -EINVAL; + } + break; + case BPF_PROG_TYPE_GHOST_MSG: + switch (ea_type) { + case BPF_GHOST_MSG_SEND: + break; + default: + return -EINVAL; + } + break; + default: + return -EINVAL; + } + + sc_link = kzalloc(sizeof(*sc_link), GFP_USER); + if (!sc_link) + return -ENOMEM; + bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, + &bpf_ghost_link_ops, prog); + sc_link->prog_type = prog->type; + sc_link->ea_type = ea_type; + + err = bpf_link_prime(&sc_link->link, &link_primer); + if (err) { + kfree(sc_link); + return -EINVAL; + } + + e = ctlfd_enclave_get_kref(attr->link_create.target_fd); + if (!e) { + /* bpf_link_cleanup() triggers .dealloc, but not .release. */ + bpf_link_cleanup(&link_primer); + return -EBADF; + } + /* ABI bug for us to have a false positive fd->enclave lookup. */ + if (WARN_ON(ea_abi != enclave_abi(e))) { + bpf_link_cleanup(&link_primer); + kref_put(&e->kref, enclave_release); + return -EBADF; + } + + /* + * On success, sc_link will hold the kref we got from + * ctlfd_enclave_get_kref(), which will get put when the link's FD is + * closed (and thus bpf_link_put -> bpf_link_free -> our release). This + * is similar to how ghostfs files hold a kref on the enclave. Release + * is not called on failure. + */ + sc_link->e = e; + err = bpf_link_attach(e, prog, prog->type, ea_type); + if (err) { + bpf_link_cleanup(&link_primer); + kref_put(&e->kref, enclave_release); + return err; + } + + return bpf_link_settle(&link_primer); +} + + DEFINE_GHOST_ABI(current_abi) = { .version = GHOST_VERSION, .abi_init = abi_init, .create_enclave = create_enclave, .enclave_release = enclave_release, .enclave_add_cpu = ___enclave_add_cpu, - .ctlfd_enclave_get = ctlfd_enclave_get, - .ctlfd_enclave_put = ctlfd_enclave_put, .setscheduler = _ghost_setscheduler, .fork = _ghost_sched_fork, .cleanup_fork = _ghost_sched_cleanup_fork, @@ -7575,8 +7686,7 @@ DEFINE_GHOST_ABI(current_abi) = { .bpf_run_gtid = bpf_run_gtid, .bpf_resched_cpu = bpf_resched_cpu, .ghost_msg_is_valid_access = ghost_msg_is_valid_access, - .bpf_link_attach = bpf_link_attach, - .bpf_link_detach = bpf_link_detach, + .bpf_link_attach = _ghost_bpf_link_attach, /* ghost_agent_sched_class callbacks */ .pick_next_ghost_agent = pick_next_ghost_agent, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 25be66a4c3d0..012a41a652dd 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -459,73 +459,6 @@ void __init init_sched_ghost_class(void) per_cpu(sync_group_cookie, cpu) = cpu << SG_COOKIE_CPU_SHIFT; } -/* - * Helper to get an fd and translate it to an enclave. Ghostfs will maintain a - * kref on the enclave. That kref is increffed when the file was opened, and - * that kref is maintained by this call (a combination of fdget and - * kernfs_get_active). Once you call ghost_fdput_enclave(), the enclave kref - * can be released. - * - * Returns NULL if there is not a ghost_enclave for this FD, which could be due - * to concurrent enclave destruction. - * - * Caller must call ghost_fdput_enclave with e and &fd_to_put, even on error. - */ -static struct ghost_enclave *ghost_fdget_enclave(int fd, struct fd *fd_to_put) -{ - const struct ghost_abi *abi; - struct ghost_enclave *e = NULL; - - static const struct ghost_abi *last_abi; - - *fd_to_put = fdget(fd); - if (!fd_to_put->file) - return NULL; - - /* - * Try the ABI that we last used successfully to first do the lookup. - * - * In the common case of a single active enclave we avoid looping over - * all the ABIs compiled into the kernel. - */ - abi = READ_ONCE(last_abi); - if (abi) { - e = abi->ctlfd_enclave_get(fd_to_put->file); - if (e) - return e; - } - - for_each_abi(abi) { - e = abi->ctlfd_enclave_get(fd_to_put->file); - if (e) { - /* - * Update 'last_abi' if necessary. - * - * N.B. multiple CPUs may race to mutate 'last_abi' - * concurrently. This is okay as long as each update - * is atomic (i.e. last_abi always points to a valid - * abi). - */ - if (abi != READ_ONCE(last_abi)) - WRITE_ONCE(last_abi, abi); - - break; - } - } - return e; -} - -/* - * Pairs with calls to ghost_fdget_enclave(). You can't call this while holding - * the rq lock. - */ -static void ghost_fdput_enclave(struct ghost_enclave *e, struct fd *fd) -{ - if (e) - e->abi->ctlfd_enclave_put(fd->file); - fdput(*fd); -} - int ghost_setscheduler(struct task_struct *p, struct rq *rq, const struct sched_attr *attr, int *reset_on_fork) @@ -1289,125 +1222,22 @@ const struct bpf_verifier_ops ghost_msg_verifier_ops = { const struct bpf_prog_ops ghost_msg_prog_ops = {}; -struct bpf_ghost_link { - struct bpf_link link; - struct ghost_enclave *e; - enum bpf_prog_type prog_type; - enum bpf_attach_type ea_type; -}; - -static void bpf_ghost_link_dealloc(struct bpf_link *link) +int ghost_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { - struct bpf_ghost_link *sc_link = - container_of(link, struct bpf_ghost_link, link); - - kfree(sc_link); -} - -static void bpf_ghost_link_release(struct bpf_link *link) -{ - struct bpf_ghost_link *sc_link = - container_of(link, struct bpf_ghost_link, link); - struct ghost_enclave *e = sc_link->e; - - if (WARN_ONCE(!e, "Missing enclave for bpf link ea_type %d!", - sc_link->ea_type)) - return; - - e->abi->bpf_link_detach(e, link->prog, - sc_link->prog_type, sc_link->ea_type); - - kref_put(&e->kref, e->abi->enclave_release); - sc_link->e = NULL; -} - -static const struct bpf_link_ops bpf_ghost_link_ops = { - .release = bpf_ghost_link_release, - .dealloc = bpf_ghost_link_dealloc, -}; - -int ghost_bpf_link_attach(const union bpf_attr *attr, - struct bpf_prog *prog) -{ - struct bpf_link_primer link_primer; - struct bpf_ghost_link *sc_link; + const struct ghost_abi *abi; enum bpf_attach_type ea_type; - struct ghost_enclave *e; - struct fd f_enc = {0}; - int err, ea_abi; - - if (attr->link_create.flags) - return -EINVAL; - if (prog->expected_attach_type != attr->link_create.attach_type) - return -EINVAL; + int ea_abi; - /* Mask the ABI value encoded in the upper 16 bits. */ ea_type = bpf_prog_eat_type(prog); ea_abi = bpf_prog_eat_abi(prog); - - switch (prog->type) { - case BPF_PROG_TYPE_GHOST_SCHED: - switch (ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - case BPF_GHOST_SCHED_PNT: - break; - default: - return -EINVAL; - } - break; - case BPF_PROG_TYPE_GHOST_MSG: - switch (ea_type) { - case BPF_GHOST_MSG_SEND: - break; - default: - return -EINVAL; - } - break; - default: + if (attr->link_create.flags) return -EINVAL; - } - - sc_link = kzalloc(sizeof(*sc_link), GFP_USER); - if (!sc_link) - return -ENOMEM; - bpf_link_init(&sc_link->link, BPF_LINK_TYPE_UNSPEC, - &bpf_ghost_link_ops, prog); - sc_link->prog_type = prog->type; - sc_link->ea_type = ea_type; - - err = bpf_link_prime(&sc_link->link, &link_primer); - if (err) { - kfree(sc_link); + if (prog->expected_attach_type != attr->link_create.attach_type) return -EINVAL; - } - e = ghost_fdget_enclave(attr->link_create.target_fd, &f_enc); - if (!e || ea_abi != enclave_abi(e)) { - ghost_fdput_enclave(e, &f_enc); - /* bpf_link_cleanup() triggers .dealloc, but not .release. */ - bpf_link_cleanup(&link_primer); + abi = ghost_abi_lookup(ea_abi); + if (!abi) return -EBADF; - } - - /* - * On success, sc_link will hold a kref on the enclave, which will get - * put when the link's FD is closed (and thus bpf_link_put -> - * bpf_link_free -> our release). This is similar to how ghostfs files - * hold a kref on the enclave. Release is not called on failure. - */ - kref_get(&e->kref); - sc_link->e = e; - ghost_fdput_enclave(e, &f_enc); - - err = e->abi->bpf_link_attach(e, prog, prog->type, ea_type); - - if (err) { - /* bpf_link_cleanup() triggers .dealloc, but not .release. */ - bpf_link_cleanup(&link_primer); - kref_put(&e->kref, e->abi->enclave_release); - return err; - } - - return bpf_link_settle(&link_primer); + return abi->bpf_link_attach(attr, prog, ea_type, ea_abi); } #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 288dd20b5312..23e221a70399 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2178,8 +2178,6 @@ struct ghost_abi { struct kernfs_node *dir, ulong id); void (*enclave_release)(struct kref *k); void (*enclave_add_cpu)(struct ghost_enclave *e, int cpu); - struct ghost_enclave *(*ctlfd_enclave_get)(struct file *file); - void (*ctlfd_enclave_put)(struct file *file); int (*setscheduler)(struct ghost_enclave *e, struct task_struct *p, struct rq *rq, const struct sched_attr *attr, int *reset_on_fork); @@ -2203,10 +2201,9 @@ struct ghost_abi { enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info); - int (*bpf_link_attach)(struct ghost_enclave *e, struct bpf_prog *prog, - int prog_type, int attach_type); - void (*bpf_link_detach)(struct ghost_enclave *e, struct bpf_prog *prog, - int prog_type, int attach_type); + int (*bpf_link_attach)(const union bpf_attr *attr, + struct bpf_prog *prog, + int ea_type, int ea_abi); /* ghost_agent_sched_class callbacks */ struct task_struct *(*pick_next_ghost_agent)(struct rq *rq); From d345fb718937cfc71b4dd5784a655c9c8739bb83 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 27 Jan 2022 15:16:42 -0500 Subject: [PATCH 150/165] sched/ghost: split out bpf helper ID definitions enum bpf_func_id is created by a macro that is used for an array lookup from helper ID to string (func_id_str in disasm.c). We don't want that array to be too big. However, we also don't want our helper ID numbers to clash with upstream's function IDs. That will happen eventually. Move the ghost bpf helper IDs out of bpf_func_id so that we can use a larger number without affecting the array. The array mapping is optional - it's used in func_id-to-name lookups, and those lookups are protected in case the func_id > __BPF_FUNC_MAX_ID. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I5b221d1834f1afe11aed55f73216b9e3d8915f46 --- include/uapi/linux/bpf.h | 106 ++++++++++++--------------------- kernel/bpf/verifier.c | 6 +- kernel/sched/ghost_core.c | 6 +- scripts/bpf_helpers_doc.py | 27 +++++++++ tools/include/uapi/linux/bpf.h | 39 ++++++++++++ 5 files changed, 114 insertions(+), 70 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 9090dcd43571..16dad79edfdd 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3836,29 +3836,6 @@ union bpf_attr { * A pointer to a struct socket on success or NULL if the file is * not a socket. * - * long bpf_ghost_wake_agent(u32 cpu) - * Description - * Wakes the ghost agent on *cpu* - * - * Return - * 0 on success, < 0 on error. - * - * long bpf_ghost_run_gtid(s64 gtid, u32 task_barrier, s32 run_flags) - * Description - * Runs (latches) task **gtid** on this cpu. - * - * Return - * 0 on success, < 0 on error. - * - * long bpf_ghost_resched_cpu(u32 cpu, u64 cpu_seqnum) - * Description - * Reschedules **cpu** if its state is still **cpu_seqnum**, such - * that it calls pick_next_task and will not pick prev again. - * Typically, it will run BPF-PNT. - * - * Return - * 0 on success, < 0 on error. - * */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -4024,50 +4001,6 @@ union bpf_attr { FN(ktime_get_coarse_ns), \ FN(ima_inode_hash), \ FN(sock_from_file), \ - FN(placeholder_998), \ - FN(placeholder_999), \ - FN(placeholder_001), \ - FN(placeholder_002), \ - FN(placeholder_003), \ - FN(placeholder_004), \ - FN(placeholder_005), \ - FN(placeholder_006), \ - FN(placeholder_007), \ - FN(placeholder_008), \ - FN(placeholder_009), \ - FN(placeholder_010), \ - FN(placeholder_011), \ - FN(placeholder_012), \ - FN(placeholder_013), \ - FN(placeholder_014), \ - FN(placeholder_015), \ - FN(placeholder_016), \ - FN(placeholder_017), \ - FN(placeholder_018), \ - FN(placeholder_019), \ - FN(placeholder_020), \ - FN(placeholder_021), \ - FN(placeholder_022), \ - FN(placeholder_023), \ - FN(placeholder_024), \ - FN(placeholder_025), \ - FN(placeholder_026), \ - FN(placeholder_027), \ - FN(placeholder_028), \ - FN(placeholder_029), \ - FN(placeholder_030), \ - FN(placeholder_031), \ - FN(placeholder_032), \ - FN(placeholder_033), \ - FN(placeholder_034), \ - FN(placeholder_035), \ - FN(placeholder_036), \ - FN(placeholder_037), \ - FN(placeholder_038), \ - FN(placeholder_039), \ - FN(ghost_wake_agent), \ - FN(ghost_run_gtid), \ - FN(ghost_resched_cpu), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -4080,6 +4013,45 @@ enum bpf_func_id { }; #undef __BPF_ENUM_FN +/* + * The formatting of these is particular so that bpf_doc.py can parse them, just + * like the normal BPF helper descriptions. The first function needs to be + * called bpf_ghost. If you change __BPF_FUNC_GHOST_BASE, change bpf_doc.py + * too. + * + * Start of Ghost BPF helper function descriptions: + * + * long bpf_ghost_wake_agent(u32 cpu) + * Description + * Wakes the ghost agent on **cpu**. + * + * Return + * 0 on success, < 0 on error. + * + * long bpf_ghost_run_gtid(s64 gtid, u32 task_barrier, s32 run_flags) + * Description + * Runs (latches) task **gtid** on this cpu. + * + * Return + * 0 on success, < 0 on error. + * + * long bpf_ghost_resched_cpu(u32 cpu, u64 cpu_seqnum) + * Description + * Reschedules **cpu** if its state is still **cpu_seqnum**, such + * that it calls pick_next_task and will not pick prev again. + * Typically, it will run BPF-PNT. + * + * Return + * 0 on success, < 0 on error. + */ +enum { + __BPF_FUNC_GHOST_BASE = 204, + BPF_FUNC_ghost_wake_agent = __BPF_FUNC_GHOST_BASE, + BPF_FUNC_ghost_run_gtid, + BPF_FUNC_ghost_resched_cpu, + __BPF_FUNC_GHOST_MAX_ID, +}; + /* All flags used by eBPF helper functions, placed here. */ /* BPF_FUNC_skb_store_bytes flags. */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7bcb4b369d38..f613e9d82661 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5053,7 +5053,11 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn int i, err; /* find function prototype */ - if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { + BUILD_BUG_ON(__BPF_FUNC_MAX_ID > __BPF_FUNC_GHOST_BASE); + + if (func_id < 0 || (func_id >= __BPF_FUNC_MAX_ID && + !(func_id >= __BPF_FUNC_GHOST_BASE && + func_id < __BPF_FUNC_GHOST_MAX_ID))) { verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id); return -EINVAL; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 012a41a652dd..2b3ee7d2b4fe 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1154,7 +1154,8 @@ ghost_sched_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { int eat = bpf_prog_eat_type(prog); - switch (func_id) { + /* ghost func_ids are not in enum bpf_func_id */ + switch ((int)func_id) { case BPF_FUNC_ghost_wake_agent: return &bpf_ghost_wake_agent_proto; case BPF_FUNC_ghost_run_gtid: @@ -1191,7 +1192,8 @@ const struct bpf_prog_ops ghost_sched_prog_ops = {}; static const struct bpf_func_proto * ghost_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { - switch (func_id) { + /* ghost func_ids are not in enum bpf_func_id */ + switch ((int)func_id) { case BPF_FUNC_ghost_wake_agent: return &bpf_ghost_wake_agent_proto; case BPF_FUNC_ghost_resched_cpu: diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index d7f88bd35e1f..d195f039c5df 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -157,6 +157,24 @@ def run(self): except NoHelperFound: break + # Ghost helpers are separate to avoid func_id collisions + self.reader.seek(0) + offset = self.reader.read().find('* Start of Ghost BPF helper function descriptions:') + + if offset != -1: + self.reader.seek(offset) + self.reader.readline() + self.reader.readline() + self.line = self.reader.readline() + + while True: + try: + helper = self.parse_helper() + self.helpers.append(helper) + except NoHelperFound: + break + # end of ghost hack + self.reader.close() ############################################################################### @@ -541,6 +559,15 @@ def print_one(self, helper): if proto['name'] in self.seen_helpers: return + + # Ghost hack: the len of seen_helpers determines the func_id we print. + # Extend it to __BPF_FUNC_GHOST_BASE - 1 + to_len = 204 - 1 + if "bpf_ghost" in proto['name'] and len(self.seen_helpers) < to_len: + for i in range(0, to_len - len(self.seen_helpers)): + self.seen_helpers.add("fake_" + str(i)) + # end of ghost hack + self.seen_helpers.add(proto['name']) print('/*') diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 77d7c1bb2923..a2939dd4817c 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4007,6 +4007,45 @@ enum bpf_func_id { }; #undef __BPF_ENUM_FN +/* + * The formatting of these is particular so that bpf_doc.py can parse them, just + * like the normal BPF helper descriptions. The first function needs to be + * called bpf_ghost. If you change __BPF_FUNC_GHOST_BASE, change bpf_doc.py + * too. + * + * Start of Ghost BPF helper function descriptions: + * + * long bpf_ghost_wake_agent(u32 cpu) + * Description + * Wakes the ghost agent on **cpu**. + * + * Return + * 0 on success, < 0 on error. + * + * long bpf_ghost_run_gtid(s64 gtid, u32 task_barrier, s32 run_flags) + * Description + * Runs (latches) task **gtid** on this cpu. + * + * Return + * 0 on success, < 0 on error. + * + * long bpf_ghost_resched_cpu(u32 cpu, u64 cpu_seqnum) + * Description + * Reschedules **cpu** if its state is still **cpu_seqnum**, such + * that it calls pick_next_task and will not pick prev again. + * Typically, it will run BPF-PNT. + * + * Return + * 0 on success, < 0 on error. + */ +enum { + __BPF_FUNC_GHOST_BASE = 204, + BPF_FUNC_ghost_wake_agent = __BPF_FUNC_GHOST_BASE, + BPF_FUNC_ghost_run_gtid, + BPF_FUNC_ghost_resched_cpu, + __BPF_FUNC_GHOST_MAX_ID, +}; + /* All flags used by eBPF helper functions, placed here. */ /* BPF_FUNC_skb_store_bytes flags. */ From e80efd175033c8699e12093354e945aa87bb76a6 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 27 Jan 2022 15:59:09 -0500 Subject: [PATCH 151/165] sched/ghost: increase bpf magic numbers With the growing number of BPF helpers, we'll soon run out of space in the bpf_func_id space. Increase the base numbers for PROG_TYPEs, the ATTACH_TYPEs, and helpers: - progs: 1000 - attach: 2000 - helpers: 3000 There is enough room for growth in upstream's numbers that we'll either be merged upstream or dead before we run into conflicts. We're still well under the 0xffff requirement for attach types (since the upper bits are the enclave ABI). To be submitted with cl/424689285. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ie0600e0b4478d3289377e4a4d58c9e13b4a491aa --- include/uapi/linux/bpf.h | 6 +++--- include/uapi/linux/ghost.h | 2 +- kernel/sched/ghost_core.c | 6 +++--- scripts/bpf_helpers_doc.py | 2 +- tools/include/uapi/linux/bpf.h | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 16dad79edfdd..77aa4f788c54 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -200,7 +200,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_EXT, BPF_PROG_TYPE_LSM, BPF_PROG_TYPE_SK_LOOKUP, - BPF_PROG_TYPE_GHOST_SCHED = 35, + BPF_PROG_TYPE_GHOST_SCHED = 1000, BPF_PROG_TYPE_GHOST_MSG, }; @@ -243,7 +243,7 @@ enum bpf_attach_type { BPF_XDP_CPUMAP, BPF_SK_LOOKUP, BPF_XDP, - BPF_GHOST_SCHED_SKIP_TICK = 50, + BPF_GHOST_SCHED_SKIP_TICK = 2000, BPF_GHOST_SCHED_PNT, BPF_GHOST_MSG_SEND, __MAX_BPF_ATTACH_TYPE @@ -4045,7 +4045,7 @@ enum bpf_func_id { * 0 on success, < 0 on error. */ enum { - __BPF_FUNC_GHOST_BASE = 204, + __BPF_FUNC_GHOST_BASE = 3000, BPF_FUNC_ghost_wake_agent = __BPF_FUNC_GHOST_BASE, BPF_FUNC_ghost_run_gtid, BPF_FUNC_ghost_resched_cpu, diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 6e80f1760614..41664bc4f690 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 64 +#define GHOST_VERSION 65 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 2b3ee7d2b4fe..e18e181b9e37 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -1074,7 +1074,7 @@ BPF_CALL_1(bpf_ghost_wake_agent, u32, cpu) * similar to syscall numbers. Catch any ABI inconsistencies between * prodkernel and open source. */ - BUILD_BUG_ON(BPF_FUNC_ghost_wake_agent != 204); + BUILD_BUG_ON(BPF_FUNC_ghost_wake_agent != 3000); e = get_target_enclave(); @@ -1103,7 +1103,7 @@ BPF_CALL_3(bpf_ghost_run_gtid, s64, gtid, u32, task_barrier, int, run_flags) * similar to syscall numbers. Catch any ABI inconsistencies between * prodkernel and open source. */ - BUILD_BUG_ON(BPF_FUNC_ghost_run_gtid != 205); + BUILD_BUG_ON(BPF_FUNC_ghost_run_gtid != 3001); e = get_target_enclave(); @@ -1130,7 +1130,7 @@ BPF_CALL_2(bpf_ghost_resched_cpu, u32, cpu, u64, cpu_seqnum) VM_BUG_ON(preemptible()); - BUILD_BUG_ON(BPF_FUNC_ghost_resched_cpu != 206); + BUILD_BUG_ON(BPF_FUNC_ghost_resched_cpu != 3002); e = get_target_enclave(); diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index d195f039c5df..c872fa4988da 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -562,7 +562,7 @@ def print_one(self, helper): # Ghost hack: the len of seen_helpers determines the func_id we print. # Extend it to __BPF_FUNC_GHOST_BASE - 1 - to_len = 204 - 1 + to_len = 3000 - 1 if "bpf_ghost" in proto['name'] and len(self.seen_helpers) < to_len: for i in range(0, to_len - len(self.seen_helpers)): self.seen_helpers.add("fake_" + str(i)) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index a2939dd4817c..7933f635a990 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4039,7 +4039,7 @@ enum bpf_func_id { * 0 on success, < 0 on error. */ enum { - __BPF_FUNC_GHOST_BASE = 204, + __BPF_FUNC_GHOST_BASE = 3000, BPF_FUNC_ghost_wake_agent = __BPF_FUNC_GHOST_BASE, BPF_FUNC_ghost_run_gtid, BPF_FUNC_ghost_resched_cpu, From db41b56ac7f4296644b8bb8ead39ce330462ad59 Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 15 Jun 2022 15:24:42 -0700 Subject: [PATCH 152/165] BPF fixes Move the ghost attach types to their own enum. Change-Id: I91ad42032a21ead9dd99741040f26fbfcc4004a2 --- include/uapi/linux/bpf.h | 14 +++++++++++++- kernel/bpf/syscall.c | 11 ++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 77aa4f788c54..8a8a56f0ba59 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -243,10 +243,22 @@ enum bpf_attach_type { BPF_XDP_CPUMAP, BPF_SK_LOOKUP, BPF_XDP, + __MAX_BPF_ATTACH_TYPE, +}; + +enum { + /* + * There are arrays in the kernel that use MAX_BPF_ATTACH_TYPE, notably + * in kernel/bpf/cgroup.c. That code will essentially ignore ghost + * programs. Note that bpftool won't know about our types either. + * + * If we ever merge upstream, we can go back below + * __MAX_BPF_ATTACH_TYPE. We have our own numbers to avoid rebase hell. + */ BPF_GHOST_SCHED_SKIP_TICK = 2000, BPF_GHOST_SCHED_PNT, BPF_GHOST_MSG_SEND, - __MAX_BPF_ATTACH_TYPE + __MAX_BPF_GHOST_ATTACH_TYPE }; #define MAX_BPF_ATTACH_TYPE __MAX_BPF_ATTACH_TYPE diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index ff69d7fc77bd..fc5b5626c351 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1929,7 +1929,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, struct bpf_prog *dst_prog) { #ifdef CONFIG_SCHED_CLASS_GHOST - BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE > 0xFFFF); + BUILD_BUG_ON(__MAX_BPF_GHOST_ATTACH_TYPE > 0xFFFF); expected_attach_type &= 0xFFFF; #endif @@ -2008,7 +2008,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, return 0; return -EINVAL; case BPF_PROG_TYPE_GHOST_SCHED: - switch (expected_attach_type) { + switch ((int)expected_attach_type) { case BPF_GHOST_SCHED_SKIP_TICK: case BPF_GHOST_SCHED_PNT: return 0; @@ -2016,7 +2016,7 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, return -EINVAL; } case BPF_PROG_TYPE_GHOST_MSG: - switch (expected_attach_type) { + switch ((int)expected_attach_type) { case BPF_GHOST_MSG_SEND: return 0; default: @@ -2927,11 +2927,12 @@ static enum bpf_prog_type attach_type_to_prog_type(enum bpf_attach_type attach_type) { #ifdef CONFIG_SCHED_CLASS_GHOST - BUILD_BUG_ON(__MAX_BPF_ATTACH_TYPE > 0xFFFF); + BUILD_BUG_ON(__MAX_BPF_GHOST_ATTACH_TYPE > 0xFFFF); attach_type &= 0xFFFF; #endif - switch (attach_type) { + /* Cast to int for ghost attach types */ + switch ((int)attach_type) { case BPF_CGROUP_INET_INGRESS: case BPF_CGROUP_INET_EGRESS: return BPF_PROG_TYPE_CGROUP_SKB; From 4b300a188843fbb73c2f7dcb1db6645b44adb688 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 3 Feb 2022 14:34:47 -0500 Subject: [PATCH 153/165] sched/ghost: pass the "create" cmd buffer to the abi This lets us do abi-specific parsing of the create command. Specifically, we can add abi-specific commands after the ABI part of "create $ID $ABI". Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: If9a8d678ec8a3072c83768672c57523a04cb3d2e Origin-9xx-SHA1: b549601a7b45001f48237968d9ba3c8b1cfc56a6 Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 2 +- kernel/sched/ghost_core.c | 15 +++++++++------ kernel/sched/sched.h | 3 ++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 65c1ce1938cc..41e82e273828 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -7134,7 +7134,7 @@ static int __init abi_init(const struct ghost_abi *abi) static struct ghost_enclave *create_enclave(const struct ghost_abi *abi, struct kernfs_node *dir, - ulong id) + ulong id, const char *cmd_extra) { bool vmalloc_failed = false; struct ghost_enclave *e; diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index e18e181b9e37..c9fab0d75df2 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -58,7 +58,7 @@ static const struct ghost_abi *ghost_abi_lookup(uint version) } static int make_enclave(struct kernfs_node *parent, unsigned long id, - unsigned int version) + unsigned int version, const char *cmd_extra) { struct kernfs_node *dir; struct ghost_enclave *e; @@ -82,7 +82,7 @@ static int make_enclave(struct kernfs_node *parent, unsigned long id, * discoverable, usable, or otherwise hooked into the kernel until * kernfs_active(). */ - e = abi->create_enclave(abi, dir, id); + e = abi->create_enclave(abi, dir, id, cmd_extra); if (IS_ERR(e)) { kernfs_remove(dir); /* recursive */ return PTR_ERR(e); @@ -106,14 +106,17 @@ static ssize_t gf_top_ctl_write(struct kernfs_open_file *of, char *buf, struct kernfs_node *top_dir = ctl->parent; unsigned long x; unsigned int abi_num; - int ret; + int ret, so_far; gf_strip_slash_n(buf, len); - /* This will ignore any extra digits or characters beyond the %u. */ - ret = sscanf(buf, "create %lu %u", &x, &abi_num); + /* + * Any extra spaces, digits, or characters beyond the abi_num are passed + * to the abi's create_enclave(). + */ + ret = sscanf(buf, "create %lu %u%n", &x, &abi_num, &so_far); if (ret == 2) { - ret = make_enclave(top_dir, x, abi_num); + ret = make_enclave(top_dir, x, abi_num, buf + so_far); return ret ? ret : len; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 23e221a70399..0b464a5d0e01 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2175,7 +2175,8 @@ struct ghost_abi { int (*abi_init)(const struct ghost_abi *abi); struct ghost_enclave * (*create_enclave)(const struct ghost_abi *abi, - struct kernfs_node *dir, ulong id); + struct kernfs_node *dir, ulong id, + const char *cmd_extra); void (*enclave_release)(struct kref *k); void (*enclave_add_cpu)(struct ghost_enclave *e, int cpu); int (*setscheduler)(struct ghost_enclave *e, struct task_struct *p, From 4bc32e6f3ffd8af81834bbf33a053ee30ea29d72 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 3 Feb 2022 15:47:50 -0500 Subject: [PATCH 154/165] sched/ghost: set uid/gid when creating an enclave MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an optional parameter to "create" for a GID of the new enclave. Default is root. Use the caller's EUID as the UID for the enclave. This parameter to create is ABI-dependent, since it is after version parsing, meaning we can change it easily. In an effort to run the agent as someone other than root, we'd like to chown the files in the enclave_dir to whoever will run there. Prior to this commit, all ghostfs files are owned by root:root. Additionally, we'd like to restrict the agent from changing the cpus. We can chmod g-w, so long as the agent is not the owner. Putting these together, the agent can use the group access bits, and whoever creates the enclave (some trusted daemon) can be the owner. The reason to make the creator the owner is so that they can chmod cpulist and cpumask. Example: enclave ID = 565, ABI = 65, desired GID = 716 (root):/sys/fs/ghost# echo create 565 65 716 > ctl (root):/sys/fs/ghost# echo 0-10 > enclave_565/cpulist (root):/sys/fs/ghost# chmod g-w enclave_565/cpulist (root):/sys/fs/ghost# chmod g-w enclave_565/cpumask (root):/sys/fs/ghost# ls -la enclave_565/ total 0 dr-xr-xr-x 3 root spaceghostd 0 Feb 3 13:02 . dr-xr-xr-x 3 root root 0 Feb 3 12:49 .. -r--r--r-- 1 root spaceghostd 0 Feb 3 13:02 abi_version -rw-rw-r-- 1 root spaceghostd 0 Feb 3 13:02 agent_online -rw-rw-r-- 1 root spaceghostd 0 Feb 3 13:02 commit_at_tick -rw-rw---- 1 root spaceghostd 458752 Feb 3 13:02 cpu_data -rw-r--r-- 1 root spaceghostd 0 Feb 3 13:09 cpulist -rw-r--r-- 1 root spaceghostd 0 Feb 3 13:02 cpumask -rw-rw-r-- 1 root spaceghostd 0 Feb 3 13:02 ctl -rw-rw-r-- 1 root spaceghostd 0 Feb 3 13:02 runnable_timeout -r--r--r-- 1 root spaceghostd 0 Feb 3 13:02 status -rw-rw-r-- 1 root spaceghostd 0 Feb 3 13:02 switchto_disabled dr-xr-xr-x 2 root spaceghostd 0 Feb 3 13:02 sw_regions -rw-rw-rw- 1 root spaceghostd 0 Feb 3 13:02 tasks -rw-rw-r-- 1 root spaceghostd 0 Feb 3 13:02 wake_on_waker_cpu (spaceghostd):/sys/fs/ghost/enclave_565$ chmod g+w cpulist chmod: changing permissions of ‘cpulist’: Operation not permitted (spaceghostd):/sys/fs/ghost/enclave_565$ cat cpulist 0-10 (spaceghostd):/tmp/$ ./agent_muppet --enclave /sys/fs/ghost/enclave_565/ That actually fails due to missing CAP_BPF, but the ghostfs stuff worked. Note that to create an enclave, you need write access to ghostfs/ctl, which is owned root:root. Tested: creating enclave and attached to it, edf_test, agent_muppet Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I93a6cca6c29e7a318460ef599c0b269d516ec4f2 Origin-9xx-SHA1: 799b9eb05be77331ec9b64eb9abe8dcc2b40182c Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 35 +++++++++++++++++++++++++++++------ kernel/sched/ghost_core.c | 20 ++++++++++++++++++++ kernel/sched/sched.h | 4 ++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 41e82e273828..2bbf4c32de1f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6631,7 +6631,8 @@ static int create_sw_region(struct kernfs_open_file *ctl_of, unsigned int id, goto err_snprintf; } - swr_kn = kernfs_create_file(dir, name, 0440, 0, &gf_ops_e_swr, e); + swr_kn = kernfs_create_file_ns(dir, name, 0440, e->uid, e->gid, 0, + &gf_ops_e_swr, e, NULL); if (IS_ERR(swr_kn)) { err = PTR_ERR(swr_kn); goto err_create_kn; @@ -7076,18 +7077,19 @@ static struct gf_dirent enclave_dirtab[] = { /* Caller is responsible for cleanup. Removing the parent will suffice. */ static int gf_add_files(struct kernfs_node *parent, struct gf_dirent *dirtab, - struct ghost_enclave *priv) + struct ghost_enclave *e) { struct gf_dirent *gft; struct kernfs_node *kn; for (gft = dirtab; gft->name; gft++) { if (gft->is_dir) { - kn = kernfs_create_dir(parent, gft->name, gft->mode, - NULL); + kn = kernfs_create_dir_ns(parent, gft->name, gft->mode, + e->uid, e->gid, NULL, NULL); } else { - kn = kernfs_create_file(parent, gft->name, gft->mode, - gft->size, gft->ops, priv); + kn = kernfs_create_file_ns(parent, gft->name, gft->mode, + e->uid, e->gid, gft->size, + gft->ops, e, NULL); } if (IS_ERR(kn)) return PTR_ERR(kn); @@ -7132,6 +7134,19 @@ static int __init abi_init(const struct ghost_abi *abi) return 0; } +static kgid_t parse_cmd_gid(const char *cmd_extra) +{ + gid_t gid; + kgid_t kgid = GLOBAL_ROOT_GID; + int ret; + + /* sscanf will snarf the leading whitespace, unlike kstrtoint. */ + ret = sscanf(cmd_extra, "%u", &gid); + if (ret == 1) + kgid = KGIDT_INIT(gid); + return kgid; +} + static struct ghost_enclave *create_enclave(const struct ghost_abi *abi, struct kernfs_node *dir, ulong id, const char *cmd_extra) @@ -7185,6 +7200,14 @@ static struct ghost_enclave *create_enclave(const struct ghost_abi *abi, return ERR_PTR(-ENOMEM); } + e->uid = current_euid(); + e->gid = parse_cmd_gid(cmd_extra); + ret = ghostfs_set_ugid(dir, e->uid, e->gid); + if (ret) { + kref_put(&e->kref, enclave_release); + return ERR_PTR(ret); + } + ret = gf_add_files(dir, enclave_dirtab, e); if (ret) { kref_put(&e->kref, enclave_release); diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index c9fab0d75df2..537de08d77f0 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -45,6 +45,26 @@ static void gf_strip_slash_n(char *buf, size_t count) *n = '\0'; } +/* Helper to set the uid/gid of a kn */ +int ghostfs_set_ugid(struct kernfs_node *kn, kuid_t uid, kgid_t gid) +{ + struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = uid, + .ia_gid = gid, }; + + /* + * cgroup_kn_set_ugid() returns immediately if uid == gid == 0. That + * will work if we know the old uid/gid are 0, such as when a file is + * first created (default is GLOBAL_ROOT, both when making a new attr as + * well as if there are no attrs). + * + * In case we reuse this function for changing a kn from something else + * back to GLOBAL_ROOT, always call kernfs_setattr(). + */ + + return kernfs_setattr(kn, &iattr); +} + static const struct ghost_abi *ghost_abi_lookup(uint version) { const struct ghost_abi *abi; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0b464a5d0e01..0e2c457fcd8c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -197,6 +197,8 @@ struct ghost_enclave { int is_dying; bool agent_online; /* userspace says agent can schedule. */ struct kernfs_node *enclave_dir; + kuid_t uid; + kgid_t gid; #ifdef CONFIG_BPF struct bpf_prog *bpf_tick; @@ -2265,6 +2267,8 @@ struct ghost_enclave *get_target_enclave(void); struct ghost_enclave *set_target_enclave(struct ghost_enclave *e); void restore_target_enclave(struct ghost_enclave *old); +int ghostfs_set_ugid(struct kernfs_node *kn, kuid_t uid, kgid_t gid); + void init_sched_ghost_class(void); int ghost_add_cpus(struct ghost_enclave *e, const struct cpumask *cpus); void ghost_remove_cpu(struct ghost_enclave *e, int cpu); From 669f3f4f14b1f6551edb13ee49bd004336823fa0 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Mon, 7 Feb 2022 15:57:11 -0500 Subject: [PATCH 155/165] sched/ghost: always defer ghost_destroy_enclave to a CFS task It should come as no surprise that calling the old ghost_destroy_enclave() directly can cause issues. We already had infrastructure to destroy the enclave with a work_struct, which helped when destroying the enclave from tricky sched code (IRQ context). It turns out that we also need the work_struct to handle destroying the enclave from ghostfs! Why? Consider the case where the task destroying the enclave is in the enclave, e.g. bash joined the enclave. We'll hang after the synchronize_rcu() call. Essentially, we're still in the enclave, the agents were killed, and we need someone to schedule us. We haven't reached the part of the code that kicks all the enclave tasks back to CFS. The cleanest thing is to just defer the work to CFS, which we already do when we destroy the enclave from IRQ context. Now, anyone can call ghost_destroy_enclave() without worrying about their context or other race conditions. Even an agent task should be able to do it. Tested: edf_test, moved bash into an enclave and destroyed via ghostfs Effort: sched/ghost Google-Bug-Id: 216664048 Signed-off-by: Barret Rhoden Change-Id: Ief60ecfec4df05bab7d157f60e04d5e5f42a25bd Origin-9xx-SHA1: 28e5f4621e10640f34f6ac8b885239c4f51d0c51 Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 53 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 2bbf4c32de1f..324f12efeddb 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -838,6 +838,8 @@ static bool check_runnable_timeout(struct ghost_enclave *e, struct rq *rq) if (ktime_before(ktime_add_safe(p->ghost.last_runnable_at, timeout), ktime_get())) { + pr_info("enclave violation: enclave_%lu failed to run pid %u for over %lums\n", + e->id, p->pid, ktime_to_ms(timeout)); ok = false; } @@ -852,16 +854,8 @@ static void tick_handler(struct ghost_enclave *e, struct rq *rq) { if (READ_ONCE(e->commit_at_tick)) ghost_commit_all_greedy_txns(); - if (!check_runnable_timeout(e, rq)) { - kref_get(&e->kref); - if (!schedule_work(&e->enclave_destroyer)) { - /* - * Safe since it is not the last reference, due - * to RCU protecting per_cpu(enclave). - */ - kref_put(&e->kref, enclave_release); - } - } + if (!check_runnable_timeout(e, rq)) + ghost_destroy_enclave(e); } static void _task_tick_ghost(struct rq *rq, struct task_struct *p, int queued) @@ -1935,24 +1929,13 @@ static void enclave_reap_tasks(struct work_struct *work) kref_put(&e->kref, enclave_release); } -/* Helper work, which we can kick from IRQ context. */ -static void enclave_destroyer(struct work_struct *work) -{ - struct ghost_enclave *e = container_of(work, struct ghost_enclave, - enclave_destroyer); - - pr_info("Killing enclave %lu for a violation\n", e->id); - ghost_destroy_enclave(e); - kref_put(&e->kref, enclave_release); -} - /* * Here we can trigger any destruction we want, such as killing agents, kicking * tasks to CFS, etc. Undo things that were done at runtime. * * The enclave itself isn't freed until enclave_release(). */ -static void ghost_destroy_enclave(struct ghost_enclave *e) +static void __ghost_destroy_enclave(struct ghost_enclave *e) { ulong flags; int cpu; @@ -2073,6 +2056,26 @@ static void ghost_destroy_enclave(struct ghost_enclave *e) kref_put(&e->kref, enclave_release); } +static void enclave_destroyer(struct work_struct *work) +{ + struct ghost_enclave *e = container_of(work, struct ghost_enclave, + enclave_destroyer); + + __ghost_destroy_enclave(e); + kref_put(&e->kref, enclave_release); +} + +static void ghost_destroy_enclave(struct ghost_enclave *e) +{ + /* + * Defer destruction to a CFS task. Need this in case we're in IRQ ctx + * or if current is in the enclave. + */ + kref_get(&e->kref); + if (!schedule_work(&e->enclave_destroyer)) + kref_put(&e->kref, enclave_release); +} + /* * Sets the cpus of the enclave to `cpus`, adding and removing as necessary. * Returns an error code on failure and does not change the enclave. Will fail @@ -3107,7 +3110,7 @@ static struct task_struct *find_task_by_gtid(gtid_t gtid) * A dying task ensures that the task_struct will remain stable for * an rcu grace period via call_rcu(ghost_delayed_put_task_struct). * - * A dying enclave will synchronize_rcu() in ghost_destroy_enclave() + * A dying enclave will synchronize_rcu() in __ghost_destroy_enclave() * before moving its tasks out of ghost (we don't use call_rcu() here * because the task may transition in and out of ghost any number of * times independent of call_rcu() invocation). @@ -4295,7 +4298,7 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) */ ghost_claim_and_kill_txn(rq->cpu, GHOST_TXN_NO_AGENT); - /* See ghost_destroy_enclave() */ + /* See __ghost_destroy_enclave() */ if (p->ghost.__agent_decref_enclave) { /* Agents should only exit, never setsched out. */ WARN_ON_ONCE(p->state != TASK_DEAD); @@ -4339,7 +4342,7 @@ static void ghost_delayed_put_task_struct(struct rcu_head *rhp) ghost.rcu); /* * Step 3 of "destroyed with agent": decref. - * See ghost_destroy_enclave(). + * See __ghost_destroy_enclave(). */ if (tsk->ghost.__agent_decref_enclave) { kref_put(&tsk->ghost.__agent_decref_enclave->kref, From 15bff6aaf4c373ae489697aa5c6ba191b9e7c6ae Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Tue, 8 Feb 2022 15:11:27 -0500 Subject: [PATCH 156/165] sched/ghost: replace BPF_GHOST_SCHED_SKIP_TICK with deliver_ticks Skip-tick was our first BPF use case. Anything you could do with skip-tick can be done with BPF-MSG. However, not all agents want to use BPF, particularly for something as trivial as suppressing all cpu tick messages. Add an enclave tunable, "deliver_ticks", which defaults to 0. When set, we'll deliver cpu tick messages during the ghost timer tick. At this point, BPF-MSG can intercept the message, handle it, and optionally suppress it - just like any other message. To be submitted with cl/427258783. Tested: edf_test, agent_exp with/without --ticks, agent_muppet + simple_exp, api_test Effort: sched/ghost Google-Bug-Id: 210860596 Signed-off-by: Barret Rhoden Change-Id: I8beae844f241b94d8e80dbdbb12660e536bda607 Origin-9xx-SHA1: 4ee077db4d91c8cc3307188e953e0b336cf5eebf Rebase-Tested-11xx: compiled --- include/uapi/linux/bpf.h | 3 +- include/uapi/linux/ghost.h | 2 +- kernel/bpf/syscall.c | 2 - kernel/sched/ghost.c | 94 +++++++++++++++----------------------- kernel/sched/sched.h | 2 +- 5 files changed, 40 insertions(+), 63 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 8a8a56f0ba59..1c8632c9db83 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -255,8 +255,7 @@ enum { * If we ever merge upstream, we can go back below * __MAX_BPF_ATTACH_TYPE. We have our own numbers to avoid rebase hell. */ - BPF_GHOST_SCHED_SKIP_TICK = 2000, - BPF_GHOST_SCHED_PNT, + BPF_GHOST_SCHED_PNT = 2000, BPF_GHOST_MSG_SEND, __MAX_BPF_GHOST_ATTACH_TYPE }; diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 41664bc4f690..65ff188f35ee 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 65 +#define GHOST_VERSION 66 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index fc5b5626c351..291b79d78c8e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2009,7 +2009,6 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, return -EINVAL; case BPF_PROG_TYPE_GHOST_SCHED: switch ((int)expected_attach_type) { - case BPF_GHOST_SCHED_SKIP_TICK: case BPF_GHOST_SCHED_PNT: return 0; default: @@ -2978,7 +2977,6 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) return BPF_PROG_TYPE_SK_LOOKUP; case BPF_XDP: return BPF_PROG_TYPE_XDP; - case BPF_GHOST_SCHED_SKIP_TICK: case BPF_GHOST_SCHED_PNT: return BPF_PROG_TYPE_GHOST_SCHED; case BPF_GHOST_MSG_SEND: diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 324f12efeddb..8b87738aa28e 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -136,26 +136,6 @@ static inline bool enclave_is_reapable(struct ghost_enclave *e) #ifdef CONFIG_BPF #include -static bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) -{ - struct bpf_ghost_sched_kern ctx = {}; - struct bpf_prog *prog; - struct ghost_enclave *old_target; - bool ret; - - lockdep_assert_held(&rq->lock); - - prog = rcu_dereference(e->bpf_tick); - if (!prog) - return false; - - old_target = set_target_enclave(e); - /* prog returns 1 if we want a tick on this cpu. */ - ret = BPF_PROG_RUN(prog, &ctx) != 1; - restore_target_enclave(old_target); - return ret; -} - #define BPF_GHOST_PNT_DONT_IDLE 1 static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, @@ -229,10 +209,6 @@ static bool ghost_bpf_msg_send(struct ghost_enclave *e, return send; } #else -static inline bool ghost_bpf_skip_tick(struct ghost_enclave *e, struct rq *rq) -{ - return false; -} static inline void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, struct rq_flags *rf) @@ -3816,7 +3792,7 @@ static inline bool cpu_deliver_msg_tick(struct rq *rq) return false; rcu_read_lock(); e = rcu_dereference(per_cpu(enclave, cpu_of(rq))); - if (!e || ghost_bpf_skip_tick(e, rq)) { + if (!e || !e->deliver_ticks) { rcu_read_unlock(); return false; } @@ -6772,6 +6748,37 @@ static struct kernfs_ops gf_ops_e_ctl = { .ioctl = gf_ctl_ioctl, }; +static int gf_deliver_ticks_show(struct seq_file *sf, void *v) +{ + struct ghost_enclave *e = seq_to_e(sf); + + seq_printf(sf, "%d", READ_ONCE(e->deliver_ticks)); + return 0; +} + +static ssize_t gf_deliver_ticks_write(struct kernfs_open_file *of, + char *buf, size_t len, loff_t off) +{ + struct ghost_enclave *e = of_to_e(of); + int err; + int tunable; + + err = kstrtoint(buf, 0, &tunable); + if (err) + return -EINVAL; + + WRITE_ONCE(e->deliver_ticks, !!tunable); + + return len; +} + +static struct kernfs_ops gf_ops_e_deliver_ticks = { + .open = gf_e_open, + .release = gf_e_release, + .seq_show = gf_deliver_ticks_show, + .write = gf_deliver_ticks_write, +}; + /* * Returns a kreffed pointer for the enclave for f, if f is a ghostfs ctl file. * NULL otherwise. @@ -7050,6 +7057,11 @@ static struct gf_dirent enclave_dirtab[] = { .mode = 0664, .ops = &gf_ops_e_ctl, }, + { + .name = "deliver_ticks", + .mode = 0664, + .ops = &gf_ops_e_deliver_ticks, + }, { .name = "runnable_timeout", .mode = 0664, @@ -7427,31 +7439,6 @@ static bool ghost_msg_is_valid_access(int off, int size, return true; } -static int ghost_sched_tick_attach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - if (e->bpf_tick) { - spin_unlock_irqrestore(&e->lock, irq_fl); - return -EBUSY; - } - rcu_assign_pointer(e->bpf_tick, prog); - spin_unlock_irqrestore(&e->lock, irq_fl); - return 0; -} - -static void ghost_sched_tick_detach(struct ghost_enclave *e, - struct bpf_prog *prog) -{ - unsigned long irq_fl; - - spin_lock_irqsave(&e->lock, irq_fl); - rcu_replace_pointer(e->bpf_tick, NULL, lockdep_is_held(&e->lock)); - spin_unlock_irqrestore(&e->lock, irq_fl); -} - static int ghost_sched_pnt_attach(struct ghost_enclave *e, struct bpf_prog *prog) { @@ -7510,9 +7497,6 @@ static int bpf_link_attach(struct ghost_enclave *e, struct bpf_prog *prog, switch (prog_type) { case BPF_PROG_TYPE_GHOST_SCHED: switch (attach_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - err = ghost_sched_tick_attach(e, prog); - break; case BPF_GHOST_SCHED_PNT: err = ghost_sched_pnt_attach(e, prog); break; @@ -7547,9 +7531,6 @@ static void bpf_link_detach(struct ghost_enclave *e, struct bpf_prog *prog, switch (prog_type) { case BPF_PROG_TYPE_GHOST_SCHED: switch (attach_type) { - case BPF_GHOST_SCHED_SKIP_TICK: - ghost_sched_tick_detach(e, prog); - break; case BPF_GHOST_SCHED_PNT: ghost_sched_pnt_detach(e, prog); break; @@ -7625,7 +7606,6 @@ static int _ghost_bpf_link_attach(const union bpf_attr *attr, switch (prog->type) { case BPF_PROG_TYPE_GHOST_SCHED: switch (ea_type) { - case BPF_GHOST_SCHED_SKIP_TICK: case BPF_GHOST_SCHED_PNT: break; default: diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0e2c457fcd8c..80bae847e972 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -192,6 +192,7 @@ struct ghost_enclave { bool switchto_disabled; bool wake_on_waker_cpu; bool commit_at_tick; + bool deliver_ticks; unsigned long id; int is_dying; @@ -201,7 +202,6 @@ struct ghost_enclave { kgid_t gid; #ifdef CONFIG_BPF - struct bpf_prog *bpf_tick; struct bpf_prog *bpf_pnt; struct bpf_prog *bpf_msg_send; #endif From 01b9b3b2e8b6a4a474bd48f9e9b24f605ad0fb79 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 9 Feb 2022 12:16:02 -0500 Subject: [PATCH 157/165] sched/ghost: add ghost BPF bits to tools/ header GHOST_BPF and bpf_ghost_sched were in the main uapi bpf.h, but not the tools/ version of the header. It's not as simple as copying the entire bpf.h header from the kernel to tools/, since there are a bunch of other differences in non-ghost code, mostly in comments. Tested: //net/bpf:bpf_test_suite:test_progs Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ica002dabf68fba633fc511b6a4130a809256f552 Origin-9xx-SHA1: 9321a72de9b96bd566cacc5867e2799507aa01f4 Rebase-Tested-11xx: compiled --- tools/include/uapi/linux/bpf.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 7933f635a990..976221c0beed 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -5160,6 +5160,11 @@ struct bpf_sk_lookup { __u32 local_port; /* Host byte order */ }; +#define GHOST_BPF + +struct bpf_ghost_sched { +}; + /* * struct btf_ptr is used for typed pointer representation; the * type id is used to render the pointer data as the appropriate type From ddafcb848ddbecee100f5a16fb7c8d5b3fc5f9fb Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 16 Feb 2022 16:43:36 -0500 Subject: [PATCH 158/165] sched/ghost: add a ctl command to disable the ability to load bpf programs Once we've inserted the bpf programs (attached or linked), we no longer need to load or attach/link more programs. In the off chance our agent is compromised, by disabling bpf_prog_load, the agent would be unable to load new hostile programs that it could attach to other places in the kernel. Our agents typically run with CAP_BPF, though not necessarily CAP_PERFMON, so the list of bpf programs we could load and attach is limited. This helper limits it further. The existing programs we already loaded are considered "known good". They cannot be attached to random places in the kernel: only to BPF-PNT and BPF-MSG. This helper is a one-way operation for the calling process. A new agent that attaches to the enclave will be able to load and attach new programs. Note you cannot have more than one program attached per enclave, per attach point. To be submitted with cl/429130082. Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I4895e78b0c7e49006f16dddb7341f109a3bd006b Origin-9xx-SHA1: 49667d54efeb559e9dd375f68ff88da0524e0f3b Rebase-Tested-11xx: compiled --- include/linux/sched.h | 9 +++++---- include/uapi/linux/ghost.h | 2 +- kernel/bpf/syscall.c | 5 +++++ kernel/sched/ghost.c | 2 ++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index dfb429bbda89..4de979a738af 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -607,10 +607,11 @@ struct sched_ghost_entity { * See also ghost_prepare_task_switch() and ghost_deferred_msgs() * for flags that are used to defer messages. */ - uint blocked_task : 1; - uint yield_task : 1; - uint new_task : 1; - uint agent : 1; + uint blocked_task : 1; + uint yield_task : 1; + uint new_task : 1; + uint agent : 1; + uint bpf_cannot_load_prog : 1; /* * Locking of 'twi' is awkward: diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index 65ff188f35ee..f522278d9274 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 66 +#define GHOST_VERSION 67 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 291b79d78c8e..e2873a8f51be 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2091,6 +2091,11 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) char license[128]; bool is_gpl; +#ifdef CONFIG_SCHED_CLASS_GHOST + if (current->group_leader->ghost.bpf_cannot_load_prog) + return -EPERM; +#endif + if (CHECK_ATTR(BPF_PROG_LOAD)) return -EINVAL; diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 8b87738aa28e..9a404cf58ec0 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -6662,6 +6662,8 @@ static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, err = ctl_become_agent(of, qfd); if (err) return err; + } else if (!strcmp(buf, "disable my bpf_prog_load")) { + current->group_leader->ghost.bpf_cannot_load_prog = 1; } else { pr_err("%s: bad cmd :%s:", __func__, buf); return -EINVAL; From bf32ce96e29e68f27756cb5d38443602c52776aa Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 17 Mar 2022 11:02:35 -0400 Subject: [PATCH 159/165] sched/ghost: make _ghost_commit_pending_txn() return void No callers were checking the return value. Tested: edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I2bbb912c61b11358d2cf8dd3593c689f647415c2 Origin-9xx-SHA1: 8bb13a445545f9cdd0dac7eab761311e781bd056 Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 9a404cf58ec0..da2652e2ae58 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -66,7 +66,6 @@ static bool cpu_deliver_msg_tick(struct rq *rq); static int task_target_cpu(struct task_struct *p); static int agent_target_cpu(struct rq *rq); static inline bool ghost_txn_ready(int cpu); -static bool _ghost_commit_pending_txn(int cpu, int where); static inline void ghost_claim_and_kill_txn(int cpu, enum ghost_txn_state err); static void ghost_commit_all_greedy_txns(void); static void ghost_commit_pending_txn(int where); @@ -5545,11 +5544,10 @@ static inline void ghost_send_reschedule(struct cpumask *mask) } #endif -static bool _ghost_commit_pending_txn(int cpu, int where) +static void _ghost_commit_pending_txn(int cpu, int where) { if (unlikely(ghost_claim_txn(cpu, where))) - return ghost_commit_txn(cpu, false, NULL); - return false; + ghost_commit_txn(cpu, false, NULL); } static void ghost_commit_pending_txn(int where) From 3820d33e9d7ade7083bcffcb7a44242c9a559892 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Thu, 17 Mar 2022 15:37:58 -0400 Subject: [PATCH 160/165] sched/ghost: move BPF-PNT to pnt-prologue Previously, BPF-PNT ran only as a last resort, after several things: 1) if there is no runnable agent (not blocked_in_run) 2) if we will not yield to any other class between PNT-A and PNT-G, specifically to CFS (rq_adj_nr_running) 3) if there is no latched task 4) if the current task won't keep running (must_resched) 5) if there's no pending transaction (unlatched, but sitting in the txn region, perhaps with COMMIT_AT_SCHEDULE). After all of that, we'll run BPF, then since we unlocked, go back and recheck things like if the agent woke, if CFS woke, if bpf latched a task, etc. At the moment we unlocked the RQ to run BPF-PNT, cases 1, 2, and 3 were all possible to become false, which we were able to handle. However, that's a rare race. By exposing the RQ status via the context struct bpf_ghost_sched, BPF-PNT can make the same decisions it did before: e.g. if ctx->agent_runnable, then don't bother latching a task. The BPF program can detect cases 3 and 4 from ctx->next_gtid, but there is no nice way to detect case 5. The only real user was COMMIT_AT_SCHEDULE, which was added to speed up slow global agents and is largely unused. BPF-PNT solves the issues of COMMIT_AT_SCHEDULE, so I recommend not using both BPF-PNT and COMMIT_AT_SCHEDULE. Running BPF later and redoing the checks was a minor inconvenience, but we were essentially deciding for BPF-PNT what it wanted to do, and presumed that there was nothing other than latching a task that it could do. Submitted with cl/435677777. Tested: api_test, edf_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I4ba012add422ee34c4d2a103b1e19e9bd95cd765 Origin-9xx-SHA1: dc74a30f80d9b03483ef95f5797ae9fd7d3609d2 Rebase-Tested-11xx: compiled --- include/uapi/linux/bpf.h | 21 ++++++++-- include/uapi/linux/ghost.h | 2 +- kernel/sched/core.c | 2 +- kernel/sched/ghost.c | 78 +++++++++++++++++++++++++++----------- kernel/sched/ghost_core.c | 15 +++++--- kernel/sched/sched.h | 11 ++++-- 6 files changed, 94 insertions(+), 35 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 1c8632c9db83..97277911c1af 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -5177,6 +5177,24 @@ struct bpf_sk_lookup { __u32 local_port; /* Host byte order */ }; +#define GHOST_BPF + +struct bpf_ghost_sched { + __u8 agent_on_rq; /* there is an agent, it can run if poked */ + __u8 agent_runnable; /* there is an agent, it will run */ + __u8 might_yield; /* other classes (CFS) probably want to run. + * the agent will supercede these. latched + * tasks will not. + */ + __u8 dont_idle; /* set true to prevent the cpu from idling */ + __u64 next_gtid; /* scheduler will run this next, unless you do + * something (or agent_runnable or + * should_yield). This is either a latched task + * or was current and still TASK_RUNNING in PNT. + * It will be preempted if you latch. + */ +}; + /* * struct btf_ptr is used for typed pointer representation; the * type id is used to render the pointer data as the appropriate type @@ -5209,7 +5227,4 @@ enum { #define GHOST_BPF -struct bpf_ghost_sched { -}; - #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/include/uapi/linux/ghost.h b/include/uapi/linux/ghost.h index f522278d9274..748aff5d4dc3 100644 --- a/include/uapi/linux/ghost.h +++ b/include/uapi/linux/ghost.h @@ -31,7 +31,7 @@ * process are the same version as each other. Each successive version changes * values in this header file, assumptions about operations in the kernel, etc. */ -#define GHOST_VERSION 67 +#define GHOST_VERSION 68 /* * Define SCHED_GHOST via the ghost uapi unless it has already been defined diff --git a/kernel/sched/core.c b/kernel/sched/core.c index a973c505c6f1..f74892c21f95 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5031,7 +5031,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) struct task_struct *p; #ifdef CONFIG_SCHED_CLASS_GHOST - ghost_pnt_prologue(rq, prev); + ghost_pnt_prologue(rq, prev, rf); #endif /* diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index da2652e2ae58..e97d2dde1619 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -90,6 +90,11 @@ static const struct file_operations queue_fops; static void ghost_destroy_enclave(struct ghost_enclave *e); static void enclave_release(struct kref *k); +static inline gtid_t gtid(struct task_struct *p) +{ + return p->gtid; +} + /* True if X and Y have the same enclave, including having no enclave. */ static bool check_same_enclave(int cpu_x, int cpu_y) { @@ -135,15 +140,13 @@ static inline bool enclave_is_reapable(struct ghost_enclave *e) #ifdef CONFIG_BPF #include -#define BPF_GHOST_PNT_DONT_IDLE 1 - static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, - struct rq_flags *rf) + struct task_struct *prev, struct rq_flags *rf) { - struct bpf_ghost_sched_kern ctx = {}; + struct task_struct *agent = rq->ghost.agent; + struct bpf_ghost_sched_kern ctx[1]; struct bpf_prog *prog; struct ghost_enclave *old_target; - int ret; lockdep_assert_held(&rq->lock); @@ -153,6 +156,25 @@ static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, rcu_read_unlock(); return; } + /* Zero the struct to avoid leaking stack data in struct padding */ + memset(ctx, 0, sizeof(struct bpf_ghost_sched_kern)); + + ctx->agent_on_rq = agent && agent->on_rq; + ctx->agent_runnable = ctx->agent_on_rq && !rq->ghost.blocked_in_run; + + ctx->might_yield = rq_adj_nr_running(rq) > 0; + ctx->dont_idle = false; + + if (rq->ghost.latched_task) { + ctx->next_gtid = gtid(rq->ghost.latched_task); + } else if (task_has_ghost_policy(prev) + && prev->state == TASK_RUNNING + && prev != agent + && !rq->ghost.must_resched) { + ctx->next_gtid = gtid(prev); + } else { + ctx->next_gtid = 0; + } /* * BPF programs attached here may call ghost_run_gtid(), which requires @@ -163,7 +185,9 @@ static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, raw_spin_unlock(&rq->lock); old_target = set_target_enclave(e); - ret = BPF_PROG_RUN(prog, &ctx); + rq->ghost.in_pnt_bpf = true; + BPF_PROG_RUN(prog, ctx); + rq->ghost.in_pnt_bpf = false; restore_target_enclave(old_target); raw_spin_lock(&rq->lock); @@ -171,7 +195,7 @@ static void ghost_bpf_pnt(struct ghost_enclave *e, struct rq *rq, rcu_read_unlock(); - if (ret == BPF_GHOST_PNT_DONT_IDLE) { + if (ctx->dont_idle) { /* * The next time this rq selects the idle task, it will bail out * of do_idle() quickly. Since we unlocked the RQ lock, it's @@ -355,11 +379,6 @@ static void submit_enclave_work(struct ghost_enclave *e, struct rq *rq, queue_balance_callback(rq, &rq->ghost.ew_head, __do_enclave_work); } -static inline gtid_t gtid(struct task_struct *p) -{ - return p->gtid; -} - static inline bool on_ghost_rq(struct rq *rq, struct task_struct *p) { lockdep_assert_held(&rq->lock); @@ -902,14 +921,6 @@ static int _balance_ghost(struct rq *rq, struct task_struct *prev, rq_repin_lock(rq, rf); } - if (!rq->ghost.latched_task && rq->ghost.pnt_bpf_once) { - rq->ghost.pnt_bpf_once = false; - /* If there is a BPF program, this will unlock the RQ */ - rq->ghost.in_pnt_bpf = true; - ghost_bpf_pnt(agent->ghost.enclave, rq, rf); - rq->ghost.in_pnt_bpf = false; - } - /* * We have something to run in 'latched_task' or a higher priority * sched_class became runnable while the rq->lock was dropped. @@ -7238,8 +7249,11 @@ static struct ghost_enclave *create_enclave(const struct ghost_abi *abi, /* * Called at the start of every pick_next_task() via __schedule(). */ -static void pnt_prologue(struct rq *rq, struct task_struct *prev) +static void pnt_prologue(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) { + struct ghost_enclave *e; + rq->ghost.check_prev_preemption = ghost_produce_prev_msgs(rq, prev); if (unlikely(rq->ghost.ignore_prev_preemption)) { rq->ghost.check_prev_preemption = false; @@ -7249,7 +7263,6 @@ static void pnt_prologue(struct rq *rq, struct task_struct *prev) /* a negative 'switchto_count' indicates end of the chain */ rq->ghost.switchto_count = -rq->ghost.switchto_count; WARN_ON_ONCE(rq->ghost.switchto_count > 0); - rq->ghost.pnt_bpf_once = true; /* * Lockless way to set must_resched, which kicks prev off cpu. The @@ -7258,6 +7271,13 @@ static void pnt_prologue(struct rq *rq, struct task_struct *prev) */ if (READ_ONCE(rq->ghost.prev_resched_seq) == rq->ghost.cpu_seqnum) rq->ghost.must_resched = true; + + rcu_read_lock(); + e = rcu_dereference(per_cpu(enclave, cpu_of(rq))); + /* If there is a BPF program, this will unlock the RQ */ + if (e) + ghost_bpf_pnt(e, rq, prev, rf); + rcu_read_unlock(); } /* @@ -7427,6 +7447,19 @@ static int bpf_resched_cpu(int cpu, u64 cpu_seqnum) return 0; } +static bool ghost_sched_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + /* The verifier guarantees that size > 0. */ + if (off < 0 || off + size > sizeof(struct bpf_ghost_sched) + || off % size) + return false; + + return true; +} + static bool ghost_msg_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, @@ -7691,6 +7724,7 @@ DEFINE_GHOST_ABI(current_abi) = { .bpf_wake_agent = bpf_wake_agent, .bpf_run_gtid = bpf_run_gtid, .bpf_resched_cpu = bpf_resched_cpu, + .ghost_sched_is_valid_access = ghost_sched_is_valid_access, .ghost_msg_is_valid_access = ghost_msg_is_valid_access, .bpf_link_attach = _ghost_bpf_link_attach, diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index 537de08d77f0..bace6eff627a 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -464,14 +464,15 @@ static struct ghost_enclave *_ghost_resolve_enclave(struct rq *rq, return rq_enclave ? rq_enclave : task_enclave; } -void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev) +void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) { struct ghost_enclave *e = _ghost_resolve_enclave(rq, prev); VM_BUG_ON(rq != this_rq()); if (e != NULL) - e->abi->pnt_prologue(rq, prev); + e->abi->pnt_prologue(rq, prev, rf); } void __init init_sched_ghost_class(void) @@ -1200,10 +1201,14 @@ static bool ghost_sched_is_valid_access(int off, int size, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { - /* struct bpf_ghost_sched is empty so all accesses are invalid. */ - return false; -} + int version = bpf_prog_eat_abi(prog); + const struct ghost_abi *abi = ghost_abi_lookup(version); + if (WARN_ON_ONCE(!abi)) + return false; + + return abi->ghost_sched_is_valid_access(off, size, type, prog, info); +} const struct bpf_verifier_ops ghost_sched_verifier_ops = { .get_func_proto = ghost_sched_func_proto, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 80bae847e972..6a580b4491bc 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -112,7 +112,6 @@ struct ghost_rq { bool ignore_prev_preemption; bool check_prev_preemption; /* see 'ghost_prepare_task_switch()' */ bool skip_latched_preemption; - bool pnt_bpf_once; /* BPF runs at most once in PNT */ bool in_pnt_bpf; /* running BPF at PNT */ bool dont_idle_once; /* Don't idle next time rq->idle runs */ int ghost_nr_running; @@ -2187,7 +2186,8 @@ struct ghost_abi { int (*fork)(struct ghost_enclave *e, struct task_struct *p); void (*cleanup_fork)(struct ghost_enclave *e, struct task_struct *p); void (*wait_for_rendezvous)(struct rq *rq); - void (*pnt_prologue)(struct rq *rq, struct task_struct *prev); + void (*pnt_prologue)(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf); void (*prepare_task_switch)(struct rq *rq, struct task_struct *prev, struct task_struct *next); void (*tick)(struct ghost_enclave *e, struct rq *rq); @@ -2200,6 +2200,10 @@ struct ghost_abi { int (*bpf_wake_agent)(int cpu); int (*bpf_run_gtid)(s64 gtid, u32 task_barrier, int run_flags, int cpu); int (*bpf_resched_cpu)(int cpu, u64 cpu_seqnum); + bool (*ghost_sched_is_valid_access)(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info); bool (*ghost_msg_is_valid_access)(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, @@ -2275,7 +2279,8 @@ void ghost_remove_cpu(struct ghost_enclave *e, int cpu); int64_t ghost_sync_group_cookie(void); void ghost_wait_for_rendezvous(struct rq *rq); -void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev); +void ghost_pnt_prologue(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf); void ghost_tick(struct rq *rq); int ghost_setscheduler(struct task_struct *p, struct rq *rq, From 2599fa7b3d93b287740c617e15e595e96035512b Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 23 Mar 2022 15:23:19 -0400 Subject: [PATCH 161/165] sched/ghost: remove stale comment Fixed in commit 875ee64be8cd ("sched/ghost: fix bpf_helper enclave discovery"). Tested: compiled Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I01ca180a51ec04e4adfcc135ebd1cf2e63fafbc3 Origin-9xx-SHA1: 79d2ede01b2e3c178f72daf32426ea29fe385d0a Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index e97d2dde1619..1b7130cf816f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -3740,12 +3740,6 @@ static inline int __produce_for_task(struct task_struct *p, WARN(1, "unknown bpg_ghost_msg type %d!\n", msg->type); return -EINVAL; }; - /* - * XXX for a task that enters ghost into enclave_X but on a cpu - * that belongs to enclave_Y then we'll call the 'bpf_msg_send' - * program attached to enclave_X but any BPF helpers used by the - * program will callback into enclave_Y. - */ if (!ghost_bpf_msg_send(p->ghost.enclave, msg)) return -ENOMSG; return _produce(p->ghost.dst_q, barrier, msg->type, From 4b336dfbcaeaa9f116bfe3f5e474cf851a6222bb Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 6 Apr 2022 13:18:55 -0400 Subject: [PATCH 162/165] sched/ghost: always use select_task_rq_ghost() select_task_rq() did not always call select_task_rq_ghost(), and even if it did, it could ignore the answer - particularly if the cpu was not in the task's allowed_cpus. The RQ doesn't matter much for ghost. The "real" RQ is in the agent, but the kernel still uses the (per-cpu) RQ for its own bookkeeping, including the steps needed to wake a task. For an in-kernel scheduler like CFS, the RQ matters: it's where the task will compete for cpu time and may ultimately run. For ghost, it's a temporary staging ground until a task gets latched (at which point it will change RQs). Of note is the TTWU_QUEUE feature. Normally, when we do a 'remote' wakeup, meaning select_task_rq() picks a cpu other than the caller's, we'd grab their rq lock, do the wakeup, and maybe IPI if we need to preempt. With TTWU_QUEUE, we can put the task on the RQ's wake_list, then send an IPI (amortized). This trades an IPI for lock bouncing - presumably if the task will actually run on its old cpu, then this will help. The problem arises when we pick a rq that is neither the task_cpu (i.e. previous cpu it ran on) nor the current cpu. This third-party cpu can get overloaded with resched IPIs from TTWU_QUEUE remote wakeups. CFS tries to find an idle RQ, and doesn't aggressively load balance or otherwise migrate tasks from a cpu. Ghost does neither. We offer a choice between task_cpu and the current cpu (waker's cpu). And we can yank tasks off remote RQs without hesitation: this is what happens during a latch. select_task_rq_ghost() would normally be OK, but prior to this commit, select_task_rq() would ignore the result under certain circumstances, specifically if the cpu was not in p->allowed_cpus. The ghost kernel ignores affinity: that's up to the agent to enforce, yet select_task_rq() was enforcing affinity, but only for the 'temporary' rq - not the actual execution of a task. Consider a task that was affined to numa node 1, yet the agent ran it on node 0. It wakes on node 0, and select_task_rq_ghost() returns a cpu from node 0. select_fallback_rq() will pick the first cpu on node 1 to do the wakeup, and using TTWU_QUEUE, it will send an IPI and that cpu will have to do all the wakeup bits. For ghost, that includes sending the wakeup message, which can run a bpf-msg program. Once the task is woken up, another cpu (perhaps from node 0) can latch it - essentially pulling it away from the victim cpu. If that task blocks and wakes quickly, and it was running on node 0, the cycle will continue. If you have enough cpus running/blocking/waking tasks, and shipping their wakeup work to a victim cpu, then that cpu will get stuck in IRQ context, constantly handling new resched IPIs. You'd need enough cpus to keep the victim busy. The aggrivating factor to this is bpf-msg. bpf programs will finish in a bounded amount of time, but there's nothing that stops them from being inefficient. The world's simplest scheduler (my Biff scheduler in a forthcoming CL) is a global queue, implemented with a BPF_MAP_TYPE_QUEUE, which is protected by a kernel spinlock, which is an MCS queue lock. If you have all the cpus hammering that lock too, it will make bpf-msg less efficient, to the tune of 40-50us per task ttwu(). Even without bpf-msg, you'd need only enough cpus to keep the victim busy. If the wakeup cycles is W_c, and the time to block, wake, run, and reblock is T_c (task), you'd need T_c / W_c cpus to keep the victim stuck in its IPI handlers. But a combination of bpf-msg and a really dumb scheduler increased W_c. The trick is to break the cycle: by requiring that select_task_rq() returns either task_cpu() or the current cpu, we avoid this "stuck in resched IPI" scenario: - if it's the current cpu, we don't need an IPI. - if it's task_cpu, then when some other cpu pulls it from the victim's RQ, that cpu becomes the new task_cpu, breaking the cycle. Tested: SLL Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: Ib65a1672420c72ff9d3bf71c0d50eba245a1019d Origin-9xx-SHA1: 37588684a0ce59fb763a4deb2fea31078e47984c Rebase-Tested-11xx: compiled --- kernel/sched/core.c | 43 +++++++++++++++++++++++++++++++++++++++ kernel/sched/ghost_core.c | 2 +- kernel/sched/sched.h | 2 ++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f74892c21f95..e0450d5b6e78 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2904,6 +2904,49 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) { lockdep_assert_held(&p->pi_lock); +#ifdef CONFIG_SCHED_CLASS_GHOST + if (task_has_ghost_policy(p)) { + /* + * Agents must always stay on the rq of their pinned cpu. + */ + if (p->ghost.agent) + return task_cpu(p); + /* + * ghost ignores p->cpus_allowed when it latches task, so we + * only ever want to call select_task_rq_ghost(). + * + * select_task_rq() doesn't affect where a task will run next - + * the agent decides that. Instead, it decides where a task + * wakes up. The rq is more of a temporary staging ground, and + * the agent maintains the 'real' runqueue(s). + * + * You might be tempted to let the agent pick any cpu here, but + * that is dangerous. Since we use TTWU_QUEUE, we'll send a + * resched IPI to that cpu, and the wakeup (ttwu_do_activate()) + * will happen in IRQ context. It's possible to overload a cpu + * with resched IPIs, so long as that cpu's *execution* is not + * required for the wake-run-block-wake loop. i.e. it is stuck + * handling IPIs for wakeups, and other cpus latch and run the + * task, generating an endless stream of wakeup IPIs. This is + * exacerbated by bpf-msg, but could happen with enough cpus + * waking tasks or with some inefficient/unscalable + * sched_class->task_woken(). + * + * There are two 'safe' cpus to select: our current cpu, which + * won't require an IPI, and task_cpu(p). task_cpu will change + * once a task runs, so if a victim cpu is overloaded and + * another cpu runs the task, that cpu becomes task_cpu, + * breaking the endless stream. + */ + cpu = select_task_rq_ghost(p, cpu, wake_flags); + if (WARN_ON_ONCE(cpu != task_cpu(p) + && cpu != smp_processor_id())) + cpu = smp_processor_id(); + if (!cpu_online(cpu)) + cpu = smp_processor_id(); + return cpu; + } +#endif if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) cpu = p->sched_class->select_task_rq(p, cpu, wake_flags); else diff --git a/kernel/sched/ghost_core.c b/kernel/sched/ghost_core.c index bace6eff627a..5fa6528ec34d 100644 --- a/kernel/sched/ghost_core.c +++ b/kernel/sched/ghost_core.c @@ -971,7 +971,7 @@ static void yield_task_ghost(struct rq *rq) p->pid, p->comm, cpu_of(rq)); } -static int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) +int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags) { struct ghost_enclave *e; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 6a580b4491bc..08daf5240dec 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2304,6 +2304,8 @@ unsigned long ghost_cfs_added_load(struct rq *rq); int64_t ghost_alloc_gtid(struct task_struct *p); void init_ghost_rq(struct ghost_rq *ghost_rq); +int select_task_rq_ghost(struct task_struct *p, int cpu, int wake_flags); + #ifdef CONFIG_SWITCHTO_API void ghost_switchto(struct rq *rq, struct task_struct *prev, struct task_struct *next, int switchto_flags); From ce47f9ffd4d54e0d8c299ebe79332f414b609c2b Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 6 Apr 2022 14:46:56 -0400 Subject: [PATCH 163/165] sched/ghost: free status words for suppressed messages The agent's bpf-msg program returns whether or not they want us to send the message to their Channel. If they do not want the message, then we can free the status word directly in the kernel. Normally agents free the status word after a dead or departed message. If there is no message, userspace will not free the status word. Tested: edf_test, api_test Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I01f02e92581fe27405af1cae4345bb78b9aafc7b Origin-9xx-SHA1: d673ad0e788d84be97a83b984367b84c64c83011 Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 91 +++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 1b7130cf816f..61c2bd08a55f 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -58,7 +58,7 @@ static void task_deliver_msg_task_new(struct rq *rq, struct task_struct *p, bool runnable); static void task_deliver_msg_affinity_changed(struct rq *rq, struct task_struct *p); -static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p); +static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p); static void task_deliver_msg_wakeup(struct rq *rq, struct task_struct *p); static void task_deliver_msg_latched(struct rq *rq, struct task_struct *p, bool latched_preempt); @@ -4019,26 +4019,26 @@ static void task_deliver_msg_blocked(struct rq *rq, struct task_struct *p) produce_for_task(p, msg); } -static void task_deliver_msg_dead(struct rq *rq, struct task_struct *p) +static bool task_deliver_msg_dead(struct rq *rq, struct task_struct *p) { struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); struct ghost_msg_payload_task_dead *payload = &msg->dead; if (__task_deliver_common(rq, p)) - return; + return false; msg->type = MSG_TASK_DEAD; payload->gtid = gtid(p); - produce_for_task(p, msg); + return produce_for_task(p, msg) == 0; } -static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) +static bool task_deliver_msg_departed(struct rq *rq, struct task_struct *p) { struct bpf_ghost_msg *msg = &per_cpu(bpf_ghost_msg, cpu_of(rq)); struct ghost_msg_payload_task_departed *payload = &msg->departed; if (__task_deliver_common(rq, p)) - return; + return false; msg->type = MSG_TASK_DEPARTED; payload->gtid = gtid(p); @@ -4050,7 +4050,7 @@ static void task_deliver_msg_departed(struct rq *rq, struct task_struct *p) payload->from_switchto = false; payload->was_current = task_current(rq, p); - produce_for_task(p, msg); + return produce_for_task(p, msg) == 0; } static void task_deliver_msg_affinity_changed(struct rq *rq, @@ -4183,37 +4183,42 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_F_CANFREE|GHOST_SW_TASK_MSG_GATED); - task_deliver_msg_departed(rq, p); - - /* - * Inhibit all msgs produced by this task until the agent - * acknowledges receipt of TASK_DEPARTED msg (implicitly by - * freeing the status_word). This ensures that when TASK_NEW - * is produced any msgs associated with an earlier incarnation - * have been consumed. - */ - p->inhibit_task_msgs = enclave_abi(e); + if (!task_deliver_msg_departed(rq, p)) { + free_status_word_locked(e, p->ghost.status_word); + } else { + /* + * Inhibit all msgs produced by this task until the + * agent acknowledges receipt of TASK_DEPARTED msg + * (implicitly by freeing the status_word). This ensures + * that when TASK_NEW is produced any msgs associated + * with an earlier incarnation have been consumed. + */ + p->inhibit_task_msgs = enclave_abi(e); - /* - * Track the task in the enclave's 'inhibited_task_list'. - * - * We must do this because a departed task can re-enter ghost - * while it is exiting and subsequently lose its identity - * (e.g. see go/kcl/390722). It is possible for the task to - * enter a different enclave in which case the fallback in - * kcl/390722 is ineffective. - * - * TODO: declare enclave failure if agent does not free the - * status_word of a departed task within X msec (X should be - * smaller than the e->max_unscheduled timeout): - * - misbehavior of one enclave should not result in a - * (false-positive) failure of a different enclave. - * - limit the task_struct memory held hostage while they are - * on the inhibited_task_list. - */ - get_task_struct(p); - WARN_ON_ONCE(!list_empty(&p->inhibited_task_list)); - list_add_tail(&p->inhibited_task_list, &e->inhibited_task_list); + /* + * Track the task in the enclave's + * 'inhibited_task_list'. + * + * We must do this because a departed task can re-enter + * ghost while it is exiting and subsequently lose its + * identity (e.g. see go/kcl/390722). It is possible for + * the task to enter a different enclave in which case + * the fallback in kcl/390722 is ineffective. + * + * TODO: declare enclave failure if agent does not free + * the status_word of a departed task within X msec (X + * should be smaller than the e->max_unscheduled + * timeout): + * - misbehavior of one enclave should not result in a + * (false-positive) failure of a different enclave. + * - limit the task_struct memory held hostage while + * they are on the inhibited_task_list. + */ + get_task_struct(p); + WARN_ON_ONCE(!list_empty(&p->inhibited_task_list)); + list_add_tail(&p->inhibited_task_list, + &e->inhibited_task_list); + } } else { /* * Annotate the status_word so it can be freed by the agent. @@ -4226,9 +4231,17 @@ static void release_from_ghost(struct rq *rq, struct task_struct *p) * before delivery of the TASK_DEAD message below. */ ghost_sw_set_flag(p->ghost.status_word, GHOST_SW_F_CANFREE); - task_deliver_msg_dead(rq, p); + /* + * Automatically free the status_word if we didn't deliver a + * TASK_MSG_DEAD but only for non-agent ghost tasks. + * + * Otherwise, userspace will spin-wait for GHOST_SW_F_CANFREE as + * a sign that the agent fully exited, and then it destroys the + * SW. + */ + if (!task_deliver_msg_dead(rq, p) && !is_agent(rq, p)) + free_status_word_locked(e, p->ghost.status_word); } - WARN_ON_ONCE(!p->inhibit_task_msgs && p->state != TASK_DEAD); /* status_word is off-limits to the kernel */ p->ghost.status_word = NULL; From 5c33bc585a25bf66978bfccbb4f1f30a29cda103 Mon Sep 17 00:00:00 2001 From: Barret Rhoden Date: Wed, 6 Apr 2022 17:53:05 -0400 Subject: [PATCH 164/165] sched/ghost: add ctl command to discover tasks "discover tasks" will generate a task_new for each task in the enclave. This is a mechanism for live update: an agent can discover tasks that were already in the enclave before it started handling messages. Userspace agents that inherit from BasicDispatchScheduler() have their own discovery mechanism based on the StatusWordTable: for each status word, associate the task with the agent's channel. We use association to make sure we run exactly one TaskNew() for each task, including tasks that arrive concurrently. For agents that do not have a substantial userspace presence, such as the biff scheduler (bpf-only), it's simpler to generate task_new messages. Your scheduler just needs to ignore task_new for tasks it already knows about. Tested: agent_biff handoff Effort: sched/ghost Signed-off-by: Barret Rhoden Change-Id: I70d796d11074b4cb7d638d8531cead9fe3ef0b2b Origin-9xx-SHA1: 9dbec5a13eba0a8cce16bc2582f0191afb26aaa5 Rebase-Tested-11xx: compiled --- kernel/sched/ghost.c | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/kernel/sched/ghost.c b/kernel/sched/ghost.c index 61c2bd08a55f..0941de6b41b3 100644 --- a/kernel/sched/ghost.c +++ b/kernel/sched/ghost.c @@ -2170,6 +2170,48 @@ static void __enclave_remove_task(struct ghost_enclave *e, e->nr_tasks--; } +/* + * Run f() on every task in the enclave. There's no guarantee p is still in the + * enclave, so f() may have to check for it. + */ +static int enclave_for_each_task(struct ghost_enclave *e, + void (*f)(struct ghost_enclave *e, + struct task_struct *p, void *arg), + void *arg) +{ + unsigned long irq_fl; + struct task_struct **for_each; + size_t nr_tasks = 0; + struct task_struct *p; + int i; + + spin_lock_irqsave(&e->lock, irq_fl); + + /* Don't hold e->lock during f(), since f() may grab the rq lock. */ + for_each = kcalloc(e->nr_tasks, sizeof(struct task_struct *), + GFP_ATOMIC); + if (WARN_ON_ONCE(!for_each)) { + spin_unlock_irqrestore(&e->lock, irq_fl); + return -ENOMEM; + } + list_for_each_entry(p, &e->task_list, ghost.task_list) { + if (WARN_ON_ONCE(nr_tasks >= e->nr_tasks)) + break; + get_task_struct(p); + for_each[nr_tasks++] = p; + } + spin_unlock_irqrestore(&e->lock, irq_fl); + + for (i = 0; i < nr_tasks; i++) { + p = for_each[i]; + f(e, p, arg); + put_task_struct(p); + } + kfree(for_each); + + return 0; +} + static void enclave_add_sw_region(struct ghost_enclave *e, struct ghost_sw_region *region) { @@ -6653,6 +6695,23 @@ static int create_sw_region(struct kernfs_open_file *ctl_of, unsigned int id, return err; } +static void generate_task_new(struct ghost_enclave *e, struct task_struct *p, + void *arg) +{ + struct rq_flags rf; + struct rq *rq; + + rq = task_rq_lock(p, &rf); + /* + * Make sure p is in the enclave. It could have departed after we + * unlocked the enclave. P could also have rejoined too, and it's OK + * to send another task_new. + */ + if (p->ghost.enclave == e) + task_deliver_msg_task_new(rq, p, task_on_rq_queued(p)); + task_rq_unlock(rq, p, &rf); +} + static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, size_t len, loff_t off) { @@ -6680,6 +6739,11 @@ static ssize_t gf_ctl_write(struct kernfs_open_file *of, char *buf, return err; } else if (!strcmp(buf, "disable my bpf_prog_load")) { current->group_leader->ghost.bpf_cannot_load_prog = 1; + } else if (!strcmp(buf, "discover tasks")) { + err = enclave_for_each_task(of_to_e(of), generate_task_new, + NULL); + if (err) + return err; } else { pr_err("%s: bad cmd :%s:", __func__, buf); return -EINVAL; From c21b36f87663efa2189876b2caa701fe74e72adf Mon Sep 17 00:00:00 2001 From: Hannah Pan Date: Wed, 13 Jul 2022 17:03:48 -0700 Subject: [PATCH 165/165] ghost: add #define GHOST_BPF to scripts/bpf_helpers_doc.py Add #define GHOST_BPF to scripts/bpf_helpers_doc.py so that the GHOST_BPF macro is generated into bpf_helper_defs.h, which makes it visible to common.bpf.h (see cl/460745992). This allows us to guard the magic numbers there with #ifndef GHOST_BPF. Tested: pushed this change to test branch on github repo, edited WORKSPACE file locally in userspace repo and was able to build on GCE instance https://github.com/google/ghost-kernel/commit/eb2fd69a1928aaf818a6628eef93cf93b98ca5b6 Effort: sched/ghost Change-Id: I9edc790956aa92f4ca157180891c7fd6727a99d6 --- scripts/bpf_helpers_doc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/bpf_helpers_doc.py b/scripts/bpf_helpers_doc.py index c872fa4988da..5970c27d14fe 100755 --- a/scripts/bpf_helpers_doc.py +++ b/scripts/bpf_helpers_doc.py @@ -532,6 +532,8 @@ def print_header(self): header = '''\ /* This is auto-generated file. See bpf_helpers_doc.py for details. */ +#define GHOST_BPF + /* Forward declarations of BPF structs */''' print(header)