5.11 ghost current version

Gconfig.ghost

@@ -0,0 +1,2 @@
+x86_64:CONFIG_SCHED_CLASS_GHOST=y
+x86_64 and debug:CONFIG_DEBUG_GHOST=y

README.md

@@ -0,0 +1,25 @@
+# ghOSt: Fast & Flexible User-Space Delegation of Linux Scheduling
+
+ghOSt is a general-purpose delegation of scheduling policy implemented on top of
+the Linux kernel. The ghOSt framework provides a rich API that receives
+scheduling decisions for processes from userspace and actuates them as
+transactions. Programmers can use any language or tools to develop policies,
+which can be upgraded without a machine reboot. ghOSt supports policies for a
+range of scheduling objectives, from µs-scale latency, to throughput, to energy
+efficiency, and beyond, and incurs low overheads for scheduling actions. Many
+policies are just a few hundred lines of code. Overall, ghOSt provides a
+performant framework for delegation of thread scheduling policy to userspace
+processes that enables policy optimization, non-disruptive upgrades, and fault
+isolation.
+
+[SOSP '21 Paper](https://dl.acm.org/doi/10.1145/3477132.3483542)\
+[SOSP '21 Talk](https://youtu.be/j4ABe4dsbIY)
+
+You must compile and install this kernel in order to use the [ghOSt userspace
+component](https://www.github.com/google/ghost-userspace).
+
+This kernel is Linux 5.11. We recommend installing Ubuntu 20.04 LTS, which ships
+with Linux 5.4 by default, as the distribution. Compile this kernel on Ubuntu
+20.04 and then replace the existing 5.4 kernel with this one.
+
+This is not an officially supported Google product.

arch/x86/include/asm/preempt.h

@@ -94,11 +94,18 @@ static __always_inline bool __preempt_count_dec_and_test(void)
 	return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));
 }
 
+#ifdef CONFIG_SCHED_CLASS_GHOST
+extern void ghost_commit_greedy_txn(void);
+#endif
+
 /*
  * Returns true when we need to resched and can (barring IRQ state).
  */
 static __always_inline bool should_resched(int preempt_offset)
 {
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	ghost_commit_greedy_txn();
+#endif
 	return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
 }
 

fs/eventpoll.c

@@ -1823,7 +1823,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
 		 * the same lock on wakeup ep_poll_callback() side, so it
 		 * is safe to avoid an explicit barrier.
 		 */
-		__set_current_state(TASK_INTERRUPTIBLE);
+		__set_current_state(TASK_INTERRUPTIBLE|__TASK_DEFERRABLE_WAKEUP);
 
 		/*
 		 * Do the final check under the lock. ep_scan_ready_list()

fs/kernfs/file.c

@@ -843,6 +843,29 @@ static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait)
 	return ret;
 }
 
+static long kernfs_fop_ioctl(struct file *file, unsigned int cmd,
+			     unsigned long arg)
+{
+	struct kernfs_open_file *of = kernfs_of(file);
+	const struct kernfs_ops *ops;
+	long ret;
+
+	/*
+	 * Not locking of->mutex: you can have concurrent ioctl calls on the
+	 * same file.
+	 */
+	if (!kernfs_get_active(of->kn))
+		return -ENODEV;
+	ops = kernfs_ops(of->kn);
+	if (ops->ioctl)
+		ret = ops->ioctl(of, cmd, arg);
+	else
+		ret = -ENOIOCTLCMD;
+	kernfs_put_active(of->kn);
+
+	return ret;
+}
+
 static void kernfs_notify_workfn(struct work_struct *work)
 {
 	struct kernfs_node *kn;
@@ -948,11 +971,19 @@ const struct file_operations kernfs_file_fops = {
 	.open		= kernfs_fop_open,
 	.release	= kernfs_fop_release,
 	.poll		= kernfs_fop_poll,
+	.unlocked_ioctl	= kernfs_fop_ioctl,
 	.fsync		= noop_fsync,
 	.splice_read	= generic_file_splice_read,
 	.splice_write	= iter_file_splice_write,
 };
 
+struct kernfs_node *kernfs_node_from_file(struct file *file)
+{
+	if (file->f_op != &kernfs_file_fops)
+		return NULL;
+	return file->f_inode->i_private;
+}
+
 /**
  * __kernfs_create_file - kernfs internal function to create a file
  * @parent: directory to create the file in

fs/proc/base.c

@@ -3156,6 +3156,11 @@ static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns,
 static const struct file_operations proc_task_operations;
 static const struct inode_operations proc_task_inode_operations;
 
+#ifdef CONFIG_SCHED_CLASS_GHOST
+static const struct file_operations proc_ghost_dir_operations;
+static const struct inode_operations proc_ghost_dir_inode_operations;
+#endif
+
 static const struct pid_entry tgid_base_stuff[] = {
 	DIR("task",       S_IRUGO|S_IXUGO, proc_task_inode_operations, proc_task_operations),
 	DIR("fd",         S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations),
@@ -3266,6 +3271,10 @@ static const struct pid_entry tgid_base_stuff[] = {
 #ifdef CONFIG_SECCOMP_CACHE_DEBUG
 	ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache),
 #endif
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	DIR("ghost",	S_IRUGO|S_IXUGO, proc_ghost_dir_inode_operations,
+	    proc_ghost_dir_operations),
+#endif
 };
 
 static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx)
@@ -3831,3 +3840,58 @@ void __init set_proc_pid_nlink(void)
 	nlink_tid = pid_entry_nlink(tid_base_stuff, ARRAY_SIZE(tid_base_stuff));
 	nlink_tgid = pid_entry_nlink(tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff));
 }
+
+#ifdef CONFIG_SCHED_CLASS_GHOST
+
+static ssize_t proc_ghost_gtid_read(struct file *file, char __user *buf,
+				    size_t count, loff_t *ppos)
+{
+	struct task_struct *task = get_proc_task(file_inode(file));
+	char buffer[32];
+	size_t len;
+
+	if (!task)
+		return -ESRCH;
+
+	len = snprintf(buffer, sizeof(buffer), "%ld\n",
+		       task->gtid);
+	WARN_ON_ONCE(len >= sizeof(buffer));
+	put_task_struct(task);
+	return simple_read_from_buffer(buf, count, ppos, buffer, len);
+}
+
+static const struct file_operations proc_ghost_gtid_operations = {
+	.read           = proc_ghost_gtid_read,
+	.llseek         = generic_file_llseek,
+};
+
+static const struct pid_entry ghost_dir_stuff[] = {
+	REG("gtid",     S_IRUGO, proc_ghost_gtid_operations),
+};
+
+static struct dentry *proc_ghost_dir_lookup(struct inode *dir,
+					    struct dentry *dentry,
+					    unsigned int flags)
+{
+	return proc_pident_lookup(dir, dentry, ghost_dir_stuff,
+				  ghost_dir_stuff + ARRAY_SIZE(ghost_dir_stuff));
+}
+
+static int proc_ghost_dir_readdir(struct file *file, struct dir_context *ctx)
+{
+	return proc_pident_readdir(file, ctx, ghost_dir_stuff,
+				   ARRAY_SIZE(ghost_dir_stuff));
+}
+
+static const struct inode_operations proc_ghost_dir_inode_operations = {
+	.lookup         = proc_ghost_dir_lookup,
+	.getattr        = pid_getattr,
+};
+
+static const struct file_operations proc_ghost_dir_operations = {
+	.read           = generic_read_dir,
+	.iterate_shared = proc_ghost_dir_readdir,
+	.llseek         = default_llseek,
+};
+
+#endif

fs/timerfd.c

@@ -44,6 +44,9 @@ struct timerfd_ctx {
 	struct list_head clist;
 	spinlock_t cancel_lock;
 	bool might_cancel;
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	struct __kernel_timerfd_ghost timerfd_ghost;
+#endif
 };
 
 static LIST_HEAD(cancel_list);
@@ -64,11 +67,26 @@ static void timerfd_triggered(struct timerfd_ctx *ctx)
 {
 	unsigned long flags;
 
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	struct __kernel_timerfd_ghost timerfd_ghost = {
+		.enabled = false,
+	};
+#endif
+
 	spin_lock_irqsave(&ctx->wqh.lock, flags);
 	ctx->expired = 1;
 	ctx->ticks++;
 	wake_up_locked_poll(&ctx->wqh, EPOLLIN);
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	if (unlikely(&ctx->timerfd_ghost.enabled))
+		timerfd_ghost = ctx->timerfd_ghost;
+#endif
 	spin_unlock_irqrestore(&ctx->wqh.lock, flags);
+
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	if (unlikely(timerfd_ghost.enabled))
+		ghost_timerfd_triggered(&timerfd_ghost);
+#endif
 }
 
 static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr)
@@ -435,9 +453,15 @@ SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags)
 	return ufd;
 }
 
-static int do_timerfd_settime(int ufd, int flags, 
+#ifdef CONFIG_SCHED_CLASS_GHOST
+int do_timerfd_settime(int ufd, int flags, const struct itimerspec64 *new,
+		       struct itimerspec64 *old,
+		       struct __kernel_timerfd_ghost *ktfd)
+#else
+static int do_timerfd_settime(int ufd, int flags,
 		const struct itimerspec64 *new,
 		struct itimerspec64 *old)
+#endif
 {
 	struct fd f;
 	struct timerfd_ctx *ctx;
@@ -501,7 +525,12 @@ static int do_timerfd_settime(int ufd, int flags,
 	 * Re-program the timer to the new value ...
 	 */
 	ret = timerfd_setup(ctx, flags, new);
-
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	if (ktfd)
+		memcpy(&ctx->timerfd_ghost, ktfd, sizeof(*ktfd));
+	else
+		ctx->timerfd_ghost.enabled = false;
+#endif
 	spin_unlock_irq(&ctx->wqh.lock);
 	fdput(f);
 	return ret;
@@ -548,7 +577,11 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
 
 	if (get_itimerspec64(&new, utmr))
 		return -EFAULT;
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	ret = do_timerfd_settime(ufd, flags, &new, &old, NULL);
+#else
 	ret = do_timerfd_settime(ufd, flags, &new, &old);
+#endif
 	if (ret)
 		return ret;
 	if (otmr && put_itimerspec64(&old, otmr))
@@ -576,7 +609,11 @@ SYSCALL_DEFINE4(timerfd_settime32, int, ufd, int, flags,
 
 	if (get_old_itimerspec32(&new, utmr))
 		return -EFAULT;
+#ifdef CONFIG_SCHED_CLASS_GHOST
+	ret = do_timerfd_settime(ufd, flags, &new, &old, NULL);
+#else
 	ret = do_timerfd_settime(ufd, flags, &new, &old);
+#endif
 	if (ret)
 		return ret;
 	if (otmr && put_old_itimerspec32(&old, otmr))

include/asm-generic/vmlinux.lds.h

@@ -125,15 +125,29 @@
  * used to determine the order of the priority of each sched class in
  * relation to each other.
  */
+#ifdef CONFIG_SCHED_CLASS_GHOST
 #define SCHED_DATA				\
 	STRUCT_ALIGN();				\
 	__begin_sched_classes = .;		\
 	*(__idle_sched_class)			\
+	*(__ghost_sched_class)			\
 	*(__fair_sched_class)			\
 	*(__rt_sched_class)			\
 	*(__dl_sched_class)			\
+	*(__ghost_agent_sched_class)		\
 	*(__stop_sched_class)			\
 	__end_sched_classes = .;
+#else
+#define SCHED_DATA				\
+	STRUCT_ALIGN();				\
+	__begin_sched_classes = .;		\
+	*(__idle_sched_class)			\
+	*(__fair_sched_class)			\
+	*(__rt_sched_class)			\
+	*(__dl_sched_class)			\
+	*(__stop_sched_class)			\
+	__end_sched_classes = .;
+#endif
 
 /* The actual configuration determine if the init/exit sections
  * are handled as text/data or they can be discarded (which
@@ -409,13 +423,24 @@
 	__end_ro_after_init = .;
 #endif
 
+#ifdef CONFIG_SCHED_CLASS_GHOST
+#define	GHOST_ABI_RODATA		\
+	STRUCT_ALIGN();			\
+	__begin_ghost_abi = .;		\
+	*(.rodata.ghost_abi)		\
+	__end_ghost_abi = .;
+#else
+#define	GHOST_ABI_RODATA
+#endif
+
 /*
  * Read only Data
  */
 #define RO_DATA(align)							\
 	. = ALIGN((align));						\
 	.rodata           : AT(ADDR(.rodata) - LOAD_OFFSET) {		\
 		__start_rodata = .;					\
+		GHOST_ABI_RODATA					\
 		*(.rodata) *(.rodata.*)					\
 		SCHED_DATA						\
 		RO_AFTER_INIT_DATA	/* Read only after init */	\

include/linux/bpf_types.h

@@ -77,7 +77,12 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LSM, lsm,
 	       void *, void *)
 #endif /* CONFIG_BPF_LSM */
 #endif
-
+#ifdef CONFIG_SCHED_CLASS_GHOST
+BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_SCHED, ghost_sched, struct bpf_ghost_sched,
+	      struct bpf_ghost_sched_kern)
+BPF_PROG_TYPE(BPF_PROG_TYPE_GHOST_MSG, ghost_msg, struct bpf_ghost_msg,
+	      struct bpf_ghost_msg_kern)
+#endif
 BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_PROG_ARRAY, prog_array_map_ops)

include/linux/ghost.h

@@ -0,0 +1,13 @@
+#ifndef _LINUX_GHOST_H_
+#define _LINUX_GHOST_H_
+
+/*
+ * Empty header to satisfy inclusions like <linux/ghost.h> which if this
+ * file were not present would end up resolving to <uapi/linux/ghost.h>
+ *
+ * We care about this because including <uapi/linux/ghost.h> has abi
+ * implications so we want this header to be included explicitly.
+ */
+
+#endif  /* _LINUX_GHOST_H_ */
+

include/linux/kernfs.h

@@ -263,6 +263,9 @@ struct kernfs_ops {
 	__poll_t (*poll)(struct kernfs_open_file *of,
 			 struct poll_table_struct *pt);
 
+	long (*ioctl)(struct kernfs_open_file *of, unsigned int cmd,
+		      unsigned long arg);
+
 	int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma);
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -357,6 +360,7 @@ void kernfs_get(struct kernfs_node *kn);
 void kernfs_put(struct kernfs_node *kn);
 
 struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry);
+struct kernfs_node *kernfs_node_from_file(struct file *file);
 struct kernfs_root *kernfs_root_from_sb(struct super_block *sb);
 struct inode *kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn);