Better C API + native Rust API

[mtjhrc]

For 2.0 we want to add a native Rust API, but we still plan to support users using a C API.

The C API will have breaking changes too - we want to namely don't having any implicit devices and providing more clear errors.

We don't want 2 entry points for every call in libkrun - C API should be a layer built on top of Rust API. The new C API and Rust API should be reasonably similar which is why I think we need to think about this together.

[mtjhrc]

Here's a rough API I have so far, if anyone thinks this is going in the wrong direction:

Using Rust you con configure libkrun as this:

let rootfs = FsDevice::new("/dev/root", new_root).context("create rootfs")?;

let mut console_builder = ConsoleDevice::builder();
let payload = Init::builder(&rootfs, &mut console_builder)
    .exec(guest_cmd, &guest_args)
    .context("exec")?
    .workdir("/")
    .context("workdir")?
    .env(&["HOME=/root", "TERM=xterm-256color"])
    .context("env")?
    .build()
    .context("build payload")?;
let console = console_builder.build().context("build console")?;

let balloon = BalloonDevice::new().context("create balloon")?;
let rng = RngDevice::new().context("create rng")?;

let mut devices = MmioDeviceManager::new();
devices.add(rootfs);
devices.add(console);
devices.add(balloon);
devices.add(rng);

let mut vmm = VmmBuilder::new()
    .vcpus(2)
    .context("vcpus")?
    .ram_mib(512)
    .context("ram")?
    .payload(payload)
    .devices(devices)
    .build()
    .context("build vmm")?;
vmm.run();

This API Rust API can actually be called 2 different ways just by changing dependency in Cargo.toml from libkrun to libkrun-via-cdylib:

• natively linked statically as a rust crate
• dynamcially linked via C abi - this acts as Rust bindings to a C

corresponding C side looks like:

#define KRUN_CALL(call)                                                        \
    do {                                                                       \
        KrunError _err = (call);                                               \
        if (_err.code) {                                                       \
            fprintf(stderr, "%s failed: %s\n", #call,                          \
                    krun_error_message(&_err));                                \
            krun_error_free(&_err);                                            \
            return 1;                                                          \
        }                                                                      \
    } while (0)

KRUN_CALL(krun_init_log(KRUN_LOG_TARGET_DEFAULT, KRUN_LOG_LEVEL_WARN,
                      KRUN_LOG_STYLE_AUTO));

KrunFsDevice rootfs;
KRUN_CALL(
  krun_fs_device_new(KRUN_STR("/dev/root"), KRUN_STR(new_root), &rootfs));

KrunConsoleBuilder console_builder = krun_console_device_builder();
KrunInitBuilder payload_builder =
  krun_init_builder(rootfs, console_builder);

KrunStr guest_args[argc - 3];
for (int i = 3; i < argc; i++)
  guest_args[i - 3] = KRUN_STR(argv[i]);

KRUN_CALL(krun_init_builder_exec(&payload_builder, KRUN_STR(guest_cmd),
                               guest_args, argc - 3));
KRUN_CALL(krun_init_builder_workdir(&payload_builder, KRUN_STR("/")));

KrunStr env[] = {KRUN_STR("HOME=/root"), KRUN_STR("TERM=xterm-256color")};
KRUN_CALL(krun_init_builder_env(&payload_builder, env, 2));

KrunInit payload;
KRUN_CALL(krun_init_builder_build(payload_builder, &payload));

KrunConsoleDevice console;
KRUN_CALL(krun_console_builder_build(console_builder, &console));

KrunBalloonDevice balloon;
KRUN_CALL(krun_balloon_device_new(&balloon));

KrunRngDevice rng;
KRUN_CALL(krun_rng_device_new(&rng));

KrunMmioDeviceManager devices = krun_mmio_device_manager_new();
krun_mmio_device_manager_add(devices, rootfs);
krun_mmio_device_manager_add(devices, console);
krun_mmio_device_manager_add(devices, balloon);
krun_mmio_device_manager_add(devices, rng);

KrunVmmBuilder builder = krun_vmm_builder_new();
KRUN_CALL(krun_vmm_builder_vcpus(&builder, 2));
KRUN_CALL(krun_vmm_builder_ram_mib(&builder, 512));
krun_vmm_builder_payload(&builder, payload);
krun_vmm_builder_devices(&builder, devices);

KrunVmm vmm;
KRUN_CALL(krun_vmm_builder_build(builder, &vmm));

krun_vmm_run(vmm);
krun_vmm_destroy(vmm);

Autogenerated C header

#ifndef LIBKRUN_H
#define LIBKRUN_H

#include <stdint.h>
#include <stdbool.h>
#include <string.h>

typedef void* KrunMmioDeviceManager;
typedef void* KrunFsDevice;
typedef void* KrunConsoleDevice;
typedef void* KrunConsoleBuilder;
typedef void* KrunBalloonDevice;
typedef void* KrunRngDevice;
typedef void* KrunInit;
typedef void* KrunInitBuilder;
typedef void* KrunVmmBuilder;
typedef void* KrunVmm;

/* Caller must ensure data is valid UTF-8 */
typedef struct {
    const char* data;
    uintptr_t len;
} KrunStr;

/* Caller must ensure data is a valid UTF-8 path */
typedef KrunStr KrunPath;

typedef struct {
    const uint8_t* data;
    uintptr_t len;
} KrunBytes;

#define KRUN_STR(s) ((KrunStr){ .data = (s), .len = strlen(s) })
#if defined(__GNUC__)
#define KRUN_BYTES(arr) ({ \
    _Static_assert( \
        !__builtin_types_compatible_p(typeof(arr), typeof(&(arr)[0])), \
        "KRUN_BYTES() requires an array, not a pointer"); \
    ((KrunBytes){ .data = (const uint8_t*)(arr), .len = sizeof(arr) }); \
})
#else
#define KRUN_BYTES(arr) \
    ((KrunBytes){ .data = (const uint8_t*)(arr), .len = sizeof(arr) })
#endif
/* Header auto-generated by ffier */

/* Error ------------------------------------------------------------- */

typedef struct {
    uint64_t code;
    char* _msg; /* private */
} KrunError;

#define KRUN_ERROR_INVALID_PARAM 100
#define KRUN_ERROR_DUPLICATE_DEVICE 101
#define KRUN_ERROR_DEVICE_LIMIT_EXCEEDED 102
#define KRUN_ERROR_MISSING_CONFIG 103
#define KRUN_ERROR_CONFLICTING_CONFIG 104
#define KRUN_ERROR_OUT_OF_RANGE 105
#define KRUN_ERROR_FILE_NOT_FOUND 200
#define KRUN_ERROR_PERMISSION_DENIED 201
#define KRUN_ERROR_RESOURCE_ALLOC 202
#define KRUN_ERROR_BAD_FD 203
#define KRUN_ERROR_BACKEND_UNAVAILABLE 300
#define KRUN_ERROR_FEATURE_DISABLED 301
#define KRUN_ERROR_DISK_FORMAT_ERROR 302
#define KRUN_ERROR_ALREADY_STARTED 400
#define KRUN_ERROR_VALIDATION_FAILED 401
#define KRUN_ERROR_HYPERVISOR_ERROR 402
#define KRUN_ERROR_BOOT_ERROR 403
#define KRUN_ERROR_INTERNAL 900

const char* krun_error_message(const KrunError* err);
void krun_error_free(KrunError* err);

/* MmioDeviceManager ------------------------------------------------- */

typedef void* KrunAttachDevice; /* KrunFsDevice | KrunConsoleDevice | KrunBalloonDevice | KrunRngDevice */
/**
 * Create an empty device manager.
 */
KrunMmioDeviceManager krun_mmio_device_manager_new(void);
/**
 * Add a device to this manager.
 *
 * Devices are attached in the order they are added. The device must
 * implement [`AttachDevice`] — all built-in device types
 * (`FsDevice`, `ConsoleDevice`, etc.) implement this trait.
 *
 * @param device
 */
void krun_mmio_device_manager_add(KrunMmioDeviceManager handle, KrunAttachDevice device);
void krun_mmio_device_manager_destroy(KrunMmioDeviceManager handle);

/* FsDevice ---------------------------------------------------------- */

/**
 * Create a new virtiofs device.
 *
 * @param tag the filesystem tag visible to the guest (e.g. `"/dev/root"`).
 * @param host_path the host directory to share.
 * @param[out] result
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_fs_device_new(KrunStr tag, KrunStr host_path, KrunFsDevice* result);
/**
 * Set the size of the DAX (direct access) shared memory window.
 *
 * When set, the guest can memory-map files from the shared filesystem
 * directly into its address space, avoiding data copies. If not set,
 * no DAX window is allocated.
 *
 * @param bytes
 */
void krun_fs_device_set_dax_window_size(KrunFsDevice handle, uint64_t bytes);
void krun_fs_device_destroy(KrunFsDevice handle);

/* ConsoleDevice ----------------------------------------------------- */

/**
 * Create a new console builder.
 */
KrunConsoleBuilder krun_console_device_builder(void);
void krun_console_device_destroy(KrunConsoleDevice handle);

/* ConsoleBuilder ---------------------------------------------------- */

/**
 * Add a TTY-backed port to the console.
 *
 * If the fd refers to a real terminal, raw mode will be enabled on it
 * when the VM starts, and restored on shutdown.
 *
 * @param name the port name visible to the guest (e.g. `"tty0"`).
 * @param tty_fd borrowed fd for the host TTY; duplicated internally, caller retains ownership.
 * @param[out] result The zero-based port index, usable with [`set_kernel_console`](ConsoleBuilder::set_kernel_console).
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_console_builder_add_tty_port(KrunConsoleBuilder handle, KrunStr name, int tty_fd, uint32_t* result);
/**
 * Designate a port as the kernel console (`console=hvcN`).
 *
 * @param port_index a value returned by [`add_tty_port`](ConsoleBuilder::add_tty_port).
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_console_builder_set_kernel_console(KrunConsoleBuilder handle, uint32_t port_index);
/**
 * Build the console device. At least one port must have been added.
 *
 * @param[out] result
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_console_builder_build(KrunConsoleBuilder handle, KrunConsoleDevice* result);
void krun_console_builder_destroy(KrunConsoleBuilder handle);

/* BalloonDevice ----------------------------------------------------- */

/**
 * Create a new balloon device.
 *
 * @param[out] result
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_balloon_device_new(KrunBalloonDevice* result);
void krun_balloon_device_destroy(KrunBalloonDevice handle);

/* RngDevice --------------------------------------------------------- */

/**
 * Create a new RNG device.
 *
 * @param[out] result
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_rng_device_new(KrunRngDevice* result);
void krun_rng_device_destroy(KrunRngDevice handle);

/* Init -------------------------------------------------------------- */

/**
 * Create a new init payload builder.
 *
 * @param _rootfs the root filesystem device (reserved for future validation).
 * @param console console builder; an output-only port for boot messages is added automatically.
 */
KrunInitBuilder krun_init_builder(KrunFsDevice _rootfs, KrunConsoleBuilder console);
void krun_init_destroy(KrunInit handle);

/* InitBuilder ------------------------------------------------------- */

/**
 * Auto-detect console setup.
 *
 * Tries /dev/tty (the controlling terminal) first. If available, creates
 * a single TTY port for payload I/O. Otherwise falls back to separate
 * krun-payload-stdin/stdout/stderr redirect ports on the stdio fds.
 *
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_console_auto(KrunInitBuilder* handle);
/**
 * Set up a single TTY port for payload I/O.
 *
 * The payload's stdin/stdout/stderr will all be connected to this
 * terminal. Raw mode is enabled automatically if `tty_fd` is a
 * real terminal.
 *
 * @param tty_fd
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_console_tty(KrunInitBuilder* handle, int tty_fd);
/**
 * Set up separate redirect ports for payload stdin, stdout, and stderr.
 *
 * Each fd is duplicated internally. Pass `-1` (or any negative value)
 * to skip a particular stream.
 *
 * @param stdin_fd
 * @param stdout_fd
 * @param stderr_fd
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_console_redirects(KrunInitBuilder* handle, int32_t stdin_fd, int32_t stdout_fd, int32_t stderr_fd);
/**
 * Set the command to execute inside the guest.
 *
 * @param exec_path absolute path to the executable within the guest rootfs.
 * @param args command-line arguments (not including argv\[0\]).
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_exec(KrunInitBuilder* handle, KrunStr exec_path, const KrunStr* args, uintptr_t args_len);
/**
 * Set environment variables for the guest process.
 *
 * @param env each string should be in `KEY=VALUE` format.
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_env(KrunInitBuilder* handle, const KrunStr* env, uintptr_t env_len);
/**
 * Set the working directory for the guest process.
 *
 * @param path absolute path within the guest rootfs.
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_workdir(KrunInitBuilder* handle, KrunStr path);
/**
 * Build the init payload.
 *
 * Requires [`exec`](InitBuilder::exec) to have been called. If no
 * console was explicitly configured (via [`console_tty`](InitBuilder::console_tty)
 * or [`console_redirects`](InitBuilder::console_redirects)), auto-detection
 * is used: `/dev/tty` if available, otherwise stdin/stdout/stderr.
 *
 * @param[out] result
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_init_builder_build(KrunInitBuilder handle, KrunInit* result);
void krun_init_builder_destroy(KrunInitBuilder handle);

/* VmmBuilder -------------------------------------------------------- */

typedef void* KrunPayload; /* KrunInit */
/**
 * Create a new VM builder with no configuration.
 */
KrunVmmBuilder krun_vmm_builder_new(void);
/**
 * Set the number of virtual CPUs. Must be at least 1.
 *
 * @param count
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_vmm_builder_vcpus(KrunVmmBuilder* handle, uint8_t count);
/**
 * Set the amount of guest RAM in mebibytes. Must be at least 1.
 *
 * @param mib
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_vmm_builder_ram_mib(KrunVmmBuilder* handle, uint32_t mib);
/**
 * Set the payload to run inside the VM.
 *
 * Currently the only payload type is `Init`, which runs a process
 * as PID 1 inside the guest using the built-in krun init.
 *
 * @param payload
 */
void krun_vmm_builder_payload(KrunVmmBuilder* handle, KrunPayload payload);
/**
 * Set the device manager containing all virtio devices.
 *
 * The device manager determines which transport bus is used (currently
 * only [`MmioDeviceManager`] for virtio-mmio).
 *
 * @param devices
 */
void krun_vmm_builder_devices(KrunVmmBuilder* handle, KrunMmioDeviceManager devices);
/**
 * Build the VM, creating guest memory, attaching devices, and starting
 * vCPUs. All required fields (`vcpus`, `ram_mib`, `payload`, `devices`)
 * must have been set.
 *
 * @param[out] result
 * @return KrunError with code 0 on success, error code on failure.
 */
KrunError krun_vmm_builder_build(KrunVmmBuilder handle, KrunVmm* result);
void krun_vmm_builder_destroy(KrunVmmBuilder handle);

/* Vmm --------------------------------------------------------------- */

/**
 * Run the VM event loop. This call blocks until the VM exits or a
 * fatal error occurs.
 */
void krun_vmm_run(KrunVmm handle);
void krun_vmm_destroy(KrunVmm handle);

/* Log --------------------------------------------------------------- */

#define KRUN_LOG_TARGET_DEFAULT (-1)
#define KRUN_LOG_STYLE_AUTO     0
#define KRUN_LOG_STYLE_ALWAYS   1
#define KRUN_LOG_STYLE_NEVER    2
#define KRUN_LOG_LEVEL_OFF      0
#define KRUN_LOG_LEVEL_ERROR    1
#define KRUN_LOG_LEVEL_WARN     2
#define KRUN_LOG_LEVEL_INFO     3
#define KRUN_LOG_LEVEL_DEBUG    4
#define KRUN_LOG_LEVEL_TRACE    5

KrunError krun_init_log(int target_fd, uint32_t level, uint32_t style);

#endif /* LIBKRUN_H */

The C header, ffi extern "C" functions are automatically generated from Rust using https://github.com/mtjhrc/ffier. Any thoughts about an approach like this? Is that too much magic?

My very WIP branch if you want to check out the details: https://github.com/mtjhrc/libkrun/tree/new-api
I'll post a draft PR later, I want to improve this a bit first (also it would help if we fork of 1.18..)

[jakecorrenti]

let balloon = BalloonDevice::new().context("create balloon")?;
let rng = RngDevice::new().context("create rng")?;

let mut devices = MmioDeviceManager::new();
devices.add(rootfs);
devices.add(console);
devices.add(balloon);
devices.add(rng);

I wonder if here we could do something along the lines of

let mut devices = MmioDeviceBuilder::new()
    .add_rootfs(rootfs)
    .add_console(console)
    .add_balloon()
    .add_rng()
    .build();

Could we get away with doing the device initialization in the builder methods?

let mut vmm = VmmBuilder::new()
.vcpus(2)
.context("vcpus")?
.ram_mib(512)
.context("ram")?
.payload(payload)
.devices(devices)
.build()
.context("build vmm")?;
vmm.run();

[mtjhrc]

I wonder if here we could do something along the lines of

let mut devices = MmioDeviceBuilder::new()
    .add_rootfs(rootfs)
    .add_console(console)
    .add_balloon()
    .add_rng()
    .build();

Tbh I am not really a fan of this:

• One of the constraints I am keeping in mind is opening this up for devices outside of libkrun - we could eventually expose a device trait and let user pass in anything that implements it. Not planned for 2.0, but I want to keep it possible sometime in 2.x without breaking the API.
• Imagine for libkrun 2.1 we to a construction parameter to e.g. a baloon device down the line but we have already added 0-arg add_rng, then you have to add a separate add_balloon_????(balloon).
• what if we add a PcieDeviceBuilder - then you have to mirror a method for every device

---

Could we get away with doing the device initialization in the builder methods?

let mut vmm = VmmBuilder::new()
.vcpus(2)
.context("vcpus")?
.ram_mib(512)
.context("ram")?
.payload(payload)
.devices(devices)
.build()
.context("build vmm")?;
vmm.run();

Not sure what you mean here. Can you clarify?

[jakecorrenti]

I wonder if here we could do something along the lines of

let mut devices = MmioDeviceBuilder::new()
.add_rootfs(rootfs)
.add_console(console)
.add_balloon()
.add_rng()
.build();

Tbh I am not really a fan of this:

* One of the constraints I am keeping in mind is opening this up for devices outside of libkrun - we could eventually expose a device trait and let user pass in anything that implements it. Not planned for 2.0, but I want to keep it possible sometime in 2.x without breaking the API.

* Imagine for libkrun 2.1 we to a construction parameter to e.g. a `baloon` device down the line but we have already added 0-arg `add_rng`, then you have to add a separate `add_balloon_????(balloon)`.

* what if we add a `PcieDeviceBuilder` - then you have to mirror a method for every device

All good points that I agree with