diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 86ddf164..d2da0e05 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,9 +1,21 @@ { + "name": "devicecode-lua", "build": { "dockerfile": "Dockerfile" - // [Optional] You can use build args to set options. e.g. 'VARIANT' below affects the image in the Dockerfile - // "args": { "VARIANT": "buster" } }, - "remoteUser": "vscode", - "postCreateCommand": "bash .devcontainer/postCreateCommand.sh" -} \ No newline at end of file + "remoteUser": "vscode", + "postCreateCommand": "bash .devcontainer/postCreateCommand.sh", + "customizations": { + "vscode": { + "extensions": [ + "ms-azuretools.vscode-docker", + "ms-vscode-remote.remote-containers", + "waderyan.gitblame", + "sumneko.lua", + "rog2.luacheck", + "tomblind.local-lua-debugger-vscode", + "bierner.markdown-mermaid" + ] + } + } +} diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 9a44016b..ac44e472 100755 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -14,6 +14,7 @@ sudo luarocks install bit32 sudo luarocks install cqueues sudo luarocks install http sudo luarocks install luaposix +sudo luarocks install luacheck # install cffi-lua @@ -29,19 +30,9 @@ sudo ninja all sudo ninja test sudo cp cffi.so /usr/local/lib/lua/5.1/cffi.so -# install go -cd /tmp - -arch=$(uname -m) +# pre-commit for CI -if [ "$arch" = "aarch64" ] -then - wget https://go.dev/dl/go1.21.0.linux-arm64.tar.gz -else - wget https://go.dev/dl/go1.21.0.linux-amd64.tar.gz -fi -sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.21.0.linux*.tar.gz -echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.profile -echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc +sudo apt-get install -y pre-commit +pre-commit install -exit 0 \ No newline at end of file +exit 0 diff --git a/.env b/.env new file mode 100644 index 00000000..6bf250bb --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +FIBERS_VER=main +TRIE_VER=main +BUS_VER=main diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..b3978b2a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: devicecode-ci + +on: + pull_request: + branches: + - main + - dev + +jobs: + ci: + runs-on: ubuntu-latest + + steps: + - name: checkout + uses: actions/checkout@v4.2.2 + with: + submodules: recursive + token: ${{ secrets.DEVICECODE_TOKEN }} + + - name: install-luajit + uses: leafo/gh-actions-lua@v10.0.0 + with: + luaVersion: "luajit-openresty" + + - name: install-luarocks + uses: leafo/gh-actions-luarocks@v4.3.0 + + - name: install-luacheck + run: luarocks install luacheck + + - name: install-bit32 + run: luarocks install bit32 + + - name: install-cqueues + run: luarocks install cqueues + + - name: install-http + run: luarocks install http + + - name: install-luaposix + run: luarocks install luaposix + + - name: setup-env + run: make env + + - name: run-lint + run: make lint + + - name: run-tests + if: always() + run: make test-all diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..e5e8e7b5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "src/lua-fibers"] + path = src/lua-fibers + url = https://github.com/jangala-dev/lua-fibers.git +[submodule "src/lua-trie"] + path = src/lua-trie + url = https://github.com/jangala-dev/lua-trie.git +[submodule "src/lua-bus"] + path = src/lua-bus + url = https://github.com/jangala-dev/lua-bus.git diff --git a/.luacheckrc b/.luacheckrc new file mode 100644 index 00000000..eb1dc9c8 --- /dev/null +++ b/.luacheckrc @@ -0,0 +1,4 @@ +std = "lua51+luajit" +ignore = { + "212/self" +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..1ce473c4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: local + hooks: + - id: dc-pre-commit + name: dc-pre-commit + entry: ./pre-commit.sh + language: system diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..7760fd3c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,11 @@ +{ + "recommendations": [ + "ms-azuretools.vscode-docker", + "ms-vscode-remote.remote-containers", + "waderyan.gitblame", + "sumneko.lua", + "rog2.luacheck", + "tomblind.local-lua-debugger-vscode", + "bierner.markdown-mermaid" + ] + } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..e42eccbb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,17 @@ +{ + "editor.tabSize": 2, + "editor.indentSize": "tabSize", + "editor.insertSpaces": true, + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "modifications", + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "never" + }, + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + "json.schemas": [], + "terminal.integrated.defaultProfile.linux": "bash", + "[lua]": { + "editor.tabSize": 4 + }, + } diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..1fb1ebed --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +# Variables +include .env +SRC_DIR := src +BUILD_DIR := build +TEST_DIR := tests +LINTER := luacheck + +# Default target +.PHONY: all +all: env test-all build lint + +# Build: Moves source files and removes and testing from submodules +.PHONY: build +build: + @echo "Building the project..." + @mkdir -p $(BUILD_DIR) + @cp -r $(SRC_DIR)/* $(BUILD_DIR)/ + @cp $(BUILD_DIR)/lua-bus/src/* $(BUILD_DIR) + @rm -r $(BUILD_DIR)/lua-bus + @cp -r $(BUILD_DIR)/lua-fibers/fibers $(BUILD_DIR)/fibers + @rm -r $(BUILD_DIR)/lua-fibers + @cp $(BUILD_DIR)/lua-trie/src/* $(BUILD_DIR) + @rm -r $(BUILD_DIR)/lua-trie + @echo "Build complete." + +# Test: Run the project's test suite +.PHONY: test +test: + @echo "Running project tests..." + @cd $(TEST_DIR) && luajit test.lua + @echo "Tests completed." + +# Test-All: Run the project's and submodule's test suite +.PHONY: test-all +test-all: + @echo "Running all tests..." +# Devicecode tests + @cd $(TEST_DIR) && luajit test.lua +# Fiber tests + @cd $(SRC_DIR)/lua-fibers/tests && luajit test.lua +# Trie tests + @cd $(SRC_DIR)/lua-trie/tests && luajit test.lua +# Bus tests (require movement of fiber and trie then cleanup) + @cp -r $(SRC_DIR)/lua-fibers/fibers $(SRC_DIR)/lua-bus/src + @cp -r $(SRC_DIR)/lua-trie/src/* $(SRC_DIR)/lua-bus/src + @cd $(SRC_DIR)/lua-bus/tests && luajit test.lua + @rm -rf $(SRC_DIR)/lua-bus/src/fibers + @rm -rf $(SRC_DIR)/lua-bus/src/trie.lua + @echo "Tests completed." + +# Env: Initialize environment and update git submodules +.PHONY: env +env: + @echo "Updating git submodules..." + @git submodule update --init --recursive + @cd $(SRC_DIR)/lua-fibers && git checkout $(FIBERS_VER) + @cd $(SRC_DIR)/lua-trie && git checkout $(TRIE_VER) + @cd $(SRC_DIR)/lua-bus && git checkout $(BUS_VER) + @echo "Git submodules updated." + +# Lint: Run the linter to check code quality +.PHONY: lint +lint: + @echo "Running linter..." + @$(LINTER) $(SRC_DIR) $(TEST_DIR) + @echo "Linting complete." + +# Clean: Remove build artifacts +.PHONY: clean +clean: + @echo "Cleaning build artifacts..." + @rm -rf $(BUILD_DIR) + @echo "Clean complete." + +# Help: Display available targets +help: + @echo "Available targets:" + @echo "all - Build, test, lint, and update submodules" + @echo "build - Build the project (removes testing code and documentation)" + @echo "test - Run project test suite" + @echo "test-all - Run project and submodules test suite" + @echo "env - Update and initialize git submodules" + @echo "lint - Run the code linter" + @echo "clean - Remove build artifacts" + @echo "help - Display this help message" diff --git a/README.md b/README.md index 1db86c6e..1d8f4024 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,34 @@ This repository contains the Lua version of [Jangala's](https://www.janga.la) `devicecode`, the program that powers our Big Box and Get Box devices. + +# Using make script +SRC_DIR, TEST_DIR and BUILD_DIR are all optional values +## Initialise dev environment +Before testing or building the code all submodules need to be loaded in. You can select +the version of each submodule in `.env`. +``` +make env SRC_DIR= +``` + +## Build devicecode +Building devicecode creates a folder with only required source files and restructures the submodules +to make a simpler file structure. +``` +make build SRC_DIR= BUILD_DIR= +``` + +## Test devicecode +Devicecode and it's submodules can be tested, to test devicecode only +``` +make test TEST_DIR= +``` +to test devicecode and the submodules +``` +make test-all TEST_DIR= SRC_DIR= +``` + +## Linter +To run the linter +``` +make lint SRC_DIR= TEST_DIR= +``` diff --git a/pre-commit.sh b/pre-commit.sh new file mode 100755 index 00000000..9dc9c416 --- /dev/null +++ b/pre-commit.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +make env +make lint + +ret=$? +if [ $ret -ne 0 ]; then + exit $ret +fi + +make test +ret=$? +if [ $ret -ne 0 ]; then + exit $ret +fi + +exit 0 diff --git a/sprint-docs/service.md b/sprint-docs/service.md new file mode 100644 index 00000000..d0d72da7 --- /dev/null +++ b/sprint-docs/service.md @@ -0,0 +1,69 @@ +# Service structure +## service spawning +A service is created using service.spawn() which will follow the general form +- Publish service active under `/health` +- Run service start function with arguments of bus connection and context (start function should be non-blocking) +- Create a shutdown fiber +- Shutdown fiber will + - wait for a shutdown message on `/control/shutdown` + - check all fibers' state on `/health/fibers/+` + - track which fibers are not showing 'disabled' state yet + - once all fibers are showing disabled it will publish service state disabled and close + +## fiber spawning +A service fiber is a fiber that is spawned with tracking features, spawning follows the steps +- Publish 'fiber init' under `/health/fibers/` +- create fiber and pass context with cancel and values fiber_name and service_name +- when fiber spins up + - publish 'fiber active' + - run the given function with ctx as argument (this function should be blocking) + - when given function returns, publish 'fiber disabled' + +# How to use +## Spawn a service +```lua +service.spawn(service_obj, bus_connection, context) +``` + +## Spawn a service fiber +```lua +service.spawn(name, bus_connection, context, function (fiber_context) + -- do stuff +end +) +``` + +# Example +```lua +local service = require "service" +local bus_mod = require "bus" +local context = require "fibers.context" +local op = require "fibers.op" +local sleep = require "fibers.sleep" + +-- This is a dummy service +local dummy_service = {} +dummy_service.__index = dummy_service +dummy_service.name = 'dummy_service' + +-- The start function should be non-blocking +function dummy_service:start(bus_conn, ctx) + service.spawn_fiber('log-fiber', bus_conn, ctx, function (fib_ctx) + while not fib_ctx:err() do + print('fiber active') + op.choice( + sleep.sleep_op(1), + fib_ctx:done_op() + ):perform() + end + end) +end + +local ctx = context.with_cancel(context.background) +local bus = bus_mod.new({q_len=10, m_wild='#', s_wild='+', sep="/"}) + +service.spawn(dummy_service, bus, ctx) + +sleep.sleep(5) +ctx:cancel('shutdown') +``` \ No newline at end of file diff --git a/src/bus.lua b/src/bus.lua deleted file mode 100644 index 6dda4cad..00000000 --- a/src/bus.lua +++ /dev/null @@ -1,171 +0,0 @@ -local queue = require 'fibers.queue' -local op = require 'fibers.op' -local sleep = require 'fibers.sleep' -local trie = require 'trie' -local uuid = require 'uuid' - -local DEFAULT_Q_LEN = 10 - -local CREDS = { - ['user'] = 'pass', - ['user1'] = 'pass1', - ['user2'] = 'pass2', -} - -local Bus = {} -Bus.__index = Bus - -local function new(params) - params = params or {} - return setmetatable({ - q_length = params.q_length or DEFAULT_Q_LEN, - topics = trie.new(params.s_wild, params.m_wild, params.sep), --sets single_wild, multi_wild, separator - retained_messages = trie.new(params.s_wild, params.m_wild, params.sep) - }, Bus) -end - -local Subscription = {} -Subscription.__index = Subscription - -function Subscription.new(conn, topic, q) - return setmetatable({ - connection = conn, - topic = topic, - q = q - }, Subscription) -end - -function Subscription:next_msg_op(timeout) - local msg_op = op.choice( - self.q:get_op(), - timeout and sleep.sleep_op(timeout):wrap(function () return nil, "Timeout" end) or nil - ) - return msg_op -end - -function Subscription:next_msg(timeout) - return self:next_msg_op(timeout):perform() -end - -function Subscription:unsubscribe() - self.connection:unsubscribe(self.topic, self) -end - -local Connection = {} -Connection.__index = Connection - -function Connection.new(bus) - return setmetatable({bus = bus, subscriptions = {}}, Connection) -end - -function Connection:publish(message) - self.bus:publish(message) - return true -end - -function Connection:subscribe(topic) - local subscription, err = self.bus:subscribe(self, topic) - if err then return nil, err end - table.insert(self.subscriptions, subscription) - return subscription, nil -end - -function Connection:unsubscribe(topic, subscription) - self.bus:unsubscribe(topic, subscription) - - for i, sub in ipairs(self.subscriptions) do -- slow O(n) - if sub == subscription then - table.remove(self.subscriptions, i) - return - end - end -end - -function Connection:disconnect() - for _, subscription in ipairs(self.subscriptions) do - self:unsubscribe(subscription.topic, subscription) - end - self.subscriptions = {} -end - -function Connection:request(msg) - msg.reply_to = uuid.new() - local sub = self:subscribe(msg.reply_to) - self:publish(msg) - return sub -end - -function Bus:connect(creds) - -- if not Bus:authenticate(creds) then - -- return nil, 'Authentication failed' - -- end - return Connection.new(self) -end - --- Bus:subscribe function -function Bus:subscribe(connection, topic) - -- get topic from the trie, or make and add to the trie - local topic_entry, err = self.topics:retrieve(topic) - if err ~= nil then return nil, err end - if not topic_entry then - topic_entry = {subs = {}} - self.topics:insert(topic, topic_entry) - end - - -- create the subscription - we have no identity yet, UUID? - local q = queue.new(self.q_length) - local subscription = Subscription.new(connection, topic, q) - table.insert(topic_entry.subs, subscription) - - -- send any relevant retained messages - for _, v in ipairs(self.retained_messages:match(topic)) do -- wildcard search in trie - local put_operation = subscription.q:put_op(v.value) - put_operation:perform_alt(function () - -- print 'QUEUE FULL, not sent' --need to log blocked queue properly - end) - end - - return subscription -end - --- Bus:publish function -function Bus:publish(message) - local matches = self.topics:match(message.topic) - for _, topic_entry in ipairs(matches) do - for _, sub in ipairs(topic_entry.value.subs) do - local put_operation = sub.q:put_op(message) - put_operation:perform_alt(function () - -- TODO: log this properly - end) - end - -- add logic here for nats style q_subs if we go this route - end - - if message.retained then - if not message.payload then -- send msg with empty payload + ret flag to clear ret message - self.retained_messages:delete(message.topic) - else - self.retained_messages:insert(message.topic, message) - end - end -end - --- Bus:unsubscribe function -function Bus:unsubscribe(topic, subscription) - local topic_entry = self.topics:retrieve(topic) - assert(topic_entry, "error: unsubscribing from a non-existent topic") - - for i, sub in ipairs(topic_entry.subs) do -- slow O(n) - if sub == subscription then - table.remove(topic_entry.subs, i) - end - end - - if #topic_entry.subs == 0 then - self.topics:delete(topic) - end -end - -return { - new = new -} diff --git a/src/fibers/alarm.lua b/src/fibers/alarm.lua deleted file mode 100644 index 7445c360..00000000 --- a/src/fibers/alarm.lua +++ /dev/null @@ -1,278 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - --- Alarms. - -local op = require 'fibers.op' -local fiber = require 'fibers.fiber' -local timer = require 'fibers.timer' -local sc = require 'fibers.utils.syscall' - -local function days_in_year(y) - return y % 4 == 0 and (y % 100 ~= 0 or y % 400 == 0) and 366 or 365 -end - -local function to_time(t) - local new_t = {year = t.year, month=t.month, day=t.day, hour=t.hour, min=t.min, sec=t.sec} - local time = os.time(new_t) + t.msec/1e3 - local time_t = os.date("*t", time) - time_t.msec = t.msec - return time, time_t -end - --- let's define some constants -local periods = {"year", "month", "day", "hour", "min", "sec", "msec"} -local default = {month=1, day=1, hour=0, min=0, sec=0, msec=0} - --- This function validates a table t intended for scheduling an alarm, ensuring --- only appropriate fields are specified based on the scheduling type. -local function validate_next_table(t) - local inc_field - if t.year then - return nil, "year should not be specified for a relative alarm" - elseif t.yday then inc_field = "year" - if t.month or t.wday or t.day then - return nil, "neither month, weekday or day of month valid for day of year alarm" - end - elseif t.month then inc_field = "year" - if t.wday then - return nil, "day of week not valid for yearly alarm" - end - elseif t.day then inc_field = "month" - if t.wday then - return nil, "day of week not valid for monthly alarm" - end - elseif t.wday then inc_field = "day" - elseif t.hour then inc_field = "day" - elseif t.min then inc_field = "hour" - elseif t.sec then inc_field = "min" - elseif t.msec then inc_field = "sec" - else - return nil, "a next alarm must specify at least one of yday, month, day, wday, hour, minute, sec or msec" - end - - return inc_field, nil -end - --- calculates the absolute time until the next occurrence based on a given time --- structure t and the current epoch. -local function calculate_next(t, epoch) - - -- first let's make sure that the provided struct makes sense - local inc_field, _ = validate_next_table(t) -- the time table is pre-validated - - -- let's construct the new date table - local new_t = {} - - local now = os.date("*t", epoch) - now.msec = (epoch - math.floor(epoch)) * 1e3 - - local default_switch = false - for _, name in ipairs(periods) do - if not default_switch and t[name] then default_switch = true end - if (t.wday or t.yday) and name=="hour" then default_switch = true end - new_t[name] = (not default_switch and now[name]) or t[name] or default[name] - end - - -- now let's get the struct we need - local new_time, new_table = to_time(new_t) - - -- wday and yday are weird ones and we need to renormalise - if t.wday then - local increment = (t.wday - new_table.wday + 7) % 7 - new_table.day = new_table.day + increment - new_time, new_table = to_time(new_table) - elseif t.yday then - local no_days = days_in_year(new_table.year) - local increment = (t.yday - new_table.yday + no_days) % no_days - new_table.day = new_table.day + increment - new_time, new_table = to_time(new_table) - end - - if new_time < epoch then - new_table[inc_field] = new_table[inc_field] + 1 - new_time, new_table = to_time(new_table) - end - - return new_time, new_table -end - - -local AlarmHandler = {} -AlarmHandler.__index = AlarmHandler - -local function new_alarm_handler() - local now = sc.realtime() - return setmetatable( - { - realtime = false, - abs_buffer = {}, - next_buffer = {}, - abs_timer = timer.new(now), -- Task list for absolute time scheduling - }, AlarmHandler) -end - -local installed_alarm_handler = nil - ---- Installs the Alarm Handler into the current scheduler. --- Must be called before any alarm operations are used. --- @return The installed AlarmHandler instance. -local function install_alarm_handler() - if not installed_alarm_handler then - installed_alarm_handler = new_alarm_handler() - fiber.current_scheduler:add_task_source(installed_alarm_handler) - end - return installed_alarm_handler -end - ---- Uninstalls the Alarm Handler from the current scheduler. --- This should be called to clean up when the Alarm Handler is no longer needed. -local function uninstall_alarm_handler() - if installed_alarm_handler then - for i, source in ipairs(fiber.current_scheduler.sources) do - if source == installed_alarm_handler then - table.remove(fiber.current_scheduler.sources, i) - break - end - end - installed_alarm_handler = nil - end -end - -function AlarmHandler:schedule_tasks(sched) - local now = sc.realtime() - - self.abs_timer:advance(now, sched) - - while true do - local next_time = self.abs_timer:next_entry_time() - now - if next_time > sched.maxsleep then break end -- an empty timer will return 'inf' here so nil check not needed - local task = self.abs_timer:pop() - sched:schedule_after_sleep(next_time, task.obj) - end -end - -function AlarmHandler:block(time_to_start, t, task) - if time_to_start < fiber.current_scheduler.maxsleep then - fiber.current_scheduler:schedule_after_sleep(time_to_start, task) - else - self.abs_timer:add_absolute(t, task) - end -end - -function AlarmHandler:clock_synced() - self.realtime = true - local now = sc.realtime() - -- Process buffered absolute tasks - for _, buffered in ipairs(self.abs_buffer) do - local time_to_start = buffered.t - now - self:block(time_to_start, buffered.t, buffered.task) - end - -- Process next tasks - for _, buffered in ipairs(self.next_buffer) do - local next_time = calculate_next(buffered.t, now) - local time_to_start = next_time - now - self:block(time_to_start, next_time, buffered.task) - end - self.abs_buffer, self.next_buffer = {}, {} -- Clear the buffer -end - -function AlarmHandler:clock_desynced() - self.realtime = false -end - -function AlarmHandler:wait_absolute_op(t) - local time_to_start - local function try() - if not self.realtime then return false end - time_to_start = t - sc.realtime() - if time_to_start < 0 then return true end - end - local function block(suspension, wrap_fn) - local task = suspension:complete_task(wrap_fn) - if not self.realtime then table.insert(self.abs_buffer, {t=t, task=task}) - return - end - self:block(time_to_start, t, task) - end - return op.new_base_op(nil, try, block) -end - -function AlarmHandler:wait_next_op(t) - local function try() - return false - end - local function block(suspension, wrap_fn) - local task = suspension:complete_task(wrap_fn) - if not self.realtime then table.insert(self.next_buffer, {t=t, task=task}) - return - end - local now = sc.realtime() - local target, _ = calculate_next(t, now) - self:block(target-now, target, task) - end - return op.new_base_op(nil, try, block) -end - ---- Indicates to the Alarm Handler that time synchronisation has been achieved (through NTP or other methods). --- Until the user calls clock_synced() all alarms will block. When called, --- `absolute` alarms will return immediately if their time has elapsed, whereas --- `next` alarms will be scheduled for their next instance -local function clock_synced() - return assert(installed_alarm_handler):clock_synced() -end - ---- Indicates to the Alarm Handler that time synchronisation has been lost. --- All new alarms will be buffered until real-time is achieved. -local function clock_desynced() - return assert(installed_alarm_handler):clock_desynced() -end - ---- Creates an operation for an absolute alarm. --- The operation can be performed immediately if in real-time mode, --- or buffered to be scheduled upon achieving real-time. --- @param t The absolute time (epoch) for the alarm. --- @return A BaseOp representing the absolute alarm operation. -local function wait_absolute_op(t) - return assert(installed_alarm_handler):wait_absolute_op(t) -end - ---- Schedules a task to run at an absolute time. --- Wrapper for `absolute_op` that immediately performs the operation. --- @param t The absolute time (epoch) for the alarm. -local function wait_absolute(t) - return wait_absolute_op(t):perform() -end - ---- Creates an operation for a next (relative) alarm. --- The operation is always buffered until real-time is achieved, --- then scheduled based on the calculated next time. --- @param t A table specifying the relative time for the alarm. --- @return A BaseOp representing the next alarm operation. --- @return An error if the time table is invalid. -local function wait_next_op(t) - local _, err = validate_next_table(t) - return err or assert(installed_alarm_handler):wait_next_op(t) -end - ---- Schedules a task based on a relative next time. --- Wrapper for `next_op` that immediately performs the operation. --- @param t A table specifying the relative time for the alarm. --- @return An error if the time table is invalid. -local function wait_next(t) - local _, err = validate_next_table(t) - return err or assert(installed_alarm_handler):wait_next_op(t):perform() -end - --- Public API -return { - install_alarm_handler = install_alarm_handler, - uninstall_alarm_handler = uninstall_alarm_handler, - clock_synced = clock_synced, - clock_desynced = clock_desynced, - wait_absolute_op = wait_absolute_op, - wait_absolute = wait_absolute, - wait_next_op = wait_next_op, - wait_next = wait_next, - validate_next_table = validate_next_table, - calculate_next = calculate_next -} \ No newline at end of file diff --git a/src/fibers/channel.lua b/src/fibers/channel.lua deleted file mode 100644 index c67cb88e..00000000 --- a/src/fibers/channel.lua +++ /dev/null @@ -1,124 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - ---- fibers.channel module --- Provides Concurrent ML style channels for communication between fibers. --- @module fibers.channel - -local op = require 'fibers.op' - -local Fifo = {} -Fifo.__index = Fifo -local function new_fifo() return setmetatable({}, Fifo) end -function Fifo:push(x) table.insert(self, x) end - -function Fifo:empty() return #self == 0 end - -function Fifo:peek() - assert(not self:empty()); return self[1] -end - -function Fifo:pop() - assert(not self:empty()); return table.remove(self, 1) -end - ---- Channel class --- Represents a communication channel between fibers. --- @type Channel -local Channel = {} - ---- Create a new Channel. --- @treturn Channel The created Channel. -local function new() - return setmetatable( - { getq = new_fifo(), putq = new_fifo() }, - { __index = Channel }) -end - ---- Create a put operation for the Channel. --- Make an operation that if and when it completes will rendezvous with --- a receiver fiber to send VAL over the channel. --- @param val The value to put into the Channel. --- @treturn BaseOp The created put operation. -function Channel:put_op(val) - local getq, putq = self.getq, self.putq - local function try() - while not getq:empty() do - local remote = getq:pop() - if remote.suspension:waiting() then - remote.suspension:complete(remote.wrap, val) - return true - end - -- Otherwise the remote suspension is already completed, in - -- which case we did the right thing to pop off the dead - -- suspension from the getq. - end - return false - end - local function block(suspension, wrap_fn) - -- First, a bit of GC. - while not putq:empty() and not putq:peek().suspension:waiting() do - putq:pop() - end - -- We have suspended the current fiber; arrange for the fiber - -- to be resumed by a get operation by adding it to the channel's - -- putq. - putq:push({ suspension = suspension, wrap = wrap_fn, val = val }) - end - return op.new_base_op(nil, try, block) -end - ---- Create a get operation for the Channel. --- Make an operation that if and when it completes will rendezvous with --- a sender fiber to receive one value from the channel. --- @treturn BaseOp The created get operation. -function Channel:get_op() - local getq, putq = self.getq, self.putq - local function try() - while not putq:empty() do - local remote = putq:pop() - if remote.suspension:waiting() then - remote.suspension:complete(remote.wrap) - return true, remote.val - end - -- Otherwise the remote suspension is already completed, in - -- which case we did the right thing to pop off the dead - -- suspension from the putq. - end - return false - end - local function block(suspension, wrap_fn) - -- First, a bit of GC. - while not getq:empty() and not getq:peek().suspension:waiting() do - getq:pop() - end - -- We have suspended the current fiber; arrange for the fiber to - -- be resumed by a put operation by adding it to the channel's - -- getq. - getq:push({ suspension = suspension, wrap = wrap_fn }) - end - return op.new_base_op(nil, try, block) -end - ---- Put a message into the Channel. --- Send message on the channel. If there is already another fiber --- waiting to receive a message on this channel, give it our message and --- continue. Otherwise, block until a receiver becomes available. --- @tparam any message The message to put into the Channel. -function Channel:put(message) - self:put_op(message):perform() -end - ---- Get a message from the Channel. --- Receive a message from the channel and return it. If there is --- already another fiber waiting to send a message on this channel, take --- its message directly. Otherwise, block until a sender becomes --- available. --- @treturn any The message retrieved from the Channel. -function Channel:get() - return self:get_op():perform() -end - ---- @export -return { - new = new -} diff --git a/src/fibers/cond.lua b/src/fibers/cond.lua deleted file mode 100644 index 06d4f7fa..00000000 --- a/src/fibers/cond.lua +++ /dev/null @@ -1,59 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - ---- fibers.cond module. --- This module implements a condition variable, a rendezvous point for --- fibers waiting for or announcing the occurrence of an event. --- @module fibers.cond - -local op = require 'fibers.op' - ---- Cond class. --- Represents a condition variable. --- @type Cond -local Cond = {} - ---- Create a new condition variable. --- @treturn Cond The new condition variable. -local function new() - return setmetatable({ waitq = {} }, { __index = Cond }) -end - ---- Create a new operation that will put the fiber into a wait state on the condition variable. --- @treturn operation The created operation. -function Cond:wait_op() - local function try() return not self.waitq end - local function gc() - local i = 1 - while i <= #self.waitq do - if self.waitq[i].suspension:waiting() then - i = i + 1 - else - table.remove(self.waitq, i) - end - end - end - local function block(suspension, wrap_fn) - gc() - table.insert(self.waitq, { suspension = suspension, wrap = wrap_fn }) - end - return op.new_base_op(nil, try, block) -end - ---- Put the fiber into a wait state on the condition variable. -function Cond:wait() return self:wait_op():perform() end - ---- Wake up all fibers that are waiting on this condition variable. -function Cond:signal() - if self.waitq ~= nil then - for _, remote in ipairs(self.waitq) do - if remote.suspension:waiting() then - remote.suspension:complete(remote.wrap) - end - end - self.waitq = nil - end -end - -return { - new = new -} diff --git a/src/fibers/context.lua b/src/fibers/context.lua deleted file mode 100644 index 1b1542d2..00000000 --- a/src/fibers/context.lua +++ /dev/null @@ -1,123 +0,0 @@ ---- A Lua context library for managing hierarchies of fibers with cancellation, deadlines, and values. --- This library provides a way to create and manage context objects, similar to the context package in Go. --- Each context can carry a set of key-value pairs (values), a cancellation signal, and a deadline. --- Children contexts can be derived from a parent context, inheriting and extending its values. --- @module context - -local fiber = require 'fibers.fiber' -local waitgroup = require 'fibers.waitgroup' -local op = require 'fibers.op' -local sleep = require 'fibers.sleep' - ---- Context class. --- Represents a context in the fiber system. --- @type Context -local Context = {} -Context.__index = Context - ---- Creates a new background context. --- This is the root context for all others; it is never canceled, has no deadline, and carries no values. --- @return Context A new background context. -local function background() - return setmetatable({ - values = {}, - children = {} - }, Context) -end - ---- Creates a new context with cancellation, derived from a parent context. --- @param Context parent The parent context from which to derive the new context. --- @return Context The new context. --- @return function Cancellation function. -local function with_cancel(parent) - local wg = waitgroup.new() - wg:add(1) - local ctx = setmetatable({ - wg = wg, - children = {}, - -- Creates a new table that inherits from 'parent.values', enabling access to parent's values + overriding. - values = setmetatable({}, {__index = parent.values}) - }, Context) - - function ctx:cancel(cause) - if self.cause then return end - if not cause then cause = "canceled" end - self.cause = cause - wg:done() - for _, child in ipairs(self.children) do - if child.cancel then child:cancel(cause) end - end - end - - table.insert(parent.children, ctx) - - return ctx, function(cause) ctx:cancel(cause) end -end - ---- Creates a new context with a deadline. --- The context will be canceled automatically when the deadline is exceeded. --- @param Context parent The parent context. --- @param number deadline The time at which to cancel the context. --- @return Context The new context. --- @return function Cancellation function. -local function with_deadline(parent, deadline) - local ctx, cancel = with_cancel(parent) - fiber.spawn(function() - sleep.sleep_until(deadline) - ctx:cancel("deadline_exceeded") - end) - return ctx, cancel -end - ---- Creates a new context with a timeout. --- The context will be canceled automatically after the timeout duration. --- @param parent The parent context. --- @param timeout The duration in seconds after which to cancel the context. --- @return Context The new context. --- @return function Cancellation function. -local function with_timeout(parent, timeout) - return with_deadline(parent, fiber.now() + timeout) -end - ---- Creates a new context with an additional key-value pair. --- @param Context parent The parent context. --- @param string key The key for the value to add. --- @param any value The value to add. --- @return Context The new context. -local function with_value(parent, key, value) - local ctx = setmetatable({ - children = {}, - values = setmetatable({[key] = value}, {__index = parent.values}) - }, Context) - return ctx -end - ---- Returns an operation that can be used in `op.choice` to wait for the context to be done. --- @return function An operation that can be used with `op.choice`. -function Context:done_op() - if not self.wg then - return op.new_base_op(nil, function() return true end, nil) - end - return self.wg:wait_op() -end - ---- Accesses a value stored in the context. --- @param string key The key for the value to retrieve. --- @return any The value associated with the given key, or nil if not found. -function Context:value(key) - return self.values[key] -end - ---- Returns the cause of the cancellation, if any. --- @return string|nil A string describing the cause of cancellation, or nil if not canceled. -function Context:err() - return self.cause -end - -return { - background = background, - with_cancel = with_cancel, - with_deadline = with_deadline, - with_timeout = with_timeout, - with_value = with_value -} diff --git a/src/fibers/epoll.lua b/src/fibers/epoll.lua deleted file mode 100644 index 4cbaf3a1..00000000 --- a/src/fibers/epoll.lua +++ /dev/null @@ -1,75 +0,0 @@ --- (c) Snabb project --- (c) Jangala - --- Use of this source code is governed by the XXXXXXXXX license; see COPYING. - --- Epoll. - -local sc = require 'fibers.utils.syscall' -local bit = rawget(_G, "bit") or require 'bit32' - -local Epoll = {} - -local INITIAL_MAXEVENTS = 8 - -local function new() - local ret = { - epfd = assert(sc.epoll_create()), - active_events = {}, - maxevents = INITIAL_MAXEVENTS, - } - return setmetatable(ret, { __index = Epoll }) -end - -local RD = sc.EPOLLIN + sc.EPOLLRDHUP -local WR = sc.EPOLLOUT -local RDWR = RD + WR -local ERR = sc.EPOLLERR + sc.EPOLLHUP -local PRI = sc.EPOLLPRI - -function Epoll:add(s, events) - -- local fd = type(s) == 'number' and s or sc.fileno(s) - local fd = s - local active = self.active_events[fd] or 0 - local eventmask = bit.bor(events, active, sc.EPOLLONESHOT) - local ok, _ = sc.epoll_modify(self.epfd, fd, eventmask) - if not ok then assert(sc.epoll_register(self.epfd, fd, eventmask)) end -end - -function Epoll:poll(timeout) - -- Returns a table, an iterator would be more efficient. - -- print("self.epfd", self.epfd) - -- print("self.maxevents", self.maxevents) - local events, err = sc.epoll_wait(self.epfd, timeout or 0, self.maxevents) - if not events then - error(err) - end - local count = 0 - -- Since we add fd's with EPOLL_ONESHOT, now that the event has - -- fired, the fd is now deactivated. Record that fact. - for fd, _ in pairs(events) do - count = count + 1 - self.active_events[fd] = nil - end - if count == self.maxevents then - -- If we received `maxevents' events, it means that probably there - -- are more active fd's in the queue that we were unable to - -- receive. Expand our event buffer in that case. - self.maxevents = self.maxevents * 2 - end - return events, err -end - -function Epoll:close() - sc.epoll_close(self.epfd) - self.epfd = nil -end - -return { - new = new, - RD = RD, - WR = WR, - RDWR = RDWR, - ERR = ERR, - PRI = PRI, -} diff --git a/src/fibers/exec.lua b/src/fibers/exec.lua deleted file mode 100644 index f9bb5630..00000000 --- a/src/fibers/exec.lua +++ /dev/null @@ -1,286 +0,0 @@ --- fibers/exec.lua --- Provides facilities for executing external commands asynchronously using fibers. -local file = require 'fibers.stream.file' -local pollio = require 'fibers.pollio' -local fiber = require 'fibers.fiber' -local op = require 'fibers.op' -local waitgroup = require 'fibers.waitgroup' -local string_buffer = require 'fibers.utils.string_buffer' -local sc = require 'fibers.utils.syscall' - -local io_names = {"stdin", "stdout", "stderr"} - -local function close_all(...) - local n = select('#', ...) - for i = 1, n do - local list = select(i, ...) - for k, v in pairs(list) do - if type(v) == "number" then - sc.close(v) - elseif v.close then - v:close() - end - list[k] = nil - end - end -end - --- Define the command type -local Cmd = {} -Cmd.__index = Cmd -- set metatable - ---- Constructor for Cmd. --- @param name The name or path of the command to execute. --- @param ... Additional arguments for the command. --- @return A new Cmd instance. -local function command(name, ...) - local self = setmetatable({}, Cmd) - self.path = name - self.args = {...} - self.process = {} - self.pipes = { - child = {}, - child_cmd = {}, - ext_cmd = {}, - ext = {}, - wg = waitgroup.new(), - } - return self -end - ---- Constructor for Cmd taking a `context`. --- @param ctx The context to run the command under. --- @param name The name or path of the command to execute. --- @param ... Additional arguments for the command. --- @return A new Cmd instance. -local function command_context(ctx, name, ...) - local cmd = command(name, ...) - cmd.ctx = ctx - return cmd -end - ---- Sets the command to launch with a different pgid. --- @param status True if diff pgid desired. -function Cmd:setpgid(status) - self._setpgid = status -end - ---- Launches the command. --- @param path The path to the executable. --- @param argt Table of arguments. --- @return pid, cmd_streams, result_channel, error -function Cmd:launch(path, argt) - -- Check if the executable exists and is executable - local status, errstr, _ = sc.access(path, "x") - if not status then return errstr end - - self.pipes.child.stdin, self.pipes.child_cmd.stdin = assert(sc.pipe()) - self.pipes.child_cmd.stdout, self.pipes.child.stdout = assert(sc.pipe()) - self.pipes.child_cmd.stderr, self.pipes.child.stderr = assert(sc.pipe()) - - local ready_read, ready_write = assert(sc.pipe()) - ready_read = file.fdopen(ready_read) - - self.process.pid = assert(sc.fork()) - if self.process.pid == 0 then -- child - if self._setpgid then - local result, err_msg = sc.setpid('p', 0, 0) - assert(result==0, err_msg) - end - -- close all the parent pipes - close_all(self.pipes.ext, self.pipes.ext_cmd, self.pipes.child_cmd) - ready_read:close() - - sc.dup2(self.pipes.child.stdin, sc.STDIN_FILENO) - sc.dup2(self.pipes.child.stdout, sc.STDOUT_FILENO) - sc.dup2(self.pipes.child.stderr, sc.STDERR_FILENO) - - close_all(self.pipes.child) - - sc.close(ready_write) - - local _, err, errno = sc.exec(path, argt) -- will not return unless unsuccessful - if err then - io.stderr:write(err .. "\n") -- will be sent over the stderr pipe - sc.exit(errno) -- exit with non-zero status - end - end - -- parent - close_all(self.pipes.child) - - sc.close(ready_write) - ready_read:read_some_chars() - ready_read:close() - - local pidfd, err = sc.pidfd_open(self.process.pid, 0) - self.process.pidfd = assert(pidfd, err) - - for k, v in pairs(self.pipes.child_cmd) do - self.pipes.child_cmd[k] = file.fdopen(v) - end - - self.pipes.child_cmd.stdin:setvbuf('no') - - return nil -end - ---- Gets the combined output of stdout and stderr. --- @return The combined output and any error. -function Cmd:combined_output() - local buf = string_buffer.new() - self.pipes.ext_cmd.stdout, self.pipes.ext_cmd.stderr = buf, buf - local err = self:run() - if err then return buf:read(), err end - return buf:read(), nil -end - ---- Gets the output of stdout. --- @return The output and any error. -function Cmd:output() - local buf = string_buffer.new() - self.pipes.ext_cmd.stdout = buf - local err = self:run() - if err then return buf:read(), err end - return buf:read(), nil -end - ---- Starts the command and waits for it to complete. --- @return Any error. -function Cmd:run() - local err = self:start() - if err then - close_all(self.pipes.ext_cmd, self.pipes.child_cmd) - return err - end - return self:wait() -end - ---- Starts the command. --- @return Any error. -function Cmd:start() - if self.process.pid then return "process already started" end - - if self.ctx and self.ctx:err() then - close_all(self.pipes.ext_cmd) - return "context already complete" - end - - local executable_path - -- Check if a path is provided - if self.path:find("/") then - executable_path = self.path - else - -- Search for the executable in the PATH - for dir in os.getenv("PATH"):gmatch("[^:]+") do - local full_path = dir .. "/" .. self.path - if sc.access(full_path, "x") then - executable_path = full_path - break - end - end - if not executable_path then - close_all(self.pipes.ext_cmd) - return '"'..self.path..'": executable file not found in $PATH' - end - end - - local err = self:launch(executable_path, self.args) - if err then - close_all(self.pipes.ext_cmd, self.pipes.child_cmd, self.pipes.child) - return err - end - - for _, v in ipairs(io_names) do - self.pipes.wg:add(1) - if not self.pipes.ext_cmd[v] then - self.pipes.ext_cmd[v] = assert(file.open("/dev/null", v == "stdin" and "r" or "w")) - end - local input = v=="stdin" and self.pipes.ext_cmd[v] or self.pipes.child_cmd[v] - local output = v=="stdin" and self.pipes.child_cmd[v] or self.pipes.ext_cmd[v] - fiber.spawn(function() - while true do - local received = input:read_some_chars() - output:write(received) - if not received then break end - end - if input.close then input:close() end - if output.close then output:close() end - self.pipes.ext_cmd[v], self.pipes.child_cmd[v] = nil, nil - self.pipes.wg:done() - end) - end - - -- Setup a new fiber to listen for context cancellation or command completion - if self.ctx then - fiber.spawn(function() - op.choice( - self.ctx:done_op():wrap(function () -- Context was cancelled, kill the command - self:kill() - end), - pollio.fd_readable_op(self.process.pidfd) -- Command has completed, we just may not have waited yet - ):perform() - end) - end -end - ---- Kills the command. --- @return Any error. -function Cmd:kill() - if not self.process.pid then return "process not started" end - if self.process.state then return "process has already completed" end - - local res, err, errno = sc.kill(self._setpgid and -self.process.pid or self.process.pid, sc.SIGKILL) - assert(res==0 or errno==sc.ESRCH, err) -end - ---- Sets up a pipe for the given IO type. --- @param io_type The type of IO ("stdin", "stdout", or "stderr"). --- @return The pipe or an error. -local function setup_pipe(self, io_type) - if self.pipes.ext_cmd[io_type] then return nil, io_type .. " pipe already created" end - if self.process.pid then return nil, io_type .. "_pipe after process started" end - - local rd, wr = file.pipe() - wr = wr:setvbuf('no') - - local cmd_end, ext_end = wr, rd - - if io_type == "stdin" then - cmd_end, ext_end = rd, wr - end - - self.pipes.ext_cmd[io_type] = cmd_end - self.pipes.ext[io_type] = ext_end - - return ext_end, nil -end - ---- Creates a pipe for stdout. Call `:close()` when finished. --- @return The stdout pipe or an error. -function Cmd:stdout_pipe() return setup_pipe(self, "stdout") end - ---- Creates a pipe for stderr. Call `:close()` when finished. --- @return The stderr pipe or an error. -function Cmd:stderr_pipe() return setup_pipe(self, "stderr") end - ---- Creates a pipe for stdin. Call `:close()` when finished. --- @return The stdin pipe or an error. -function Cmd:stdin_pipe() return setup_pipe(self, "stdin") end - - ---- Waits for the command to complete. --- @return The completion status or an error. -function Cmd:wait() - pollio.fd_readable_op(self.process.pidfd):perform() - self.pipes.wg:wait() - local _, _, status = sc.waitpid(self.process.pid) - self.process.state = status - sc.close(self.process.pidfd) - close_all(self.pipes.child, self.pipes.child_cmd, self.pipes.ext_cmd) - if status ~= 0 then return status end -end - -return { - command = command, - command_context = command_context -} diff --git a/src/fibers/fiber.lua b/src/fibers/fiber.lua deleted file mode 100644 index a105b77f..00000000 --- a/src/fibers/fiber.lua +++ /dev/null @@ -1,157 +0,0 @@ --- (c) Snabb project --- (c) Jangala - --- Use of this source code is governed by the XXXXXXXXX license; see COPYING. - ---- Fiber module. --- Implements a fiber system using Lua's coroutines for cooperative multitasking. --- @module fibers.fiber - --- Required packages -local sched = require 'fibers.sched' - -local current_fiber -local current_scheduler = sched.new() - ---- The Fiber class --- Represents a single fiber, or lightweight thread. --- @type Fiber -local Fiber = {} -Fiber.__index = Fiber - ---- Spawns a new fiber. --- @function spawn --- @tparam function fn The function to run in the new fiber. -local function spawn(fn) - -- Capture the traceback - local tb = debug.traceback("", 2):match("\n[^\n]*\n(.*)") or "" - -- If we're inside another fiber, append the traceback to the parent's traceback - if current_fiber and current_fiber.traceback then - tb = tb .. "\n" .. current_fiber.traceback - end - - current_scheduler:schedule( - setmetatable({ - coroutine = coroutine.create(fn), - alive = true, - sockets = {}, - traceback = tb - }, Fiber)) -end - ---- Resumes execution of the fiber. --- If the fiber is already dead, this will throw an error. --- @tparam vararg ... The arguments to pass to the fiber. -function Fiber:resume(...) - assert(self.alive, "dead fiber") -- checks that the fiber is alive - local saved_current_fiber = current_fiber -- shift the old current fiber into a safe place - current_fiber = self -- we are the new current fiber - local ok, err = coroutine.resume(self.coroutine, ...) -- rev up our coroutine - -- current_fiber = saved_current_fiber the KEY bit, we only get here when the coroutine above has yielded, - -- but we then pop back in the fiber we previously displaced - current_fiber = saved_current_fiber - if not ok then - print('Error while running fiber: ' .. tostring(err)) - print(debug.traceback(self.coroutine)) - print('fibers history:\n' .. self.traceback) - os.exit(255) - end -end - -Fiber.run = Fiber.resume - ---- Suspends execution of the fiber. --- The fiber will be resumed when the provided blocking function finishes. --- @tparam function block_fn The function to block on. --- @tparam vararg ... The arguments to pass to the blocking function. -function Fiber:suspend(block_fn, ...) - assert(current_fiber == self) - -- The block_fn should arrange to reschedule the fiber when it - -- becomes runnable. - block_fn(current_scheduler, current_fiber, ...) - return coroutine.yield() -end - ---- Returns the socket associated with the provided descriptor. --- @tparam number sd The socket descriptor. --- @treturn table The socket. -function Fiber:get_socket(sd) - return assert(self.sockets[sd]) -end - ---- Adds a new socket to the fiber. --- @tparam table sock The socket to add. --- @treturn number The descriptor of the added socket. -function Fiber:add_socket(sock) - local sd = #self.sockets - -- FIXME: add refcount on socket - self.sockets[sd] = sock - return sd -end - ---- Closes the socket associated with the provided descriptor. --- @tparam number sd The socket descriptor. -function Fiber:close_socket(sd) - self:get_socket(sd) - self.sockets[sd] = nil - -- FIXME: remove refcount on socket -end - ---- Waits until the socket associated with the provided descriptor is readable. --- @tparam number sd The socket descriptor. -function Fiber:wait_for_readable(sd) - local s = self:get_socket(sd) - current_scheduler:resume_when_readable(s, self) - return coroutine.yield() -end - ---- Waits until the socket associated with the provided descriptor is writable. --- @tparam number sd The socket descriptor. -function Fiber:wait_for_writable(sd) - local s = self:get_socket(sd) - current_scheduler:schedule_when_writable(s, self) - return coroutine.yield() -end - ---- Returns the traceback of the fiber. --- @function get_traceback -function Fiber:get_traceback() - return self.traceback or "No traceback available" -end - ---- Returns the current time according to the current scheduler. --- @treturn number The current time. -local function now() return current_scheduler:now() end - ---- Suspends execution of the current fiber. --- The fiber will be resumed when the provided blocking function finishes. --- @function suspend --- @tparam function block_fn The function to block on. --- @tparam vararg ... The arguments to pass to the blocking function. -local function suspend(block_fn, ...) return current_fiber:suspend(block_fn, ...) end - -local function schedule(scheduler, fiber) scheduler:schedule(fiber) end - ---- Suspends execution of the current fiber. --- The fiber will be resumed when the scheduler is ready to run it again. --- @function yield -local function yield() return suspend(schedule) end - ---- Stops the current scheduler from running more tasks. --- @function stop -local function stop() current_scheduler:stop() end - ---- Runs the main event loop of the current scheduler. --- The scheduler will continue to run tasks and wait for events until stopped. --- @function main -local function main() return current_scheduler:main() end - -return { - current_scheduler = current_scheduler, - spawn = spawn, - now = now, - suspend = suspend, - yield = yield, - stop = stop, - main = main -} diff --git a/src/fibers/op.lua b/src/fibers/op.lua deleted file mode 100644 index 9ed21cd8..00000000 --- a/src/fibers/op.lua +++ /dev/null @@ -1,192 +0,0 @@ --- (c) Snabb project --- (c) Jangala - --- Use of this source code is governed by the XXXXXXXXX license; see COPYING. - ---- fibers.op module --- Provides Concurrent ML style operations for managing concurrency. --- @module fibers.op - -local fiber = require 'fibers.fiber' - -local Suspension = {} -Suspension.__index = Suspension - -local CompleteTask = {} -CompleteTask.__index = CompleteTask - -function Suspension:waiting() return self.state == 'waiting' end - -function Suspension:complete(wrap, val) - assert(self:waiting()) - self.state = 'synchronized' - self.wrap = wrap - self.val = val - self.sched:schedule(self) -end - -function Suspension:complete_and_run(wrap, val) - assert(self:waiting()) - self.state = 'synchronized' - return self.fiber:resume(wrap, val) -end - -function Suspension:complete_task(wrap, val) - return setmetatable({ suspension = self, wrap = wrap, val = val }, CompleteTask) -end - -function Suspension:run() - assert(not self:waiting()) - return self.fiber:resume(self.wrap, self.val) -end - -local function new_suspension(sched, fib) - return setmetatable( - { state = 'waiting', sched = sched, fiber = fib }, - Suspension) -end - ---- A complete task is a task that when run, completes a suspension, if ---- the suspension hasn't been completed already. There can be multiple ---- complete tasks for a given suspension, if the suspension can complete ---- in multiple ways (e.g. via a choice op). -function CompleteTask:run() - if self.suspension:waiting() then - -- Use complete-and-run so that the fiber runs in this turn. - self.suspension:complete_and_run(self.wrap, self.val) - end -end - ---- A complete task can also be cancelled, which makes it complete with a ---- call to "error". --- @param reason A string describing the reason for the cancellation -function CompleteTask:cancel(reason) - if self.suspension:waiting() then - self.suspension:complete(error, reason or 'cancelled') - end -end - ---- BaseOp class --- Represents a base operation. --- @type BaseOp -local BaseOp = {} -BaseOp.__index = BaseOp - ---- Create a new base operation. --- @tparam function wrap_fn The wrap function. --- @tparam function try_fn The try function. --- @tparam function block_fn The block function. --- @treturn BaseOp The created base operation. -local function new_base_op(wrap_fn, try_fn, block_fn) - if wrap_fn == nil then wrap_fn = function(val) return val end end - return setmetatable( - { wrap_fn = wrap_fn, try_fn = try_fn, block_fn = block_fn }, - BaseOp) -end - ---- ChoiceOp class --- Represents a choice operation. --- @type ChoiceOp -local ChoiceOp = {} -ChoiceOp.__index = ChoiceOp -local function new_choice_op(base_ops) - return setmetatable( - { base_ops = base_ops }, - ChoiceOp) -end - ---- Create a choice operation from the given operations. --- @tparam vararg ops The operations. --- @treturn ChoiceOp The created choice operation. -local function choice(...) - local ops = {} - -- Build a flattened list of choices that are all base ops. - for _, op in ipairs({ ... }) do - if op.base_ops then - for _, base_op in ipairs(op.base_ops) do table.insert(ops, base_op) end - else - table.insert(ops, op) - end - end - if #ops == 1 then return ops[1] end - return new_choice_op(ops) -end - ---- Wrap the base operation with the given function. --- @tparam function f The function. --- @treturn BaseOp The created base operation. -function BaseOp:wrap(f) - local wrap_fn, try_fn, block_fn = self.wrap_fn, self.try_fn, self.block_fn - return new_base_op(function(val) return f(wrap_fn(val)) end, try_fn, block_fn) -end - ---- Wrap the choice operation with the given function. --- @tparam function f The function. --- @treturn ChoiceOp The created choice operation. -function ChoiceOp:wrap(f) - local ops = {} - for _, op in ipairs(self.base_ops) do table.insert(ops, op:wrap(f)) end - return new_choice_op(ops) -end - -local function block_base_op(sched, fib, op) - op.block_fn(new_suspension(sched, fib), op.wrap_fn) -end - ---- Perform the base operation. --- @treturn vararg The value returned by the operation. -function BaseOp:perform() - local success, val = self.try_fn() - if success then return self.wrap_fn(val) end - local wrap, new_val = fiber.suspend(block_base_op, self) - return wrap(new_val) -end - -local function block_choice_op(sched, fib, ops) - local suspension = new_suspension(sched, fib) - for _, op in ipairs(ops) do op.block_fn(suspension, op.wrap_fn) end -end - ---- Perform the choice operation. --- @treturn vararg The value returned by the operation. -function ChoiceOp:perform() - local ops = self.base_ops - local base = math.random(#ops) - for i = 1, #ops do - local op = ops[((i + base) % #ops) + 1] - local success, val = op.try_fn() - if success then return op.wrap_fn(val) end - end - local wrap, val = fiber.suspend(block_choice_op, ops) - return wrap(val) -end - ---- Perform the base operation or return the result of the function if the operation cannot be performed. --- @tparam function f The function. --- @treturn vararg The value returned by the operation or the function. -function BaseOp:perform_alt(f) - fiber.yield() -- lessens race possibility - local success, val = self.try_fn() - if success then return self.wrap_fn(val) end - return f() -end - ---- Perform the choice operation or return the result of the function if the operation cannot be performed. --- @tparam function f The function. --- @treturn vararg The value returned by the operation or the function. -function ChoiceOp:perform_alt(f) - fiber.yield() -- lessens race possibility - local ops = self.base_ops - local base = math.random(#ops) - for i = 1, #ops do - local op = ops[((i + base) % #ops) + 1] - local success, val = op.try_fn() - if success then return op.wrap_fn(val) end - end - return f() -end - -return { - new_base_op = new_base_op, - choice = choice -} diff --git a/src/fibers/pollio.lua b/src/fibers/pollio.lua deleted file mode 100644 index b5b99510..00000000 --- a/src/fibers/pollio.lua +++ /dev/null @@ -1,220 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - --- File events. - -local op = require 'fibers.op' -local fiber = require 'fibers.fiber' -local epoll = require 'fibers.epoll' -local file = require 'fibers.stream.file' -local sc = require 'fibers.utils.syscall' -local bit = rawget(_G, "bit") or require 'bit32' - -local PollIOHandler = {} -PollIOHandler. __index = PollIOHandler - -local function new_poll_io_handler() - return setmetatable( - { - epoll = epoll.new(), - waiting_for_readable = {}, -- sock descriptor => array of task - waiting_for_writable = {}, -- sock descriptor => array of task - waiting_for_priority = {} -- sock descriptor => array of task - }, - PollIOHandler) -end - --- These three methods are "blocking handler" methods and are called by --- fibers.stream.file. -function PollIOHandler:init_nonblocking(fd) - sc.set_nonblock(fd) -end - -function PollIOHandler:wait_for_readable(fd) - self:fd_readable_op(fd):perform() -end - -function PollIOHandler:wait_for_writable(fd) - self:fd_writable_op(fd):perform() -end - -function PollIOHandler:wait_for_priority(fd) - self:fd_priority_op(fd):perform() -end - -local function add_waiter(fd, waiters, task) - local tasks = waiters[fd] - if tasks == nil then - tasks = {}; waiters[fd] = tasks - end - table.insert(tasks, task) -end - -local function make_block_fn(fd, waiting, poll, events) - return function(suspension, wrap_fn) - local task = suspension:complete_task(wrap_fn) - -- local fd = sc.fileno(fd) - add_waiter(fd, waiting, task) - poll:add(fd, events) - end -end - -function PollIOHandler:fd_readable_op(fd) - local function try() return false end - local block = make_block_fn( - fd, self.waiting_for_readable, self.epoll, epoll.RD) - return op.new_base_op(nil, try, block) -end - -function PollIOHandler:fd_writable_op(fd) - local function try() return false end - local block = make_block_fn( - fd, self.waiting_for_writable, self.epoll, epoll.WR) - return op.new_base_op(nil, try, block) -end - -function PollIOHandler:fd_priority_op(fd) - local function try() return false end - local block = make_block_fn( - fd, self.waiting_for_priority, self.epoll, epoll.PRI) - return op.new_base_op(nil, try, block) -end - -function PollIOHandler:stream_readable_op(stream) - local fd = assert(stream.io.fd) - local function try() return not stream.rx:is_empty() end - local block = make_block_fn( - fd, self.waiting_for_readable, self.epoll, epoll.RD) - return op.new_base_op(nil, try, block) -end - --- A stream_writable_op is the same as fd_writable_op, as a stream's --- buffer is never left full -- any stream method that fills the buffer --- flushes it directly. Knowing something about the buffer state --- doesn't tell us anything useful. -function PollIOHandler:stream_writable_op(stream) - local fd = assert(stream.io.fd) - return self:fd_writable_op(fd) -end - --- A stream_priority_op is the same as fd_priority_op. Knowing something --- about the buffer state doesn't tell us anything useful. -function PollIOHandler:stream_priority_op(stream) - local fd = assert(stream.io.fd) - return self:fd_priority_op(fd) -end - -local function schedule_tasks(sched, tasks) - -- It's possible for tasks to be nil, as an IO error will notify for - -- both readable and writable, and maybe we only have tasks waiting - -- for one side. - if tasks == nil then return end - for i = 1, #tasks do - sched:schedule(tasks[i]) - tasks[i] = nil - end -end - --- These method is called by the fibers scheduler. -function PollIOHandler:schedule_tasks(sched, _, timeout) - if timeout == nil then timeout = 0 end - if timeout >= 0 then timeout = timeout * 1e3 end - for fd, event in pairs(self.epoll:poll(timeout)) do - if bit.band(event, epoll.RD + epoll.ERR) ~= 0 then - local tasks = self.waiting_for_readable[fd] - schedule_tasks(sched, tasks) - end - if bit.band(event, epoll.WR + epoll.ERR) ~= 0 then - local tasks = self.waiting_for_writable[fd] - schedule_tasks(sched, tasks) - end - if bit.band(event, epoll.PRI + epoll.ERR) ~= 0 then - local tasks = self.waiting_for_priority[fd] - schedule_tasks(sched, tasks) - end - end -end - -PollIOHandler.wait_for_events = PollIOHandler.schedule_tasks - -function PollIOHandler:cancel_tasks_for_fd(fd) - local function cancel_tasks(waiting) - local tasks = waiting[fd] - if tasks ~= nil then - for i = 1, #tasks do tasks[i]:cancel() end - waiting[fd] = nil - end - end - cancel_tasks(self.waiting_for_readable) - cancel_tasks(self.waiting_for_writable) - cancel_tasks(self.waiting_for_priority) -end - -function PollIOHandler:cancel_all_tasks() - for fd, _ in pairs(self.waiting_for_readable) do - self:cancel_tasks_for_fd(fd) - end - for fd, _ in pairs(self.waiting_for_writable) do - self:cancel_tasks_for_fd(fd) - end - for fd, _ in pairs(self.waiting_for_priority) do - self:cancel_tasks_for_fd(fd) - end -end - -local installed = 0 -local installed_poll_handler -local function install_poll_io_handler() - installed = installed + 1 - if installed == 1 then - installed_poll_handler = new_poll_io_handler() - file.set_blocking_handler(installed_poll_handler) - fiber.current_scheduler:add_task_source(installed_poll_handler) - end - return installed_poll_handler -end - -local function uninstall_poll_io_handler() - installed = installed - 1 - if installed == 0 then - -- file.set_blocking_handler(nil) - -- FIXME: Remove task source. - for i, source in ipairs(fiber.current_scheduler.sources) do - if source == installed_poll_handler then - table.remove(fiber.current_scheduler.sources, i) - break - end - end - installed_poll_handler.poll:close() - installed_poll_handler = nil - end -end - -local function fd_readable_op(fd) - return assert(installed_poll_handler):fd_readable_op(fd) -end -local function fd_writable_op(fd) - return assert(installed_poll_handler):fd_writable_op(fd) -end -local function fd_priority_op(fd) - return assert(installed_poll_handler):fd_priority_op(fd) -end -local function stream_readable_op(stream) - return assert(installed_poll_handler):stream_readable_op(stream) -end -local function stream_writable_op(stream) - return assert(installed_poll_handler):stream_writable_op(stream) -end -local function stream_priority_op(stream) - return assert(installed_poll_handler):stream_priority_op(stream) -end - -return { - fd_readable_op = fd_readable_op, - fd_writable_op = fd_writable_op, - fd_priority_op = fd_priority_op, - stream_readable_op = stream_readable_op, - stream_writable_op = stream_writable_op, - stream_priority_op = stream_priority_op, - install_poll_io_handler = install_poll_io_handler, - uninstall_poll_io_handler = uninstall_poll_io_handler -} diff --git a/src/fibers/queue.lua b/src/fibers/queue.lua deleted file mode 100644 index 68570ec9..00000000 --- a/src/fibers/queue.lua +++ /dev/null @@ -1,66 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - ---- fibers.queue module --- Provides Concurrent ML style buffered channels for communication between fibers. --- @module fibers.queue - -local op = require 'fibers.op' -local channel = require 'fibers.channel' -local fiber = require 'fibers.fiber' - ---- Queue class --- Represents a queue for communication between fibers. --- @type Queue --- local Queue = {} - ---- Create a new Queue. --- @int[opt] bound The upper bound for the number of items in the queue. --- @treturn Queue The created Queue. -local function new(bound) - if bound then assert(bound >= 1) end - local ch_in, ch_out = channel.new(), channel.new() - local function service_queue() - local q = {} - while true do - if #q == 0 then - -- Empty. - table.insert(q, ch_in:get()) - elseif bound and #q >= bound then - -- Full. - ch_out:put(q[1]) - table.remove(q, 1) - else - local getop = ch_in:get_op() - local putop = ch_out:put_op(q[1]) - local val = op.choice(getop, putop):perform() - if val == nil then - -- Put operation succeeded. - table.remove(q, 1) - else - -- Get operation succeeded. - table.insert(q, val) - end - end - end - end - fiber.spawn(service_queue) - local ret = {} - function ret:put_op(x) - assert(x ~= nil) - return ch_in:put_op(x) - end - - function ret:get_op() - return ch_out:get_op() - end - - function ret:put(x) self:put_op(x):perform() end - - function ret:get() return self:get_op():perform() end - - return ret -end - -return { - new = new -} diff --git a/src/fibers/sched.lua b/src/fibers/sched.lua deleted file mode 100644 index 46065fa5..00000000 --- a/src/fibers/sched.lua +++ /dev/null @@ -1,146 +0,0 @@ --- (c) Snabb project --- (c) Jangala - --- Use of this source code is governed by the XXXXXXXXX license; see COPYING. - ---- Scheduler module. --- Implements the core scheduler for managing tasks. --- @module fibers.sched - --- Required modules -local sc = require 'fibers.utils.syscall' -local timer = require 'fibers.timer' - --- Constants -local MAX_SLEEP_TIME = 10 - -local Scheduler = {} -Scheduler.__index = Scheduler - ---- Creates a new Scheduler. --- @function new --- @return A new Scheduler. -local function new() - local ret = setmetatable( - { next = {}, cur = {}, sources = {}, wheel = timer.new(nil), maxsleep = MAX_SLEEP_TIME }, - Scheduler) - local timer_task_source = { wheel = ret.wheel } - -- private method for timer_tast_source - function timer_task_source:schedule_tasks(sched, now) - self.wheel:advance(now, sched) - end - - -- private method for timer_tast_source - function timer_task_source:cancel_all_tasks() - -- Implement me! - end - - ret:add_task_source(timer_task_source) - return ret -end - ---- Adds a task source to the scheduler. --- @param source The source to add. -function Scheduler:add_task_source(source) - table.insert(self.sources, source) - if source.wait_for_events then self.event_waiter = source end -end - ---- Schedules a task. --- @param task Task to be scheduled. -function Scheduler:schedule(task) - table.insert(self.next, task) -end - ---- Gets the current time from the timer wheel. --- @return Current time. -function Scheduler:now() - return self.wheel.now -end - ---- Schedules a task to be run at a specific time. --- @tparam number t The time to run the task. --- @tparam function task The task to run. -function Scheduler:schedule_at_time(t, task) - self.wheel:add_absolute(t, task) -end - ---- Schedules a task to be run after a certain delay. --- @tparam number dt The delay after which to run the task. --- @tparam function task The task to run. -function Scheduler:schedule_after_sleep(dt, task) - self.wheel:add_delta(dt, task) -end - ---- Schedules tasks from all sources to the scheduler. --- @tparam number now The current time. -function Scheduler:schedule_tasks_from_sources(now) - for i = 1, #self.sources do - self.sources[i]:schedule_tasks(self, now) - end -end - ---- Runs all scheduled tasks in the scheduler. --- If a specific time is provided, tasks scheduled for that time are run. --- @tparam number now (optional) The time to run tasks for. -function Scheduler:run(now) - if now == nil then now = self:now() end - self:schedule_tasks_from_sources(now) - self.cur, self.next = self.next, self.cur - for i = 1, #self.cur do - local task = self.cur[i] - self.cur[i] = nil - task:run() - end -end - ---- Returns the time of the next scheduled task. --- @treturn number The time of the next task. -function Scheduler:next_wake_time() - if #self.next > 0 then return self:now() end - return self.wheel:next_entry_time() -end - ---- Waits for the next scheduled event. -function Scheduler:wait_for_events() - local now, next_time = sc.monotime(), self:next_wake_time() - local timeout = math.min(self.maxsleep, next_time - now) - timeout = math.max(timeout, 0) - if self.event_waiter then - self.event_waiter:wait_for_events(self, now, timeout) - else - sc.floatsleep(timeout) - end -end - ---- Stops the main loop of the Scheduler. -function Scheduler:stop() - self.done = true -end - ---- Runs the main event loop of the scheduler. --- The scheduler will continue to run tasks and wait for events until stopped. -function Scheduler:main() - self.done = false - repeat - self:wait_for_events() - self:run(sc.monotime()) - until self.done -end - ---- Shuts down the scheduler. --- Cancels all tasks from all sources and runs remaining tasks. --- If there are still tasks after 100 attempts, returns false. --- @treturn boolean Whether the shutdown was successful. -function Scheduler:shutdown() - for _ = 1, 100 do - for i = 1, #self.sources do self.sources[i]:cancel_all_tasks(self) end - if #self.next == 0 then return true end - self:run() - end - return false -end - -return { - new = new -} diff --git a/src/fibers/sleep.lua b/src/fibers/sleep.lua deleted file mode 100644 index 808d9cbc..00000000 --- a/src/fibers/sleep.lua +++ /dev/null @@ -1,57 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - ---- fibers.sleep module. --- Provides functions to suspend execution of fibers for a certain duration (sleep) or until a specific time. --- @module fibers.sleep - -local op = require 'fibers.op' -local fiber = require 'fibers.fiber' - ---- Timeout class. --- Represents a timeout for a fiber. --- @type Timeout --- local Timeout = {} --- Timeout.__index = Timeout - ---- Create a new operation that puts the current fiber to sleep until the time t. --- @tparam number t The time to sleep until. --- @treturn operation The created operation. -local function sleep_until_op(t) - local function try() - return t <= fiber.now() - end - local function block(suspension, wrap_fn) - suspension.sched:schedule_at_time(t, suspension:complete_task(wrap_fn)) - end - return op.new_base_op(nil, try, block) -end - ---- Put the current fiber to sleep until time t. --- @tparam number t The time to sleep until. -local function sleep_until(t) - return sleep_until_op(t):perform() -end - ---- Create a new operation that puts the current fiber to sleep for a duration dt. --- @tparam number dt The duration to sleep. --- @treturn operation The created operation. -local function sleep_op(dt) - local function try() return dt <= 0 end - local function block(suspension, wrap_fn) - suspension.sched:schedule_after_sleep(dt, suspension:complete_task(wrap_fn)) - end - return op.new_base_op(nil, try, block) -end - ---- Put the current fiber to sleep for a duration dt. --- @tparam number dt The duration to sleep. -local function sleep(dt) - return sleep_op(dt):perform() -end - -return { - sleep = sleep, - sleep_op = sleep_op, - sleep_until = sleep_until, - sleep_until_op = sleep_until_op -} diff --git a/src/fibers/stream.lua b/src/fibers/stream.lua deleted file mode 100644 index d7253a4b..00000000 --- a/src/fibers/stream.lua +++ /dev/null @@ -1,456 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - - ---- fibers.stream module --- An API-compatible replacement and extension for Lua's stdio-based --- streams. --- @module fibers.stream - -local sc = require 'fibers.utils.syscall' -local buffer = require 'fibers.utils.ring_buffer' - -local ffi = sc.is_LuaJIT and require 'ffi' or require 'cffi' - -local Stream = {} -Stream.__index = Stream - -local DEFAULT_BUFFER_SIZE = 1024 - -local function open(io, readable, writable, buffer_size) - local ret = setmetatable( - { io = io, line_buffering = false, random_access = false }, - Stream) - if readable ~= false then - ret.rx = buffer.new(buffer_size or DEFAULT_BUFFER_SIZE) - end - if writable ~= false then - ret.tx = buffer.new(buffer_size or DEFAULT_BUFFER_SIZE) - end - if io.seek and io:seek(sc.SEEK_CUR, 0) then ret.random_access = true end - return ret -end - -local function is_stream(x) - return type(x) == 'table' and getmetatable(x) == Stream -end - -function Stream:nonblock() self.io:nonblock() end - -function Stream:block() self.io:block() end - -function Stream:fill(buf, count) - if self.random_access then self:flush_output() end - while true do - local did_read = self.io:read(buf, count) - if did_read then return did_read end - self.io:wait_for_readable() - end -end - -function Stream:fill_rx_buffer() - assert(self.rx:is_empty()) - self.rx:reset() - local did_read = self:fill(self.rx.buf, self.rx.size) - -- Note that did_read may be 0 in case of EOF. - self.rx:advance_write(did_read) - return did_read -end - -function Stream:flush_input() - if self.random_access and self.rx then - local buffered = self.rx:read_avail() - if buffered ~= 0 then - assert(self.io:seek('cur', -buffered)) - self.rx:reset() - end - end -end - -function Stream:flush_some_output() - assert(not self.tx:is_empty()) - local buf, count = self.tx:peek() - local did_write = self.io:write(buf, count) - if did_write then - self.tx:advance_read(did_write) - if self.tx:is_empty() then self.tx:reset() end - else - self.io:wait_for_writable() - return self:flush_some_output() - end -end - -function Stream:flush_output() - if not self.tx then return end - if self.tx:is_empty() then return end - self:flush_some_output() - if not self.tx:is_empty() then return self:flush_output() end -end - -Stream.flush = Stream.flush_output - --- Read up to COUNT bytes into BUF. Return number of bytes read. Will --- block until at least one byte is ready, or until EOF, in which case --- the return value is 0. -function Stream:read_some_bytes(buf, count) - buf = ffi.cast('uint8_t*', buf) - if self.rx:is_empty() then - -- If the target buffer is as large or larger than the read - -- buffer, read into the target buffer directly -- that way we - -- probably reduce the number of read calls. - if count >= self.rx.size then return self:fill(buf, count) end - -- Otherwise, fill the read buffer. - self:fill_rx_buffer() - end - -- count may be 0 in case of EOF. - count = math.min(count, self.rx:read_avail()) - self.rx:read(buf, count) - return count -end - --- Read COUNT bytes from the stream into BUF, blocking until more bytes --- are available. Return number of bytes read, which may be less than --- COUNT if the stream reaches EOF beforehand. -function Stream:read_bytes(buf, count) - buf = ffi.cast('uint8_t*', buf) - -- Unrolled fast-path to avoid nested loops. - local did_read = self:read_some_bytes(buf, count) - if did_read == count then return count end - if did_read == 0 then return 0 end - local offset = did_read - while offset < count do - local did_read_more = self:read_some_bytes(buf + offset, count - offset) - if did_read_more == 0 then break end - offset = offset + did_read_more - end - return offset -end - --- Read COUNT bytes into BUF, blocking until COUNT bytes are available. --- If EOF is reached before COUNT bytes are read, signal an error. -function Stream:read_bytes_or_error(buf, count) - if self:read_bytes(buf, count) ~= count then - error("early EOF while reading from stream") - end -end - -function Stream:read_all_bytes() - local head, count, block_size = nil, 0, 1024 - while true do - local buf = ffi.new('uint8_t[?]', count + block_size) - if head then ffi.copy(buf, head, count) end - local did_read = self:read_bytes(buf + count, block_size) - count = count + did_read - if did_read < block_size then return buf, count end - head, block_size = buf, block_size * 2 - end -end - -function Stream:read_struct(buf, type) - if buf == nil then buf = type() end - self:read_bytes_or_error(buf, ffi.sizeof(type)) - return buf -end - -local array_types = {} -local function get_array_type(t) - local at = array_types[t] - if not at then - at = ffi.typeof('$[?]', t) - array_types[t] = at - end - return at -end - -function Stream:read_array(buf, type, count) - if buf == nil then buf = get_array_type(type)(count) end - self:read_bytes_or_error(buf, ffi.sizeof(type) * count) - return buf -end - -function Stream:read_scalar(buf, type) - return self:read_array(buf, type, 1)[0] -end - -function Stream:peek_byte() - if self.rx:is_empty() then - -- Return nil on EOF. - if self:fill_rx_buffer() == 0 then return nil end - end - return self.rx.buf[self.rx:read_pos()] -end - -function Stream:peek_char() - local byte = self:peek_byte() - if byte == nil then return nil end - return string.char(byte) -end - -function Stream:read_byte() - local byte = self:peek_byte() - if byte ~= nil then self.rx:advance_read(1) end - return byte -end - -function Stream:read_char() - local byte = self:read_byte() - if byte ~= nil then return string.char(byte) end -end - --- Read up to COUNT characters from a stream and return them as a --- string. Blocks until at least one character is available, or the --- stream reaches EOF, in which case return nil instead. If COUNT is --- not given, it defaults to the current read buffer size. -function Stream:read_some_chars(count) - if count == nil then count = self.rx.size end - if self.rx:is_empty() then - if self:fill_rx_buffer() == 0 then return nil end - end - local buf, avail = self.rx:peek() - count = math.min(count, avail) - local ret = ffi.string(buf, count) - self.rx:advance_read(count) - return ret -end - --- Unlike read_bytes, always reads COUNT bytes. -function Stream:read_chars(count) - local buf = ffi.new('uint8_t[?]', count) - self:read_bytes_or_error(buf, count) - return ffi.string(buf, count) -end - -function Stream:read_all_chars() - return ffi.string(self:read_all_bytes()) -end - -function Stream:write_bytes(buf, count) - if self.tx:read_avail() == 0 then self:flush_input() end - buf = ffi.cast('uint8_t*', buf) - if count >= self.tx.size then - -- Write directly. - self:flush_output() - local did_write = self.io:write(buf, count) - if did_write then - buf, count = buf + did_write, count - did_write - else - self.io:wait_for_writable() - end - else - -- Write via buffer. - local to_put = math.min(self.tx:write_avail(), count) - self.tx:write(buf, to_put) - buf, count = buf + to_put, count - to_put - if self.tx:is_full() then self:flush_some_output() end - end - if count > 0 then return self:write_bytes(buf, count) end -end - -function Stream:write_chars(str) - assert(type(str) == 'string', 'argument not a string') - local needs_flush = false - if self.line_buffering and str:match('\n') then needs_flush = true end - self:write_bytes(str, #str) - if needs_flush then self:flush_output() end -end - -function Stream:write_struct(type, ptr) - self:write_bytes(ptr, ffi.sizeof(type)) -end - -function Stream:write_array(type, ptr, count) - self:write_bytes(ptr, ffi.sizeof(type) * count) -end - -function Stream:write_scalar(type, value) - local ptr = get_array_type(type)(1) - ptr[0] = value - assert(ptr[0] == value, "value out of range") - self:write_array(type, ptr, 1) -end - -function Stream:close() - self:flush_output() - self.rx, self.tx = nil, nil - local success, exit_type, code = self.io:close() - self.io = nil - return success, exit_type, code -end - -function Stream:lines(...) - -- Returns an iterator function that, each time it is called, reads - -- the file according to the given formats. - local formats = { ... } - if #formats == 0 then - return function() return self:read_line('discard') end -- Fast path. - end - return function() return self:read(unpack(formats)) end -end - -function Stream:read_number() - error('unimplemented') -end - -function Stream:read_line(eol_style) -- 'discard' or 'keep' - local head = {} - local add_lf = assert(({ discard = 0, keep = 1 })[eol_style or 'discard']) - while true do - if self.rx:is_empty() then - if self:fill_rx_buffer() == 0 then - -- EOF. - if #head == 0 then return nil end - return table.concat(head) - end - end - local buf, avail = self.rx:peek() - local lf = string.byte("\n") - for i = 0, avail - 1 do - if buf[i] == lf then - local tail = ffi.string(buf, i + add_lf) - self.rx:advance_read(i + 1) - if #head == 0 then return tail end - table.insert(head, tail) - return table.concat(head) - end - end - local tail = ffi.string(buf, avail) - self.rx:advance_read(avail) - table.insert(head, tail) - end -end - -local function read1(stream, format) - if format == '*n' then - -- "*n": reads a number; this is the only format that returns a - -- number instead of a string. - return stream:read_number() - elseif format == '*a' then - -- "*a": reads the whole file, starting at the current - -- position. On end of file, it returns the empty string. - return stream:read_all_chars() - elseif format == '*l' then - -- "*l": reads the next line (skipping the end of line), returning - -- nil on end of file. - return stream:read_line('discard') - elseif format == '*L' then - -- "*L": reads the next line keeping the end of line (if present), - -- returning nil on end of file. (Lua 5.2, present in LuaJIT.) - return stream:read_line('keep') - else - -- /number/: reads a string with up to this number of characters, - -- returning nil on end of file. - assert(type(format) == 'number', 'bad format') - local number = format - if number == 0 then - -- If number is zero, it reads nothing and returns an empty - -- string, or nil on end of file. - if not stream.rx.buf:is_empty() then return '' end - if stream:fill_rx_buffer() == 0 then return nil end -- EOF. - return '' - end - assert(number > 0 and number == math.floor(number)) - local buf = ffi.new('char[?]', number) - -- The Lua read() method is based on fread() which only returns a - -- short read on EOF or error, therefore we use read_bytes. - local did_read = stream:read_bytes(buf, number) - return ffi.string(buf, did_read) - end -end - --- Lua 5.1's file:read() method. -function Stream:read(...) - -- Reads the file file, according to the given formats, which specify - -- what to read. For each format, the function returns a string (or - -- a number) with the characters read, or nil if it cannot read data - -- with the specified format. When called without formats, it uses a - -- default format that reads the entire next line. - assert(self.rx, "expected a readable stream") - local args = { ... } - if #args == 0 then return self:read_line('discard') end -- Default format. - if #args == 1 then return read1(self, args[1]) end -- Fast path. - local res = {} - for _, format in ipairs(args) do table.insert(res, read1(self, format)) end - return unpack(res) -end - -function Stream:seek(whence, offset) - -- Sets and gets the file position, measured from the beginning of - -- the file, to the position given by offset plus a base specified by - -- the string whence, as follows: - if not self.random_access then return nil, 'stream is not seekable' end - if whence == nil then whence = sc.SEEK_CUR end - if offset == nil then offset = 0 end - if whence == sc.SEEK_CUR and offset == 0 then - -- Just a position query. - local ret, err = self.io:seek(sc.SEEK_CUR, 0) - if ret == nil then return ret, err end - if self.tx and self.tx:read_avail() ~= 0 then - return ret + self.tx:read_avail() - end - if self.rx and self.rx:read_avail() ~= 0 then - return ret - self.rx:read_avail() - end - return ret - end - self:flush_input(); self:flush_output() - return self.io:seek(whence, offset) -end - -local function transfer_buffered_bytes(old, new) - while old:read_avail() > 0 do - local buf, count = old:peek() - new:write(buf, count) - old:advance_read(count) - end -end - -function Stream:setvbuf(mode, size) - -- Sets the buffering mode for an output file. - if mode == 'no' then - self.line_buffering, size = false, 1 - elseif mode == 'line' then - self.line_buffering = true - elseif mode == 'full' then - self.line_buffering = false - else - error('bad mode', mode) - end - - if size == nil then size = DEFAULT_BUFFER_SIZE end - if self.rx and self.rx.size ~= size then - if self.rx:read_avail() > size then - error('existing buffered input is too much for new buffer size') - end - local new_rx = buffer.new(size) - transfer_buffered_bytes(self.rx, new_rx) - self.rx = new_rx - end - if self.tx and self.tx.size ~= size then - -- Note >= rather than > as we never leave tx buffers full. - while self.tx:read_avail() >= size do self:flush_some_output() end - local new_tx = buffer.new(size) - transfer_buffered_bytes(self.tx, new_tx) - self.tx = new_tx - end - return self -end - -local function write1(stream, arg) - if type(arg) == 'number' then arg = tostring(arg) end - stream:write_chars(arg) -end - -function Stream:write(...) - -- Writes the value of each of its arguments to the file. The - -- arguments must be strings or numbers. To write other values, use - -- tostring or string.format before write. - for _, arg in ipairs({ ... }) do write1(self, arg) end - return true -end - --- The result may be nil. -function Stream:filename() return self.io.filename end - -return { - open = open, - is_stream = is_stream -} diff --git a/src/fibers/stream/compat.lua b/src/fibers/stream/compat.lua deleted file mode 100644 index 7bbe387b..00000000 --- a/src/fibers/stream/compat.lua +++ /dev/null @@ -1,88 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - --- Shim to replace Lua's built-in IO module with streams. - -local stream = require 'fibers.stream' -local file = require 'fibers.stream.file' -local sc = require 'fibers.utils.syscall' - -local original_io = _G.io -- Save the original io module -local io = {} - -function io.close(f) - if f == nil then f = io.current_output end - f:close() -end - -function io.flush() - io.current_output:flush() -end - -function io.input(new) - if new == nil then return io.current_input end - if type(new) == string then new = io.open(new, 'r') end - io.current_input = new -end - -function io.lines(filename, ...) - if filename == nil then return io.current_input:lines() end - local fileStream = assert(io.open(filename, 'r')) - local iter = fileStream:lines(...) - return function() - local line = { iter() } - if line[1] == nil then - fileStream:close() - return nil - end - return unpack(line) - end -end - -io.open = file.open - -function io.output(new) - if new == nil then return io.current_output end - if type(new) == string then new = io.open(new, 'w') end - io.current_output = new -end - -function io.popen(prog, mode) - return file.popen(prog, mode) -end - -function io.read(...) - return io.current_input:read(...) -end - -io.tmpfile = file.tmpfile - -function io.type(x) - if not stream.is_stream(x) then return nil end - if not x.io then return 'closed file' end - return 'file' -end - -function io.write(...) - return io.current_output:write(...) -end - -local function install() - if _G.io == io then return end - _G.io = io - io.stdin = file.fdopen(sc.STDIN_FILENO, sc.O_RDONLY) - io.stdout = file.fdopen(sc.STDOUT_FILENO, sc.O_WRONLY) - io.stderr = file.fdopen(sc.STDERR_FILENO, sc.O_WRONLY) - if sc.isatty(io.stdout.io.fd) then io.stdout:setvbuf('line') end - io.stderr:setvbuf('no') - io.input(io.stdin) - io.output(io.stdout) -end - -local function uninstall() - _G.io = original_io -end - -return { - install = install, - uninstall = uninstall -} diff --git a/src/fibers/stream/file.lua b/src/fibers/stream/file.lua deleted file mode 100644 index d2cfb13a..00000000 --- a/src/fibers/stream/file.lua +++ /dev/null @@ -1,263 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - ---- fibers.stream.file module --- A stream IO implementation for file descriptors. --- @module fibers.stream.file - -package.path = "../../?.lua;../?.lua;" .. package.path - -local stream = require 'fibers.stream' -local sc = require 'fibers.utils.syscall' - -local bit = rawget(_G, "bit") or require 'bit32' - --- A blocking handler provides for configurable handling of EWOULDBLOCK --- conditions. The goal is to allow for normal blocking operations, but --- also to allow for a cooperative coroutine-based multitasking system --- to run other tasks when a stream would block. --- --- In the case of normal, blocking file descriptors, the blocking --- handler will only be called if a read or a write returns EAGAIN, --- presumably because the sc was interrupted by a signal. In that --- case the correct behavior is to just return directly, which will --- cause the stream to try again. --- --- For nonblocking file descriptors, the blocking handler could suspend --- the current coroutine and arrange to restart it once the FD becomes --- readable or writable. However the default handler here doesn't --- assume that we're running in a coroutine. In that case we could --- block in a poll() without suspending. Currently however the default --- blocking handler just returns directly, which will cause the stream --- to busy-wait until the FD becomes active. - -local blocking_handler - -local default_blocking_handler = {} -function default_blocking_handler:init_nonblocking() end - -function default_blocking_handler:wait_for_readable() end - -function default_blocking_handler:wait_for_writable() end - -function default_blocking_handler:wait_for_priority() end - -local function set_blocking_handler(h) - blocking_handler = h or default_blocking_handler -end - -set_blocking_handler() - -local function init_nonblocking(fd) blocking_handler:init_nonblocking(fd) end -local function wait_for_readable(fd) blocking_handler:wait_for_readable(fd) end -local function wait_for_writable(fd) blocking_handler:wait_for_writable(fd) end -local function wait_for_priority(fd) blocking_handler:wait_for_priority(fd) end - -local File = {} -local File_mt = { __index = File } - -local function new_file_io(fd, filename) - init_nonblocking(fd) - return setmetatable({ fd = fd, filename = filename }, File_mt) -end - -function File:nonblock() sc.set_nonblock(self.fd) end - -function File:block() sc.set_block(self.fd) end - -function File:read(buf, count) - local did_read, err, errno = sc.ffi.read(self.fd, buf, count) - if errno then - -- If the read would block, indicate to caller with nil return. - if errno == sc.EAGAIN or errno == sc.EWOULDBLOCK then return nil end - -- Otherwise, signal an error. - error(err) - else - -- Success; return number of bytes read. If EOF, count is 0. - return did_read - end -end - -function File:write(buf, count) - local did_write, err, errno = sc.ffi.write(self.fd, buf, count) - if err then - -- If the write would block, indicate to caller with nil return. - if errno == sc.EAGAIN or errno == sc.EWOULDBLOCK then return nil end - -- Otherwise, signal an error. - error(err) - elseif did_write == 0 then - -- This is a bit of a squirrely case: no bytes written, but no - -- error code. Return nil to indicate that it's probably a good - -- idea to wait until the FD is writable again. - return nil - else - -- Success; return number of bytes written. - return did_write - end -end - -function File:seek(whence, offset) - -- In case of success, return the final file position, measured in - -- bytes from the beginning of the file. On failure, return nil, - -- plus a string describing the error. - return sc.lseek(self.fd, offset, whence) -end - -function File:wait_for_readable() wait_for_readable(self.fd) end - -function File:wait_for_writable() wait_for_writable(self.fd) end - -function File:wait_for_priority() wait_for_priority(self.fd) end - -function File:close() - sc.close(self.fd) - self.fd = nil -end - -local function fdopen(fd, flags, filename) - local io = new_file_io(fd, filename) - if flags == nil then - flags = assert(sc.fcntl(fd, sc.F_GETFL)) - -- this appears only to be relevant to 32 bit systems, ljsc has - -- reference to this being a flag with value octal('0100000') on such systems - else - flags = bit.bor(flags, sc.O_LARGEFILE) - end - local readable, writable = false, false - local mode = bit.band(flags, sc.O_ACCMODE) - if mode == sc.O_RDONLY or mode == sc.O_RDWR then readable = true end - if mode == sc.O_WRONLY or mode == sc.O_RDWR then writable = true end - local stat = sc.fstat(fd) - return stream.open(io, readable, writable, stat and stat.st_blksize) -end - -local modes = { - r = sc.O_RDONLY, - w = bit.bor(sc.O_WRONLY, sc.O_CREAT, sc.O_TRUNC), - a = bit.bor(sc.O_WRONLY, sc.O_CREAT, sc.O_APPEND), - ['r+'] = sc.O_RDWR, - ['w+'] = bit.bor(sc.O_RDWR, sc.O_CREAT, sc.O_TRUNC), - ['a+'] = bit.bor(sc.O_RDWR, sc.O_CREAT, sc.O_APPEND) -} -do - local binary_modes = {} - for k, v in pairs(modes) do binary_modes[k .. 'b'] = v end - for k, v in pairs(binary_modes) do modes[k] = v end -end - -local permissions = {} -permissions['rw-r--r--'] = bit.bor(sc.S_IRUSR, sc.S_IWUSR, sc.S_IRGRP, sc.S_IROTH) -permissions['rw-rw-rw-'] = bit.bor(permissions['rw-r--r--'], sc.S_IWGRP, sc.S_IWOTH) - -local function open(filename, mode, perms) - if mode == nil then mode = 'r' end - local flags = modes[mode] - if flags == nil then return nil, 'invalid mode: ' .. tostring(mode) end - -- This set of permissions is what open() uses. Note that these - -- permissions will be modulated by the umask. - if perms == nil then perms = permissions['rw-rw-rw-'] end - local fd, err, _ = sc.open(filename, flags, permissions[perms]) - if fd == nil then return nil, err end - return fdopen(fd, flags, filename) -end - -local function mktemp(name, perms) - if perms == nil then perms = permissions['rw-r--r--'] end - -- In practice this requires that someone seeds math.random with good - -- entropy. In Snabb that is the case (see core.main:initialize()). - local t = math.random(1e7) - local tmpnam, fd, err, _ - for i = t, t + 10 do - tmpnam = name .. '.' .. i - fd, err, _ = sc.open(tmpnam, bit.bor(sc.O_CREAT, sc.O_RDWR, sc.O_EXCL), perms) - if fd then return fd, tmpnam end - end - error("Failed to create temporary file " .. tmpnam .. ": " .. err) -end - -local function tmpfile(perms, tmpdir) - if tmpdir == nil then tmpdir = os.getenv("TMPDIR") or "/tmp" end - local fd, tmpnam = mktemp(tmpdir .. '/' .. 'tmp', perms) - local f = fdopen(fd, sc.O_RDWR, tmpnam) - -- FIXME: Doesn't arrange to ensure the file is removed in all cases; - -- calling close is required. - function f:rename(new) - self:flush() - sc.fsync(self.io.fd) - local res, err = sc.rename(self.io.filename, new) - if not res then - error("failed to rename " .. self.io.filename .. " to " .. new .. ": " .. tostring(err)) - end - self.io.filename = new - self.io.close = File.close -- Disable remove-on-close. - end - - function f.io:close() - File.close(self) - local res, err = sc.unlink(self.filename) - if not res then - error('failed to remove ' .. self.filename .. ': ' .. tostring(err)) - end - end - - return f -end - -local function pipe() - local rd, wr = assert(sc.pipe()) - return fdopen(rd, sc.O_RDONLY), fdopen(wr, sc.O_WRONLY) -end - -local function popen(prog, mode) - assert(type(prog) == 'string') - assert(mode == 'r' or mode == 'w') - local parent_half, child_half - do - local rd, wr = assert(sc.pipe()) - if mode == 'r' then - parent_half, child_half = rd, wr - else - parent_half, child_half = wr, rd - end - end - local pid = assert(sc.fork()) - if pid == 0 then - sc.close(parent_half) - sc.dup2(child_half, mode == 'r' and 1 or 0) - sc.close(child_half) - sc.execve('/bin/sh', { "-c", prog }) - sc.write(2, "io.popen: Failed to exec /bin/sh!") - sc.exit(255) - end - sc.close(child_half) - local io = new_file_io(parent_half) - local close = io.close - function io:close() - if not pid then return end - close(self) - local ch_pid, status, code - repeat - ch_pid, status, code = sc.waitpid(pid, sc.WNOHANG) - -- some kind of sleep here, surely, if used in a fibers context - until (ch_pid and status ~= 'running') or (not ch_pid and code ~= sc.EINTR) - pid = nil - local retval1 = (status == "exited" and code == 0) or nil - local retval2 = status == "exited" and "exit" or "signal" - local retval3 = code - return retval1, retval2, retval3 - end - - return stream.open(io, mode == 'r', mode == 'w') -end - -return { - init_nonblocking = init_nonblocking, - wait_for_readable = wait_for_readable, - wait_for_writable = wait_for_writable, - wait_for_priority = wait_for_priority, - set_blocking_handler = set_blocking_handler, - fdopen = fdopen, - open = open, - tmpfile = tmpfile, - pipe = pipe, - popen = popen, -} diff --git a/src/fibers/stream/mem.lua b/src/fibers/stream/mem.lua deleted file mode 100644 index 4d74e1a9..00000000 --- a/src/fibers/stream/mem.lua +++ /dev/null @@ -1,123 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - --- A memory-backed stream IO implementation. - -local stream = require 'fibers.stream' -local sc = require 'fibers.utils.syscall' - -local ffi = sc.is_LuaJIT and require 'ffi' or require 'cffi' - -local Mem = {} -Mem.__index = Mem - -local INITIAL_SIZE = 4096 - -local function new_buffer(len) return ffi.new('uint8_t[?]', len) end - -local function new_mem_io(buf, len, size, growable) - if buf == nil then - if size == nil then size = len or INITIAL_SIZE end - buf = new_buffer(size) - else - if size == nil then size = len end - assert(size ~= nil) - end - if len == nil then len = 0 end - return setmetatable( - { buf = buf, pos = 0, len = len, size = size, growable = growable }, - Mem) -end - -function Mem:nonblock() end - -function Mem:block() end - -function Mem:read(buf, count) - count = math.min(count, self.len - self.pos) - ffi.copy(buf, self.buf + self.pos, count) - self.pos = self.pos + count - return count -end - -function Mem:grow_buffer(count) - assert(self.growable, "ran out of space while writing") - if self.len == self.size then - self.size = math.max(self.size * 2, 1024) - local buf = new_buffer(self.size) - ffi.copy(buf, self.buf, self.len) - self.buf = buf - end - self.len = math.min(self.size, self.len + count) - return self.len -end - -function Mem:write(buf, count) - if self.pos == self.len then self:grow_buffer(count) end - count = math.min(count, self.len - self.pos) - ffi.copy(self.buf + self.pos, buf, count) - self.pos = self.pos + count - return count -end - -function Mem:seek(whence, offset) - if whence == sc.SEEK_CUR then - offset = self.pos + offset - elseif whence == sc.SEEK_END then - offset = self.len + offset - elseif whence ~= sc.SEEK_SET then - error('bad "whence": ' .. tostring(whence)) - end - if offset < 0 then return nil, "invalid offset" end - while self.len < offset do self:grow_buffer(offset - self.len) end - self.pos = offset - return offset -end - -function Mem:wait_for_readable() end - -function Mem:wait_for_writable() end - -function Mem:close() - self.buf, self.pos, self.len, self.size, self.growable = nil, nil, nil, nil, nil -end - -local readable_modes = { r = true, ['r+'] = true, ['w+'] = true } -local writable_modes = { ['r+'] = true, w = true, ['w+'] = true } - -local function open(buf, len, mode) - if mode == nil then mode = 'r+' end - local readable, writable = readable_modes[mode], writable_modes[mode] - assert(readable or writable) - local io = new_mem_io(buf, len, len, writable) - return stream.open(io, readable, writable) -end - -local function tmpfile() - return open() -end - -local function open_input_string(str) - local len = #str - local buf = new_buffer(len) - ffi.copy(buf, str, len) - local readable, writable = true, false - local io = new_mem_io(buf, len, len, writable) - return stream.open(io, readable, writable) -end - -local function call_with_output_string(f, ...) - local out = tmpfile() - local args = { ... } - table.insert(args, out) - f(unpack(args)) - out:flush_output() - -- Can take advantage of internals to read directly. - return ffi.string(out.io.buf, out.io.len) -end - -return { - open = open, - tmpfile = tmpfile, - open_input_string = open_input_string, - call_with_output_string = call_with_output_string -} diff --git a/src/fibers/stream/socket.lua b/src/fibers/stream/socket.lua deleted file mode 100644 index 6a6aa9d8..00000000 --- a/src/fibers/stream/socket.lua +++ /dev/null @@ -1,90 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - --- A stream IO implementation for sockets. - -local file = require 'fibers.stream.file' -local sc = require 'fibers.utils.syscall' - -local Socket = {} -Socket.__index = Socket - -local sigpipe_handler - -local function socket(domain, stype, protocol) - if sigpipe_handler == nil then sigpipe_handler = sc.signal(sc.SIGPIPE, sc.SIG_IGN) end - local fd = assert(sc.socket(domain, stype, protocol or 0)) - file.init_nonblocking(fd) - return setmetatable({ fd = fd }, Socket) -end - -function Socket:listen_unix(f) - local sa = sc.getsockname(self.fd) - sa.path = f - assert(sc.bind(self.fd, sa)) - assert(sc.listen(self.fd)) -end - -function Socket:accept() - while true do - local fd, err, errno = sc.accept(self.fd) - if fd then - return file.fdopen(fd) - elseif errno == sc.EAGAIN or errno == sc.EWOULDBLOCK then - file.wait_for_readable(self.fd) - else - error(err) - end - end -end - -function Socket:connect(sa) - local ok, err, errno = sc.connect(self.fd, sa) - if not ok and errno == sc.EINPROGRESS then - -- Bonkers semantics; see connect(2). - file.wait_for_writable(self.fd) - err = assert(sc.getsockopt(self.fd, sc.SOL_SOCKET, sc.SO_ERROR)) - if err == 0 then ok = true end - end - if ok then - local fd = self.fd - self.fd = nil - return file.fdopen(fd) - end - error(err) -end - -function Socket:connect_unix(f) - local sa = sc.getsockname(self.fd) - sa.path = f - return self:connect(sa) -end - -local function listen_unix(f, args) - args = args or {} - local s = socket(sc.AF_UNIX, args.stype or sc.SOCK_STREAM, args.protocol) - s:listen_unix(f) - if args.ephemeral then - local parent_close = s.close - function s:close() - parent_close(s) - sc.unlink(f) - end - end - return s -end - -local function connect_unix(f, stype, protocol) - local s = socket(sc.AF_UNIX, stype or sc.SOCK_STREAM, protocol) - return s:connect_unix(f) -end - -function Socket:close() - if self.fd then sc.close(self.fd) end - self.fd = nil -end - -return { - socket = socket, - listen_unix = listen_unix, - connect_unix = connect_unix -} diff --git a/src/fibers/timer.lua b/src/fibers/timer.lua deleted file mode 100644 index 04d7a997..00000000 --- a/src/fibers/timer.lua +++ /dev/null @@ -1,135 +0,0 @@ --- (c) Jangala - --- Use of this source code is governed by the XXXXXXXXX license; see COPYING. - ---- Binary Heap based timer. --- Implements a Binary Heap based timer. This is a time based event scheduler, --- used for efficiently scheduling and managing events. --- @module fibers.timer - --- Required packages -local sc = require 'fibers.utils.syscall' - ---- BinaryHeap class. --- @type BinaryHeap -local BinaryHeap = {} -BinaryHeap.__index = BinaryHeap - ---- BinaryHeap constructor. --- @treturn BinaryHeap BinaryHeap instance. -function BinaryHeap:new() - return setmetatable({heap = {}, size = 0}, BinaryHeap) -end - ---- Pushes a node into the heap and heapify it. --- @tparam table node The node to be pushed into the heap. -function BinaryHeap:push(node) - self.size = self.size + 1 - self.heap[self.size] = node - self:heapify_up(self.size) -end - ---- Pops a node from the underlying heap and reheapifies. Does not advance the timer! --- @treturn table|nil The root node popped from the heap, nil if the heap is empty. -function BinaryHeap:pop() - if self.size == 0 then - return nil - end - - local root = self.heap[1] - self.heap[1] = self.heap[self.size] - self.size = self.size - 1 - self:heapify_down(1) - return root -end - ---- Maintains the heap property by moving a node up the heap. --- @tparam number idx The index of the node in the heap array. -function BinaryHeap:heapify_up(idx) - if idx <= 1 then - return - end - - local parent = math.floor(idx / 2) - if self.heap[parent].time > self.heap[idx].time then - self.heap[parent], self.heap[idx] = self.heap[idx], self.heap[parent] - self:heapify_up(parent) - end -end - ---- Maintains the heap property by moving a node down the heap. --- @tparam number idx The index of the node in the heap array. -function BinaryHeap:heapify_down(idx) - local smallest = idx - local left = 2 * idx - local right = 2 * idx + 1 - - if left <= self.size and self.heap[left].time < self.heap[smallest].time then - smallest = left - end - if right <= self.size and self.heap[right].time < self.heap[smallest].time then - smallest = right - end - if smallest ~= idx then - self.heap[idx], self.heap[smallest] = self.heap[smallest], self.heap[idx] - self:heapify_down(smallest) - end -end - ---- Timer class. --- @type Timer -local Timer = {} -Timer.__index = Timer - ---- Timer constructor. --- @tparam[opt=now] number now The current time. --- @treturn Timer New Timer instance. -local function new(now) - now = now or sc.monotime() - return setmetatable({now = now, heap = BinaryHeap:new()}, Timer) -end - ---- Adds an object to the timer with an absolute time. --- @tparam number t The absolute time. --- @tparam any obj The object to add to the timer. -function Timer:add_absolute(t, obj) - self.heap:push({time = t, obj = obj}) -end - ---- Adds an object to the timer with a delta time. --- @tparam number dt The delta time. --- @tparam any obj The object to add to the timer. -function Timer:add_delta(dt, obj) - return self:add_absolute(self.now + dt, obj) -end - ---- Returns the time of the next entry in the timer. --- @treturn number The time of the next entry in the timer, or infinity if the heap is empty. -function Timer:next_entry_time() - if self.heap.size == 0 then - return 1/0 -- infinity - end - return self.heap.heap[1].time -end - ---- Returns the time of the next entry in the timer. --- @treturn number The time of the next entry in the timer, or infinity if the heap is empty. -function Timer:pop() - return self.heap:pop() -end - ---- Advances the timer, popping and scheduling objects from the heap as necessary. --- @tparam number t The time to advance the timer to. --- @tparam table sched The scheduler to use for scheduling objects. -function Timer:advance(t, sched) - while self.heap.size > 0 and t >= self.heap.heap[1].time do - local node = self.heap:pop() - self.now = node.time - sched:schedule(node.obj) - end - self.now = t -end - -return { - new = new -} \ No newline at end of file diff --git a/src/fibers/utils/helper.lua b/src/fibers/utils/helper.lua deleted file mode 100644 index 4549ac5c..00000000 --- a/src/fibers/utils/helper.lua +++ /dev/null @@ -1,46 +0,0 @@ --- Copyright Snabb --- Copyright Jangala - -local sc = require 'fibers.utils.syscall' -local ffi = sc.is_LuaJIT and require 'ffi' or require 'cffi' -ffi.type = ffi.type or type - --- Returns true if x and y are structurally similar (isomorphic). -local function equal(x, y) - if type(x) ~= type(y) then return false end - if type(x) == 'table' then - for k, v in pairs(x) do - if not equal(v, y[k]) then return false end - end - for k, _ in pairs(y) do - if x[k] == nil then return false end - end - return true - elseif ffi.type(x) == 'cdata' then - if x == y then return true end - if ffi.typeof(x) ~= ffi.typeof(y) then return false end - local size = ffi.sizeof(x) - if ffi.sizeof(y) ~= size then return false end - return sc.ffi.memcmp(x, y, size) == 0 - else - return x == y - end -end - -local function dump(o) - if type(o) == 'table' then - local s = '{ ' - for k, v in pairs(o) do - if type(k) ~= 'number' then k = '"' .. k .. '"' end - s = s .. '[' .. k .. '] = ' .. dump(v) .. ',' - end - return s .. '} ' - else - return tostring(o) - end -end - -return { - equal = equal, - dump = dump -} diff --git a/src/fibers/utils/ring_buffer.lua b/src/fibers/utils/ring_buffer.lua deleted file mode 100644 index ab4d6700..00000000 --- a/src/fibers/utils/ring_buffer.lua +++ /dev/null @@ -1,120 +0,0 @@ --- Use of this source code is governed by the Apache 2.0 license; see COPYING. - --- Ring buffer for bytes - --- detect LuaJIT (removing dependency on utils.syscall) -local is_LuaJIT = rawget(_G, "jit") and true or false - -local bit = rawget(_G, "bit") or require 'bit32' -local ffi = is_LuaJIT and require 'ffi' or require 'cffi' - -local band = bit.band - -ffi.cdef [[ - typedef struct { - uint32_t read_idx, write_idx; - uint32_t size; - uint8_t buf[?]; - } buffer_t; -]] - -local function to_uint32(n) - return n % 2 ^ 32 -end - -local buffer = {} -buffer.__index = buffer - -function buffer:init(size) - assert(size ~= 0 and band(size, size - 1) == 0, "size not power of two") - self.size = size - return self -end - -function buffer:reset() - self.write_idx, self.read_idx = 0, 0 -end - -function buffer:is_empty() - return self.write_idx == self.read_idx -end - -function buffer:read_avail() - return to_uint32(self.write_idx - self.read_idx) -end - -function buffer:is_full() - return self:read_avail() == self.size -end - -function buffer:write_avail() - return self.size - self:read_avail() -end - -function buffer:write_pos() - return band(self.write_idx, self.size - 1) -end - -function buffer:rewrite_pos(offset) - return band(self.read_idx + offset, self.size - 1) -end - -function buffer:read_pos() - return band(self.read_idx, self.size - 1) -end - -function buffer:advance_write(count) - self.write_idx = self.write_idx + ffi.cast("uint32_t", count) -end - -function buffer:advance_read(count) - self.read_idx = self.read_idx + ffi.cast("uint32_t", count) -end - -function buffer:write(bytes, count) - if count > self:write_avail() then error('write xrun') end - local pos = self:write_pos() - local count1 = math.min(self.size - pos, count) - ffi.copy(self.buf + pos, bytes, count1) - ffi.copy(self.buf, bytes + count1, count - count1) - self:advance_write(count) -end - -function buffer:rewrite(offset, bytes, count) - if offset + count > self:read_avail() then error('rewrite xrun') end - local pos = self:rewrite_pos(offset) - local count1 = math.min(self.size - pos, count) - ffi.copy(self.buf + pos, bytes, count1) - ffi.copy(self.buf, bytes + count1, count - count1) -end - -function buffer:read(bytes, count) - if count > self:read_avail() then error('read xrun') end - local pos = self:read_pos() - local count1 = math.min(self.size - pos, count) - ffi.copy(bytes, self.buf + pos, count1) - ffi.copy(bytes + count1, self.buf, count - count1) - self:advance_read(count) -end - -function buffer:drop(count) - if count > self:read_avail() then error('read xrun') end - self:advance_read(count) -end - -function buffer:peek() - local pos = self:read_pos() - return self.buf + pos, math.min(self:read_avail(), self.size - pos) -end - -local buffer_t = ffi.metatype("buffer_t", buffer) - -local function new(size) - local ret = buffer_t(size) - ret:init(size) - return ret -end - -return { - new = new -} diff --git a/src/fibers/utils/string_buffer.lua b/src/fibers/utils/string_buffer.lua deleted file mode 100644 index 9a3922f5..00000000 --- a/src/fibers/utils/string_buffer.lua +++ /dev/null @@ -1,102 +0,0 @@ --- Use of the provided source code for ring buffer - -local ring_buffer = require 'fibers.utils.ring_buffer' -- assuming the above code is saved as ring_buffer.lua - -local is_LuaJIT = rawget(_G, "jit") and true or false -local ffi = is_LuaJIT and require 'ffi' or require 'cffi' - -local str_buffer = {} -str_buffer.__index = str_buffer - -function str_buffer.new(size) - local obj = {} - obj.buf = ring_buffer.new(size or 64) -- Default size is 64 - return setmetatable(obj, str_buffer) -end - -function str_buffer:len() - return self.buf:read_avail() -end - -function str_buffer:cap() - return self.buf.size -end - -function str_buffer:reset() - self.buf:reset() -end - -function str_buffer:string() - local data, length = self.buf:peek() - return ffi.string(data, length) -end - -function str_buffer:write(data) - if not data then return end - local len = #data - - if self.buf:write_avail() < len then - self:grow(len - self.buf:write_avail()) - end - - local cdata = ffi.new("uint8_t[?]", len) - ffi.copy(cdata, data, len) - self.buf:write(cdata, len) -end - -function str_buffer:write_to(w) - -- Assuming 'w' is a function or a file-like object that can accept a string. - local data = self:string() - w(data) - self.buf:drop(#data) -end - -function str_buffer:read(n) - local count = n or self.buf:read_avail() - local buffer = ffi.new('uint8_t[?]', count) - self.buf:read(buffer, count) - return ffi.string(buffer, count) -end - -function str_buffer:next(n) - local buffer = ffi.new('uint8_t[?]', n) - self.buf:read(buffer, n) - return ffi.string(buffer, n) -end - -function str_buffer:unread_char() - -- for simplicity, we'll just advance the read index backward by 1 - self.buf:advance_read(-1) -end - -function str_buffer:grow(n) - if n < 0 then - error("str_buffer: negative count") - end - - local new_size = self:cap() + n - -- Round up to the nearest power of 2 for efficiency. - local power = 1 - while power < new_size do - power = power * 2 - end - new_size = power - - local new_buf = ring_buffer.new(new_size) - - -- Handle wrap around - local first_chunk_len = math.min(self.buf:read_avail(), self.buf.size - self.buf:read_pos()) - local second_chunk_len = self.buf:read_avail() - first_chunk_len - - -- Copy from old buffer to the new buffer in two steps if wrapped around - new_buf:write(self.buf.buf + self.buf:read_pos(), first_chunk_len) - if second_chunk_len > 0 then - new_buf:write(self.buf.buf, second_chunk_len) - end - - self.buf = new_buf -end - -return { - new = str_buffer.new -} diff --git a/src/fibers/utils/syscall.lua b/src/fibers/utils/syscall.lua deleted file mode 100644 index 719fa31f..00000000 --- a/src/fibers/utils/syscall.lua +++ /dev/null @@ -1,456 +0,0 @@ ----@diagnostic disable: inject-field --- Copyright Jangala - -local p_fcntl = require 'posix.fcntl' -local p_unistd = require 'posix.unistd' -local p_stdio = require 'posix.stdio' -local p_wait = require 'posix.sys.wait' -local p_stat = require 'posix.sys.stat' -local p_signal = require 'posix.signal' -local p_socket = require 'posix.sys.socket' -local p_errno = require 'posix.errno' -local p_time = require 'posix.time' -local bit = rawget(_G, "bit") or require 'bit32' - -local M = { ffi = {} } -- used this module format due to large number of exported functions - ---detect LuaJIT -M.is_LuaJIT = rawget(_G, "jit") and true or false - -local ffi = M.is_LuaJIT and require 'ffi' or require 'cffi' -ffi.tonumber = ffi.tonumber or tonumber -ffi.type = ffi.type or type - -local ARCH = ffi.arch - -------------------------------------------------------------------------------- --- Compatibility functions -table.pack = table.pack or function(...) -- luacheck: ignore -- Compatibility fallback - return { n = select("#", ...), ... } -end - - -------------------------------------------------------------------------------- --- Local functions (for efficiency) - -local band, bor, bnot, _ = bit.band, bit.bor, bit.bnot, bit.lshift - - -------------------------------------------------------------------------------- --- Syscall constants - -M.SEEK_CUR = p_unistd.SEEK_CUR -M.SEEK_END = p_unistd.SEEK_END -M.SEEK_SET = p_unistd.SEEK_SET - -M.O_ACCMODE = 3 -M.O_RDONLY = p_fcntl.O_RDONLY -M.O_WRONLY = p_fcntl.O_WRONLY -M.O_RDWR = p_fcntl.O_RDWR -M.O_CREAT = p_fcntl.O_CREAT -M.O_TRUNC = p_fcntl.O_TRUNC -M.O_APPEND = p_fcntl.O_APPEND -M.O_EXCL = p_fcntl.O_EXCL -M.O_NONBLOCK = p_fcntl.O_NONBLOCK -M.O_LARGEFILE = ffi.abi('32bit') and 32768 or 0 - -M.F_GETFL = p_fcntl.F_GETFL -M.F_SETFL = p_fcntl.F_SETFL - -M.EAGAIN = p_errno.EAGAIN -M.EWOULDBLOCK = p_errno.EWOULDBLOCK -M.EINTR = p_errno.EINTR -M.EINPROGRESS = p_errno.EINPROGRESS -M.ESRCH = p_errno.ESRCH - -M.SIGKILL = p_signal.SIGKILL -M.SIGTERM = p_signal.SIGTERM - -M.S_IRUSR = p_stat.S_IRUSR -M.S_IWUSR = p_stat.S_IWUSR -M.S_IXUSR = p_stat.S_IXUSR -M.S_IRGRP = p_stat.S_IRGRP -M.S_IWGRP = p_stat.S_IWGRP -M.S_IXGRP = p_stat.S_IXGRP -M.S_IROTH = p_stat.S_IROTH -M.S_IWOTH = p_stat.S_IWOTH -M.S_IXOTH = p_stat.S_IXOTH - -M.STDIN_FILENO = p_unistd.STDIN_FILENO -M.STDOUT_FILENO = p_unistd.STDOUT_FILENO -M.STDERR_FILENO = p_unistd.STDERR_FILENO - -M.SIGPIPE = p_signal.SIGPIPE -M.SIG_IGN = p_signal.SIG_IGN -M.SIGCHLD = p_signal.SIGCHLD - -M.CLOCK_REALTIME = p_time.CLOCK_REALTIME -M.CLOCK_MONOTONIC = p_time.CLOCK_MONOTONIC - -M.AF_INET = p_socket.AF_INET -M.AF_INET6 = p_socket.AF_INET6 -M.AF_NETLINK = p_socket.AF_NETLINK -M.AF_PACKET = p_socket.AF_PACKET -M.AF_UNIX = p_socket.AF_UNIX -M.AF_UNSPEC = p_socket.AF_UNSPEC -M.SO_ERROR = p_socket.SO_ERROR -M.SOCK_DGRAM = p_socket.SOCK_DGRAM -M.SOCK_RAW = p_socket.SOCK_RAW -M.SOCK_STREAM = p_socket.SOCK_STREAM -M.SOL_SOCKET = p_socket.SOL_SOCKET -M.SOMAXCONN = p_socket.SOMAXCONN - -M.WNOHANG = p_wait.WNOHANG - -------------------------------------------------------------------------------- --- Luafied stdlib syscalls - -function M.fcntl(fd, ...) return p_fcntl.fcntl(fd, ...) end -function M.open(path, mode, perm) return p_fcntl.open(path, mode, perm) end - -function M.strerror(err) return p_errno.errno(err) end - -function M.stat(path) return p_stat.stat(path) end -function M.fstat(file, ...) return p_stat.fstat(file, ...) end - -function M.signal(signum, handler) return p_signal.signal(signum, handler) end -function M.kill(pid, options) return p_signal.kill(pid, options) end -function M.killpg(pgid, sig) return p_signal.kill(pgid, sig) end - -function M.accept(fd) return p_socket.accept(fd) end -function M.bind(file, sockaddr) return p_socket.bind(file, sockaddr) end -function M.connect(fd, addr) return p_socket.connect(fd, addr) end -function M.getpeername(sockfd) return p_socket.getpeername(sockfd) end -function M.getsockname(sockfd) return p_socket.getsockname(sockfd) end -function M.getsockopt(fd, level, name) return p_socket.getsockopt(fd, level, name) end -function M.listen(fd, backlog) return p_socket.listen(fd, backlog or M.SOMAXCONN) end -function M.socket(family, socktype, protocol) return p_socket.socket(family, socktype, protocol) end - -function M.fileno(file) return p_stdio.fileno(file) end -function M.rename(from, to) return p_stdio.rename(from, to) end - -function M.clock_gettime(id) return p_time.clock_gettime(id) end - -function M.access(path, mode) return p_unistd.access(path, mode) end -function M.close(fd) return p_unistd.close(fd) end -function M.dup2(fd1, fd2) return p_unistd.dup2(fd1, fd2) end -function M.exec(path, argt) return p_unistd.exec(path, argt) end -function M.execp(path, argt) return p_unistd.execp(path, argt) end -function M.execve(path, argv, _) return p_unistd.exec(path, argv) end -function M.fork() return p_unistd.fork() end -function M.fsync(fd) return p_unistd.fsync(fd) end -function M.getpgrp() return p_unistd.getpgrp() end -function M.getpid() return p_unistd.getpid() end -function M.isatty(fd) return p_unistd.isatty(fd) end -function M.lseek(file, offset, whence) return p_unistd.lseek(file, offset, whence) end -function M.pipe() return p_unistd.pipe() end -function M.read(fd, count) return p_unistd.read(fd, count) end -function M.setpid(what, id, gid) return p_unistd.setpid(what, id, gid) end -function M.unlink(path) return p_unistd.unlink(path) end -function M.write(fd, buf) return p_unistd.write(fd, buf) end - -function M.waitpid(pid, options) return p_wait.wait(pid, options) end - -function M.exit(status) return os.exit(status) end - -------------------------------------------------------------------------------- --- Convenience functions - -function M.set_nonblock(fd) - local flags = assert(M.fcntl(fd, M.F_GETFL)) - assert( M.fcntl(fd, M.F_SETFL, bor(flags, M.O_NONBLOCK))) -end - -function M.set_block(fd) - local flags = assert(M.fcntl(fd, M.F_GETFL)) - assert( M.fcntl(fd, M.F_SETFL, band(flags, bnot(M.O_NONBLOCK)))) -end - -function M.monotime() - local time = M.clock_gettime(M.CLOCK_MONOTONIC) - return time.tv_sec + time.tv_nsec/1e9, time.tv_sec, time.tv_nsec -end - -function M.realtime() - local time = M.clock_gettime(M.CLOCK_REALTIME) - return time.tv_sec + time.tv_nsec/1e9, time.tv_sec, time.tv_nsec -end - -function M.floatsleep(t) - local sec = t - t%1 - local nsec = t%1 * 1e9 - local _, _, _, remaining = p_time.nanosleep({tv_sec=sec, tv_nsec=nsec}) - while remaining do - p_time.nanosleep(remaining) - end -end - -local function wrap_error(retval) - if retval >= 0 then - return retval - else - local errno = ffi.errno() - return nil, M.strerror(errno), errno - end -end - ------------------------------------- --- epoll - -if ARCH == "x64" or ARCH == "x86" then - ffi.cdef[[ - typedef struct epoll_event { - uint8_t raw[12]; // 4 bytes for events + 8 bytes for data - } epoll_event; - ]] -elseif ARCH == "mips" or ARCH == "mipsel" or ARCH == "arm64" then - ffi.cdef[[ - typedef struct epoll_event { - uint32_t events; - uint64_t data; - } epoll_event; - ]] -else - error(ARCH.." architecture not specified") -end - -ffi.cdef[[ - int epoll_create(int size); - int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); - int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); - - int fcntl(int fd, int cmd, ...); - int close(int fd); - char *strerror(int errnum); -]] - -M.EPOLLIN = 0x00000001 -M.EPOLLPRI = 0x00000002 -M.EPOLLOUT = 0x00000004 -M.EPOLLERR = 0x00000008 -M.EPOLLHUP = 0x00000010 -M.EPOLLNVAL = 0x00000020 -M.EPOLLRDNORM = 0x00000040 -M.EPOLLRDBAND = 0x00000080 -M.EPOLLWRNORM = 0x00000100 -M.EPOLLWRBAND = 0x00000200 -M.EPOLLMSG = 0x00000400 -M.EPOLLRDHUP = 0x00002000 - -M.EPOLLEXCLUSIVE = bit.lshift(1, 28) -M.EPOLLWAKEUP = bit.lshift(1, 29) -M.EPOLLONESHOT = bit.lshift(1, 30) -M.EPOLLET = bit.lshift(1, 31) - -local EPOLL_CTL_ADD = 1 -local EPOLL_CTL_DEL = 2 -local EPOLL_CTL_MOD = 3 - - --- Adjust helper functions based on the architecture: -local get_event -local set_event -local get_data -local set_data - -if ARCH == 'x64' or ARCH == 'x86' then - get_event = function(ev) - return ffi.cast("uint32_t*", ev.raw)[0] - end - set_event = function(ev, value) - ffi.cast("uint32_t*", ev.raw)[0] = value - end - get_data = function(ev) - return ffi.cast("uint64_t*", ev.raw + 4)[0] - end - set_data = function(ev, value) - ffi.cast("uint64_t*", ev.raw + 4)[0] = value - end -elseif ARCH == 'mips' or ARCH == 'arm64' or ARCH == 'mipsel' then - get_event = function(ev) - return ev.events - end - set_event = function(ev, value) - ev.events = value - end - get_data = function(ev) - return ev.data - end - set_data = function(ev, value) - ev.data = value - end -else - error(ARCH.." architecture not specified") -end - --- Returns an epoll file descriptor. -function M.epoll_create() - return wrap_error(ffi.C.epoll_create(1)) -end - --- Register eventmask of a file descriptor onto epoll file descriptor. -function M.epoll_register(epfd, fd, eventmask) - local event = ffi.new("struct epoll_event") - set_event(event, eventmask) - set_data(event, fd) - return wrap_error(ffi.C.epoll_ctl(epfd, EPOLL_CTL_ADD, fd, event)) -end - --- Modify eventmask of a file descriptor. -function M.epoll_modify(epfd, fd, eventmask) - local event = ffi.new("struct epoll_event") - set_event(event, eventmask) - set_data(event, fd) - return wrap_error(ffi.C.epoll_ctl(epfd, EPOLL_CTL_MOD, fd, event)) -end - --- Remove a registered file descriptor from the epoll file descriptor. -function M.epoll_unregister(epfd, fd) - return wrap_error(ffi.C.epoll_ctl(epfd, EPOLL_CTL_DEL, fd, nil)) -end - --- Wait for events. -function M.epoll_wait(epfd, timeout, max_events) - local events = ffi.new("struct epoll_event[?]", max_events) - local num_events = ffi.C.epoll_wait(epfd, events, max_events, timeout) - if num_events == -1 then - return nil, ffi.string(ffi.C.strerror(ffi.errno())) - end - - -- Create a table to hold the resulting events - local res = {} - - -- Loop over the events, inserting them into the table with their fd as the key - for i = 0, num_events - 1 do - local fd = assert(ffi.tonumber(get_data(events[i]))) - local event = assert(ffi.tonumber(get_event(events[i]))) - res[fd] = event - end - - return res, num_events -end - --- Close epoll file descriptor. -function M.epoll_close(epfd) - return wrap_error(ffi.C.close(epfd)) -end - - -------------------------------------------------------------------------------- --- FFI C structure functions (for efficiency) - -M.ffi.typeof = ffi.typeof -M.ffi.sizeof = ffi.sizeof - -ffi.cdef [[ - ssize_t write(int fildes, const void *buf, size_t nbytes); - ssize_t read(int fildes, void *buf, size_t nbytes); - int memcmp(const void *s1, const void *s2, size_t n); -]] - -function M.ffi.write(fildes, buf, nbytes) - return wrap_error(ffi.tonumber(ffi.C.write(fildes, buf, nbytes))) -end - -function M.ffi.read(fildes, buf, nbytes) - return wrap_error(ffi.tonumber(ffi.C.read(fildes, buf, nbytes))) -end - -function M.ffi.memcmp(obj1, obj2, nbytes) - return ffi.tonumber(ffi.C.memcmp(obj1, obj2, nbytes)) -end - --- Explicitly load the pthread library - -local pthread_names = { - "pthread", - "libpthread.so.0" -} - -local libpthread = nil - -for _, v in ipairs(pthread_names) do - local success - success, libpthread = pcall(ffi.load, v) - if success then break end -end - -if not libpthread then error("libpthread not found") end - -ffi.cdef[[ -typedef struct { - uint32_t ssi_signo; /* Signal number */ - int32_t ssi_errno; /* Error number (unused) */ - int32_t ssi_code; /* Signal code */ - uint32_t ssi_pid; /* PID of sender */ - uint32_t ssi_uid; /* Real UID of sender */ - int32_t ssi_fd; /* File descriptor (SIGIO) */ - uint32_t ssi_tid; /* Kernel timer ID (POSIX timers) */ - uint32_t ssi_band; /* Band event (SIGIO) */ - uint32_t ssi_overrun; /* POSIX timer overrun count */ - uint32_t ssi_trapno; /* Trap number that caused signal */ - int32_t ssi_status; /* Exit status or signal (SIGCHLD) */ - int32_t ssi_int; /* Integer sent by sigqueue(3) */ - uint64_t ssi_ptr; /* Pointer sent by sigqueue(3) */ - uint64_t ssi_utime; /* User CPU time consumed (SIGCHLD) */ - uint64_t ssi_stime; /* System CPU time consumed (SIGCHLD) */ - uint64_t ssi_addr; /* Address that generated signal (for hardware-generated signals) */ - uint16_t ssi_addr_lsb; /* Least significant bit of address (SIGBUS; since Linux 2.6.37) */ - uint16_t __pad2; - int32_t ssi_syscall; - uint64_t ssi_call_addr; - uint32_t ssi_arch; - uint8_t pad[28]; /* Pad size to 128 bytes */ -} signalfd_siginfo; - -typedef struct { - unsigned long int __val[1024 / (8 * sizeof (unsigned long int))]; -} __sigset_t; - -typedef __sigset_t sigset_t; - -int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); -int sigemptyset(sigset_t *set); -int sigaddset(sigset_t *set, int signum); -int signalfd(int fd, const sigset_t *mask, int flags); -]] - -if ARCH == "mips" or ARCH == "mipsel" then - M.SIG_BLOCK = 1 - M.SIG_UNBLOCK = 2 - M.SIG_SETMASK = 3 -elseif ARCH == "x64" or ARCH == "arm64" or ARCH == "x86" then - M.SIG_BLOCK = 0 - M.SIG_UNBLOCK = 1 - M.SIG_SETMASK = 2 -end - -function M.sigemptyset(set) return wrap_error(ffi.C.sigemptyset(set)) end -function M.sigaddset(set, signum) return wrap_error(ffi.C.sigaddset(set, signum)) end -function M.signalfd(fd, mask, flags) return wrap_error(ffi.C.signalfd(fd, mask, flags)) end - -function M.pthread_sigmask(how, set, oldset) return wrap_error(libpthread.pthread_sigmask(how, set, oldset)) end - -function M.new_sigset() return ffi.new("sigset_t") end -function M.new_fdsi() return ffi.new("signalfd_siginfo"), ffi.sizeof("signalfd_siginfo") end - --- Define syscall and pid_t -ffi.cdef[[ -long syscall(long number, ...); -typedef int pid_t; -typedef unsigned int uint; -]] - -local SYS_pidfd_open = 434 -- Good for (almost) all our platforms -if ARCH == "mips" or ARCH == "mipsel" then - SYS_pidfd_open = 4000 + 434 -- See https://www.linux-mips.org/wiki/Syscall -end - --- Function to open a pidfd -function M.pidfd_open(pid, flags) - pid = ffi.new("pid_t", pid) -- Explicitly cast pid to pid_t - flags = ffi.new("uint", flags) -- Explicitly cast flgas to uint - return wrap_error(ffi.tonumber(ffi.C.syscall(SYS_pidfd_open, pid, flags))) -end - -return M diff --git a/src/fibers/waitgroup.lua b/src/fibers/waitgroup.lua deleted file mode 100644 index bec5bb2b..00000000 --- a/src/fibers/waitgroup.lua +++ /dev/null @@ -1,45 +0,0 @@ --- waitgroup.lua -local op = require 'fibers.op' -local cond = require 'fibers.cond' - -local Waitgroup = {} -Waitgroup.__index = Waitgroup - -local function new() - local wg = setmetatable({ _counter = 0, _cond = cond.new() }, Waitgroup) - return wg -end - -function Waitgroup:add(count) - self._counter = self._counter + count - if self._counter < 0 then - error("waitgroup counter goes negative") - elseif self._counter == 0 then - self._cond:signal() - end -end - -function Waitgroup:done() - self:add(-1) -end - -function Waitgroup:wait_op() - local function try() - return self._counter == 0 - end - local function block(suspension, wrap_fn) - if self._counter > 0 then - -- Add suspension to the condition variable's wait queue. - self._cond.waitq[#self._cond.waitq + 1] = { suspension = suspension, wrap = wrap_fn } - end - end - return op.new_base_op(nil, try, block) -end - -function Waitgroup:wait() - self:wait_op():perform() -end - -return { - new = new -} diff --git a/src/lua-bus b/src/lua-bus new file mode 160000 index 00000000..87b2d6fb --- /dev/null +++ b/src/lua-bus @@ -0,0 +1 @@ +Subproject commit 87b2d6fb3c68649496a64df57530460e6bac136f diff --git a/src/lua-fibers b/src/lua-fibers new file mode 160000 index 00000000..28f78bbf --- /dev/null +++ b/src/lua-fibers @@ -0,0 +1 @@ +Subproject commit 28f78bbfb855eeac9521b87c8e2d3cc92453c0dc diff --git a/src/lua-trie b/src/lua-trie new file mode 160000 index 00000000..2896251e --- /dev/null +++ b/src/lua-trie @@ -0,0 +1 @@ +Subproject commit 2896251e96a5ee58f561041976d9ab13281d243b diff --git a/src/service.lua b/src/service.lua new file mode 100644 index 00000000..43d086de --- /dev/null +++ b/src/service.lua @@ -0,0 +1,147 @@ +local context = require "fibers.context" +local fiber = require 'fibers.fiber' + +local FiberRegister = {} +FiberRegister.__index = FiberRegister + +function FiberRegister.new() + return setmetatable({size = 0, fibers = {}}, FiberRegister) +end + +function FiberRegister:push(name, status) + if self.fibers[name] == nil then + self.size = self.size + 1 + end + self.fibers[name] = status +end + +function FiberRegister:pop(name) + if self.fibers[name] ~= nil then + self.size = self.size - 1 + self.fibers[name] = nil + end +end + +function FiberRegister:is_empty() + return self.size == 0 +end + +-- spawns a service, which involves creation of a child context, +-- bus connection and a shutdown fiber +-- (which should be adapted for update, reboot etc when system service is more built out) +local function spawn(service, bus, ctx) + local bus_connection = bus:connect() + local child_ctx = context.with_cancel(ctx) + child_ctx.values.service_name = service.name + + local health_topic = child_ctx.values.service_name..'/health' + + -- Non-blocking start function + service:start(bus_connection, child_ctx) + bus_connection:publish({ + topic = health_topic, + payload = { + name = child_ctx.values.service_name, + state = 'active' + }, + retained = true + }) + + -- Creates a shutdown fiber to handle any shutdown messages from the system service + -- Tracks all long running fibers under the current service before reporting an end to the service + fiber.spawn(function () + local system_events_sub = bus_connection:subscribe(child_ctx.values.service_name..'/control/shutdown') + local shutdown_event = system_events_sub:next_msg() + system_events_sub:unsubscribe() + + local service_fibers_status_sub = bus_connection:subscribe(child_ctx.values.service_name..'/health/fibers/+') + local active_fibers = FiberRegister.new() + + local fibers_checked = false + + -- is there a better way to do this? should be one loop or two for better readability?: + -- 1. No fibers + -- 2. All fibers already completed + -- 3. Some fibers finished + -- 4. All fibers finished + -- 5. (opt) Could a fiber spin up during the shutdown and not be detected as ongoing? + + -- collect what fibers are active or initialising + while true do + local message = service_fibers_status_sub:next_msg_op():perform_alt(function () fibers_checked = true end) + if fibers_checked then break end + if message.payload ~= '' then + if message.payload.state == 'disabled' then + active_fibers:pop(message.payload.name) + else + active_fibers:push(message.payload.name, message.payload.state) + end + end + end + + -- let every fiber know to end + child_ctx:cancel(shutdown_event.payload.cause) + + -- wait for fibers to close + while not active_fibers:is_empty() do + local message = service_fibers_status_sub:next_msg() + if message.payload ~= '' then + if message.payload.state == 'disabled' then + active_fibers:pop(message.payload.name) + else + active_fibers:push(message.payload.name, message.payload.state) + end + end + end + + bus_connection:publish({ + topic = health_topic, + payload = { + name = child_ctx.values.service_name, + state = 'disabled' + }, + retained = true + }) + end) +end + +local function spawn_fiber(name, bus_connection, ctx, fn) + local child_ctx = context.with_cancel(ctx) + child_ctx.values.fiber_name = name + + local fiber_topic = child_ctx.values.service_name..'/health/fibers/'..child_ctx.values.fiber_name + + bus_connection:publish({ + topic = fiber_topic, + payload = { + name = child_ctx.values.fiber_name, + state = 'initialising' + }, + retained = true + }) + + fiber.spawn(function () + bus_connection:publish({ + topic = fiber_topic, + payload = { + name = child_ctx.values.fiber_name, + state = 'active' + }, + retained = true + }) + fn(child_ctx) + bus_connection:publish({ + topic = fiber_topic, + payload = { + name = child_ctx.values.fiber_name, + state = 'disabled' + }, + retained = true + }) + end) +end + +return { + spawn = spawn, + spawn_fiber = spawn_fiber +} \ No newline at end of file diff --git a/src/trie.lua b/src/trie.lua deleted file mode 100644 index 0a344efd..00000000 --- a/src/trie.lua +++ /dev/null @@ -1,161 +0,0 @@ ---- Trie implementation with single and multi-level wildcard support. --- @module Trie - -local Trie = {} -Trie.__index = Trie - ---- Create a new Trie instance. --- @tparam string single_wild Single-level wildcard. --- @tparam string multi_wild Multi-level wildcard. It can only be placed at the end of a key. --- @tparam[opt=""] string separator The separator used to split the key. Default is characterwise splitting. --- @treturn table Returns a new Trie instance. -local function new(single_wild, multi_wild, separator) - local trie = setmetatable({ - root = {children = {}}, - single_wild = single_wild, - multi_wild = multi_wild, - separator = separator or "" - }, Trie) - return trie -end - -local function split(str, separator) - local result = {} - local pattern = separator=="" and "." or "[^"..separator.."]+" - for part in string.gmatch(str, pattern) do - table.insert(result, part) - end - return result -end - ---- Insert a key-value pair into the Trie. --- @tparam string key The key to be inserted. --- @param value The value associated with the key. --- @treturn boolean Indicates whether the insertion was successful or not. --- @treturn[opt] string Error message in case of failure. -function Trie:insert(key, value) - local node = self.root - key = split(key, self.separator) - for i, part in ipairs(key) do - if part == self.multi_wild and i ~= #key then - return false, "error: multi-level wildcard '"..self.multi_wild.."' permitted only at the end of insert key." - end - node.children[part] = node.children[part] or {children = {}} - node = node.children[part] - end - node.value = value - return true, nil -end - ---- Retrieve a value based on the given key from the Trie. --- @tparam string key The key to retrieve the value for. --- @return The value associated with the given key or nil if not found. -function Trie:retrieve(key) - local node = self.root - key = split(key, self.separator) - for i, part in ipairs(key) do - if part == self.multi_wild and i ~= #key then - return false, "error: multi-level wildcard '"..self.multi_wild.."' permitted only at the end of retrieval key." - end - if not node.children[part] then return nil end - node = node.children[part] - end - return node.value, nil -end - -local function collect_all(startNode, startKeypart, matches, separator) - local stack = {{node=startNode, keypart=startKeypart}} - - while #stack > 0 do - local current = table.remove(stack) - local node, keypart = current.node, current.keypart - - if node.value then - table.insert(matches, {['key']=keypart..current.keypart, ['value']=node.value}) - end - - for k, v in pairs(node.children) do - -- Push child node to the stack - table.insert(stack, {node=v, keypart=keypart .. k .. separator}) - end - end -end - ---- Matches the given key against the Trie and returns all matches. --- The function supports single and multi-level wildcards in the key. --- @tparam string key The key to match. --- @treturn table A table containing all matching key-value pairs. -function Trie:match(key) - local matches = {} - - local parts = split(key, self.separator) - local stack = {{node=self.root, i=1, keypart=""}} - - while #stack > 0 do - local current = table.remove(stack) - local node, i, keypart = current.node, current.i, current.keypart - - if self.multi_wild and parts[i] == self.multi_wild then - collect_all(node, keypart, matches, self.separator) - elseif self.single_wild and parts[i] == self.single_wild then - for k, child_node in pairs(node.children) do - if i == #parts and child_node.value then - table.insert(matches, {['key']=keypart..k, ['value']=child_node.value}) - elseif i < #parts then - table.insert(stack, {node=child_node, i=i+1, keypart=keypart..k..self.separator}) - end - end - else - local key_node = node.children[parts[i]] - if key_node then - if i == #parts and key_node.value then - table.insert(matches, {['key']=keypart..parts[i], ['value']=key_node.value}) - else - table.insert(stack, {node=key_node, i=i+1, keypart=keypart..parts[i]..self.separator}) - end - end - local single_wild_node = node.children[self.single_wild] - if single_wild_node then - if i == #parts and single_wild_node.value then - table.insert(matches, {['key']=keypart..self.single_wild, ['value']=single_wild_node.value}) - else - table.insert(stack, {node=single_wild_node, i=i+1, keypart=keypart..self.single_wild..self.separator}) - end - end - local multi_wild_node = node.children[self.multi_wild] - if multi_wild_node then - table.insert(matches, {['key']=keypart..self.multi_wild, ['value']=multi_wild_node.value}) - end - end - end - - return matches -end - ---- Deletes a key from the Trie. --- @tparam string key The key to delete. --- @treturn boolean Indicates whether the deletion was successful or not. -function Trie:delete(key) - local parentStack = {self.root} - local node = self.root - key = split(key, self.separator) - for i, part in ipairs(key) do - if not node.children[part] then return false end - table.insert(parentStack, node.children[part]) - node = node.children[part] - end - if not node.value then return false end - node.value = nil - - for i = #key, 1, -1 do - if node.value or next(node.children) then break end - node = parentStack[#parentStack] - table.remove(parentStack) - node.children[key[i]] = nil - end - return true -end - -return { - new = new -} diff --git a/src/uuid.lua b/src/uuid.lua deleted file mode 100644 index f9c68dab..00000000 --- a/src/uuid.lua +++ /dev/null @@ -1,216 +0,0 @@ ---------------------------------------------------------------------------------------- --- Copyright 2012 Rackspace (original), 2013-2021 Thijs Schreijer (modifications) --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS-IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- see http://www.ietf.org/rfc/rfc4122.txt --- --- Note that this is not a true version 4 (random) UUID. Since `os.time()` precision is only 1 second, it would be hard --- to guarantee spacial uniqueness when two hosts generate a uuid after being seeded during the same second. This --- is solved by using the node field from a version 1 UUID. It represents the mac address. --- --- 28-apr-2013 modified by Thijs Schreijer from the original [Rackspace code](https://github.com/kans/zirgo/blob/807250b1af6725bad4776c931c89a784c1e34db2/util/uuid.lua) as a generic Lua module. --- Regarding the above mention on `os.time()`; the modifications use the `socket.gettime()` function from LuaSocket --- if available and hence reduce that problem (provided LuaSocket has been loaded before uuid). --- --- **Important:** the random seed is a global piece of data. Hence setting it is --- an application level responsibility, libraries should never set it! --- --- See this issue; [https://github.com/Kong/kong/issues/478](https://github.com/Kong/kong/issues/478) --- It demonstrates the problem of using time as a random seed. Specifically when used from multiple processes. --- So make sure to seed only once, application wide. And to not have multiple processes do that --- simultaneously. - - -local M = {} -local math = require('math') -local os = require('os') -local string = require('string') - -local bitsize = 32 -- bitsize assumed for Lua VM. See randomseed function below. -local lua_version = tonumber(_VERSION:match("%d%.*%d*")) -- grab Lua version used - -local MATRIX_AND = {{0,0},{0,1} } -local MATRIX_OR = {{0,1},{1,1}} -local HEXES = '0123456789abcdef' - -local math_floor = math.floor -local math_random = math.random -local math_abs = math.abs -local string_sub = string.sub -local to_number = tonumber -local assert = assert -local type = type - --- performs the bitwise operation specified by truth matrix on two numbers. -local function BITWISE(x, y, matrix) - local z = 0 - local pow = 1 - while x > 0 or y > 0 do - z = z + (matrix[x%2+1][y%2+1] * pow) - pow = pow * 2 - x = math_floor(x/2) - y = math_floor(y/2) - end - return z -end - -local function INT2HEX(x) - local s,base = '',16 - local d - while x > 0 do - d = x % base + 1 - x = math_floor(x/base) - s = string_sub(HEXES, d, d)..s - end - while #s < 2 do s = "0" .. s end - return s -end - ----------------------------------------------------------------------------- --- Creates a new uuid. Either provide a unique hex string, or make sure the --- random seed is properly set. The module table itself is a shortcut to this --- function, so `my_uuid = uuid.new()` equals `my_uuid = uuid()`. --- --- For proper use there are 3 options; --- --- 1. first require `luasocket`, then call `uuid.seed()`, and request a uuid using no --- parameter, eg. `my_uuid = uuid()` --- 2. use `uuid` without `luasocket`, set a random seed using `uuid.randomseed(some_good_seed)`, --- and request a uuid using no parameter, eg. `my_uuid = uuid()` --- 3. use `uuid` without `luasocket`, and request a uuid using an unique hex string, --- eg. `my_uuid = uuid(my_networkcard_macaddress)` --- --- @return a properly formatted uuid string --- @param hwaddr (optional) string containing a unique hex value (e.g.: `00:0c:29:69:41:c6`), to be used to compensate for the lesser `math_random()` function. Use a mac address for solid results. If omitted, a fully randomized uuid will be generated, but then you must ensure that the random seed is set properly! --- @usage --- local uuid = require("uuid") --- print("here's a new uuid: ",uuid()) -function M.new(hwaddr) - -- bytes are treated as 8bit unsigned bytes. - local bytes = { - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255), - math_random(0, 255) - } - - if hwaddr then - assert(type(hwaddr)=="string", "Expected hex string, got "..type(hwaddr)) - -- Cleanup provided string, assume mac address, so start from back and cleanup until we've got 12 characters - local i,str = #hwaddr, hwaddr - hwaddr = "" - while i>0 and #hwaddr<12 do - local c = str:sub(i,i):lower() - if HEXES:find(c, 1, true) then - -- valid HEX character, so append it - hwaddr = c..hwaddr - end - i = i - 1 - end - assert(#hwaddr == 12, "Provided string did not contain at least 12 hex characters, retrieved '"..hwaddr.."' from '"..str.."'") - - -- no split() in lua. :( - bytes[11] = to_number(hwaddr:sub(1, 2), 16) - bytes[12] = to_number(hwaddr:sub(3, 4), 16) - bytes[13] = to_number(hwaddr:sub(5, 6), 16) - bytes[14] = to_number(hwaddr:sub(7, 8), 16) - bytes[15] = to_number(hwaddr:sub(9, 10), 16) - bytes[16] = to_number(hwaddr:sub(11, 12), 16) - end - - -- set the version - bytes[7] = BITWISE(bytes[7], 0x0f, MATRIX_AND) - bytes[7] = BITWISE(bytes[7], 0x40, MATRIX_OR) - -- set the variant - bytes[9] = BITWISE(bytes[9], 0x3f, MATRIX_AND) - bytes[9] = BITWISE(bytes[9], 0x80, MATRIX_OR) - return INT2HEX(bytes[1])..INT2HEX(bytes[2])..INT2HEX(bytes[3])..INT2HEX(bytes[4]).."-".. - INT2HEX(bytes[5])..INT2HEX(bytes[6]).."-".. - INT2HEX(bytes[7])..INT2HEX(bytes[8]).."-".. - INT2HEX(bytes[9])..INT2HEX(bytes[10]).."-".. - INT2HEX(bytes[11])..INT2HEX(bytes[12])..INT2HEX(bytes[13])..INT2HEX(bytes[14])..INT2HEX(bytes[15])..INT2HEX(bytes[16]) -end - ----------------------------------------------------------------------------- --- Improved randomseed function. --- Lua 5.1 and 5.2 both truncate the seed given if it exceeds the integer --- range. If this happens, the seed will be 0 or 1 and all randomness will --- be gone (each application run will generate the same sequence of random --- numbers in that case). This improved version drops the most significant --- bits in those cases to get the seed within the proper range again. --- @param seed the random seed to set (integer from 0 - 2^32, negative values will be made positive) --- @return the (potentially modified) seed used --- @usage --- local socket = require("socket") -- gettime() has higher precision than os.time() --- local uuid = require("uuid") --- -- see also example at uuid.seed() --- uuid.randomseed(socket.gettime()*10000) --- print("here's a new uuid: ",uuid()) -function M.randomseed(seed) - seed = math_floor(math_abs(seed)) - if seed >= (2^bitsize) then - -- integer overflow, so reduce to prevent a bad seed - seed = seed - math_floor(seed / 2^bitsize) * (2^bitsize) - end - if lua_version < 5.2 then - -- 5.1 uses (incorrect) signed int - math.randomseed(seed - 2^(bitsize-1)) - else - -- 5.2 uses (correct) unsigned int - math.randomseed(seed) - end - return seed -end - ----------------------------------------------------------------------------- --- Seeds the random generator. --- It does so in 3 possible ways; --- --- 1. if in ngx_lua, use `ngx.time() + ngx.worker.pid()` to ensure a unique seed --- for each worker. It should ideally be called from the `init_worker` context. --- 2. use luasocket `gettime()` function, but it only does so when LuaSocket --- has been required already. --- 3. use `os.time()`: this only offers resolution to one second (used when --- LuaSocket hasn't been loaded) --- --- **Important:** the random seed is a global piece of data. Hence setting it is --- an application level responsibility, libraries should never set it! --- @usage --- local socket = require("socket") -- gettime() has higher precision than os.time() --- -- LuaSocket loaded, so below line does the same as the example from randomseed() --- uuid.seed() --- print("here's a new uuid: ",uuid()) -function M.seed() - if _G.ngx ~= nil then - return M.randomseed(ngx.time() + ngx.worker.pid()) - elseif package.loaded["socket"] and package.loaded["socket"].gettime then - return M.randomseed(package.loaded["socket"].gettime()*10000) - else - return M.randomseed(os.time()) - end -end - -return setmetatable( M, { __call = function(self, hwaddr) return self.new(hwaddr) end} ) \ No newline at end of file diff --git a/tests/test.lua b/tests/test.lua new file mode 100644 index 00000000..4b90356f --- /dev/null +++ b/tests/test.lua @@ -0,0 +1,15 @@ +package.path = "../src/lua-fibers/?.lua;" -- fibers submodule src + .. "../src/lua-trie/src/?.lua;" -- trie submodule src + .. "../src/lua-bus/src/?.lua;" -- bus submodule src + .. "../src/?.lua;" + .. package.path + .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" + +local tests = { + "service", + "submodules" +} + +for _, test in ipairs(tests) do + dofile("test_" .. test .. ".lua") +end diff --git a/tests/test_service.lua b/tests/test_service.lua new file mode 100644 index 00000000..9fe66180 --- /dev/null +++ b/tests/test_service.lua @@ -0,0 +1,297 @@ +local service = require "service" +local sleep = require "fibers.sleep" +local fiber = require "fibers.fiber" +local bus_pkg = require "bus" +local context = require "fibers.context" + +local function test_service_states() + -- make a fake service + local dummy_service = {} + dummy_service.__index = dummy_service + + dummy_service.name = 'dummy-service' + + function dummy_service:start(bus_conn, ctx) + --nothing + end + + local bus = bus_pkg.new({q_len=10, m_wild='#', s_wild='+', sep="/"}) + local bus_connection = bus:connect() + + local bg_ctx = context.background() + local ctx = context.with_cancel(bg_ctx) + + local expected_states = {'active', 'disabled'} + local states = {} + + -- fiber to listen for service health updates + fiber.spawn(function () + local service_health_sub = bus_connection:subscribe('dummy-service/health') + while not ctx:err() do + local msg = service_health_sub:next_msg() + if msg.payload ~= '' then + table.insert(states, msg.payload.state) + end + end + end) + + service.spawn(dummy_service, bus, ctx) + + -- send shutdown signal to service + bus_connection:publish({ + topic = 'dummy-service/control/shutdown', + payload = { + cause = "shutdown-service" + }, + retained = true + }) + + -- a little time for messages to propagate + sleep.sleep(0.1) + + ctx:cancel('shutdown') + + -- check service states + for i=1, 2 do + assert(states[i] == expected_states[i], "service states was "..(states[i] or 'nil').." expected "..expected_states[i]) + end + + assert(#states == 2, 'service should have gone through 2 states, '..#states..' detected') +end + +local function test_fiber_states() + local dummy_service = {} + dummy_service.__index = dummy_service + + dummy_service.name = 'dummy-service' + + -- service spins up a fiber that waits for 0.1 seconds before exiting + function dummy_service:start(bus_conn, ctx) + service.spawn_fiber('sleep-fiber', bus_conn, ctx, function (fctx) + sleep.sleep(0.1) + end) + end + + local bus = bus_pkg.new({q_len=10, m_wild='#', s_wild='+', sep="/"}) + local bus_connection = bus:connect() + + local bg_ctx = context.background() + local ctx = context.with_cancel(bg_ctx) + + local expected_states = {'initialising', 'active', 'disabled'} + local states = {} + + -- fiber to listen for fiber health updates + fiber.spawn(function () + local fiber_health_sub = bus_connection:subscribe('dummy-service/health/fibers/sleep-fiber') + while not ctx:err() do + local msg = fiber_health_sub:next_msg() + if msg.payload ~= '' then + table.insert(states, msg.payload.state) + end + end + end) + + -- let listening fiber spin up + fiber.yield() + + service.spawn(dummy_service, bus, ctx) + + -- send shutdown signal to service + bus_connection:publish({ + topic = 'dummy-service/control/shutdown', + payload = { + cause = "shutdown-service" + }, + retained = true + }) + + -- let shutdown signal propagate + sleep.sleep(0.2) + + ctx:cancel('shutdown') + + -- check fiber states + for i=1, 3 do + assert(states[i] == expected_states[i], "service states was "..(states[i] or 'nil').." expected "..expected_states[i]) + end + + assert(#states == 3, 'service should have gone through 3 states, '..#states..' detected') +end + +local function check_fiber_state(bus_conn, service_name, fiber_name) + local sub = bus_conn:subscribe(service_name..'/health/fibers/'..fiber_name) + local state_msg = sub:next_msg() + sub:unsubscribe() + return state_msg.payload.state +end + +local function check_service_state(bus_conn, service_name) + local sub = bus_conn:subscribe(service_name..'/health') + local state_msg = sub:next_msg() + sub:unsubscribe() + return state_msg.payload.state +end + +local function test_blocked_shutdown() + local dummy_service = {} + dummy_service.__index = dummy_service + + dummy_service.name = 'dummy-service' + + -- service will create a fiber that will run endlessly, therefore + -- blocking the shutdown of the service + function dummy_service:start(bus_conn, ctx) + service.spawn_fiber('stuck-loop', bus_conn, ctx, function (fctx) + local i = 0 + while true do + i = i + 1 + fiber.yield() + end + end) + end + + local bus = bus_pkg.new({q_len=10, m_wild='#', s_wild='+', sep="/"}) + local bus_connection = bus:connect() + + local bg_ctx = context.background() + local ctx = context.with_cancel(bg_ctx) + + service.spawn(dummy_service, bus, ctx) + + -- send shutdown signal to service + bus_connection:publish({ + topic = 'dummy-service/control/shutdown', + payload = { + cause = "shutdown-service" + }, + retained = true + }) + + -- wait for messages to propegate + sleep.sleep(0.1) + + local fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'stuck-loop') + assert(fiber_state == 'active', 'stuck-loop should be active but is '..fiber_state) + + local service_state = check_service_state(bus_connection, 'dummy-service') + assert(service_state == 'active', 'dummy-service should be active but is '..service_state) +end + +local function test_timed_shutdown() + local dummy_service = {} + dummy_service.__index = dummy_service + + dummy_service.name = 'dummy-service' + + -- service will spin up + function dummy_service:start(bus_conn, ctx) + service.spawn_fiber('time-dependant', bus_conn, ctx, function (fctx) + sleep.sleep(0.2) + end) + end + + local bus = bus_pkg.new({q_len=10, m_wild='#', s_wild='+', sep="/"}) + local bus_connection = bus:connect() + + local bg_ctx = context.background() + local ctx = context.with_cancel(bg_ctx) + + service.spawn(dummy_service, bus, ctx) + + -- send shutdown signal to service + bus_connection:publish({ + topic = 'dummy-service/control/shutdown', + payload = { + cause = "shutdown-service" + }, + retained = true + }) + + -- wait for service and fiber to spin up + sleep.sleep(0.1) + + local fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'time-dependant') + assert(fiber_state == 'active', 'time-dependant should be active but is '..fiber_state) + + local service_state = check_service_state(bus_connection, 'dummy-service') + assert(service_state == 'active', 'dummy-service should be active but is '..service_state) + + -- wait for fiber to finish + sleep.sleep(0.3) + + fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'time-dependant') + assert(fiber_state == 'disabled', 'time-dependant should be disabled but is '..fiber_state) + + service_state = check_service_state(bus_connection, 'dummy-service') + assert(service_state == 'disabled', 'dummy-service should be disabled but is '..service_state) +end + +local function test_context_shutdown() + local dummy_service = {} + dummy_service.__index = dummy_service + + dummy_service.name = 'dummy-service' + + -- service creates a fiber that requires a context cancellation in order to exit + function dummy_service:start(bus_conn, sctx) + service.spawn_fiber('ctx-dependant', bus_conn, sctx, function (fctx) + local i = 0 + while not fctx:err() do + i = i + 1 + fiber.yield() + end + end) + end + + local bus = bus_pkg.new({q_len=10, m_wild='#', s_wild='+', sep="/"}) + local bus_connection = bus:connect() + + local bg_ctx = context.background() + local ctx = context.with_cancel(bg_ctx) + + service.spawn(dummy_service, bus, ctx) + + local fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'ctx-dependant') + assert(fiber_state == 'initialising', 'ctx-dependant should be initialising but is '..fiber_state) + + -- let fiber spin up + sleep.sleep(0.1) + + fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'ctx-dependant') + assert(fiber_state == 'active', 'ctx-dependant should be active but is '..fiber_state) + + local service_state = check_service_state(bus_connection, 'dummy-service') + assert(service_state == 'active', 'dummy-service should be active but is '..service_state) + + -- send shutdown signal to service + bus_connection:publish({ + topic = 'dummy-service/control/shutdown', + payload = { + cause = "shutdown-service" + }, + retained = true + }) + + -- wait for messages to propegate + sleep.sleep(0.1) + + fiber_state = check_fiber_state(bus_connection, 'dummy-service', 'ctx-dependant') + assert(fiber_state == 'disabled', 'ctx-dependant should be disabled but is '..fiber_state) + + service_state = check_service_state(bus_connection, 'dummy-service') + assert(service_state == 'disabled', 'dummy-service should be disabled but is '..service_state) +end + +fiber.spawn(function () + test_service_states() + test_fiber_states() + test_blocked_shutdown() + test_timed_shutdown() + test_context_shutdown() + fiber.stop() +end) + +print("starting service tests") +fiber.main() +print("tests complete") diff --git a/tests/test_submodules.lua b/tests/test_submodules.lua new file mode 100644 index 00000000..53fe3621 --- /dev/null +++ b/tests/test_submodules.lua @@ -0,0 +1,15 @@ +package.path = "../src/lua-fibers/?.lua;" + .. "../src/lua-trie/src/?.lua;" + .. "../src/lua-bus/src/?.lua;" + .. "../src/?.lua;" + .. package.path + .. ";/usr/lib/lua/?.lua;/usr/lib/lua/?/init.lua" + +print("starting submodule tests") + +assert(require 'fibers.fiber') +assert(require 'trie') +assert(require 'uuid') +assert(require 'bus') + +print("tests complete")