From 5c8aee76b2b80a0984d18d51f51eb0aff4703c83 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 21 Jul 2026 19:12:47 +0300 Subject: [PATCH 01/32] new tool: search photo in gallery --- src/Worker.cpp | 32 +++-- src/config.cpp | 4 +- src/config.h | 2 +- src/main.cpp | 2 + src/tools/search_photo_in_gallery.cpp | 122 +++++++++++++++++++ src/tools/search_photo_in_gallery.h | 7 ++ src/tools/send_telegram_message.cpp | 8 +- src/tools/take_photo.cpp | 6 +- tests/tools/search_photo_in_gallery_test.cpp | 28 +++++ 9 files changed, 185 insertions(+), 26 deletions(-) create mode 100644 src/tools/search_photo_in_gallery.cpp create mode 100644 src/tools/search_photo_in_gallery.h create mode 100644 tests/tools/search_photo_in_gallery_test.cpp diff --git a/src/Worker.cpp b/src/Worker.cpp index 8168060..901a0ea 100644 --- a/src/Worker.cpp +++ b/src/Worker.cpp @@ -41,24 +41,22 @@ AFuture> contextEmbedding(ALogger& logger, IOpenAIChat& op [[nodiscard]] static AFuture<> processRandomlyGoSleep(ALogger& logger, bool& wakeUp) { - if (config().randomlyGoSleep) { - if (std::uniform_real_distribution(0.0, 1.0)(gRandomEngine) < 0.01) { - // 1. randomly go afk is humane - // 2. reduce resource usage: - // - less conversations would be made - // - in case of group chats and telegram channels, messages would be processed in batches - const auto duration = std::chrono::minutes(std::uniform_int_distribution(15, 120)(gRandomEngine)); - logger.info(LOG_TAG) - << "Going to sleep for " << std::chrono::duration_cast(duration).count() << " minutes"; - wakeUp = false; - for (int i = 0; i < std::chrono::duration_cast(duration).count(); ++i) { - // костыль ну да сойдёт - if (wakeUp) { - logger.info(LOG_TAG) << "Early wake up"; - break; - } - co_await AThread::asyncSleep(1s); + if (std::uniform_real_distribution(0.0, 1.0)(gRandomEngine) < config().randomlyGoSleepChance) { + // 1. randomly go afk is humane + // 2. reduce resource usage: + // - less conversations would be made + // - in case of group chats and telegram channels, messages would be processed in batches + const auto duration = std::chrono::minutes(std::uniform_int_distribution(15, 120)(gRandomEngine)); + logger.info(LOG_TAG) + << "Going to sleep for " << std::chrono::duration_cast(duration).count() << " minutes"; + wakeUp = false; + for (int i = 0; i < std::chrono::duration_cast(duration).count(); ++i) { + // костыль ну да сойдёт + if (wakeUp) { + logger.info(LOG_TAG) << "Early wake up"; + break; } + co_await AThread::asyncSleep(1s); } } } diff --git a/src/config.cpp b/src/config.cpp index 11b5d1d..eac94fc 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -210,8 +210,8 @@ static const std::unordered_map CONFIG_COMMENTS = { "tokens preservation mechanisms (sleep) are not applied to them.", }, { - "misc.randomly_go_sleep", - "If true, Kuni will randomly go to sleep after some time of inactivity to save LLM tokens.\n" + "misc.randomly_go_sleep_chance", + "If greater than zero, Kuni will randomly go to sleep after some time of inactivity to save LLM tokens.\n" "While sleeping, Kuni won't respond to messages (except from papik_chat_id).\n", }, { diff --git a/src/config.h b/src/config.h index 14688cd..d1a2db0 100644 --- a/src/config.h +++ b/src/config.h @@ -18,7 +18,7 @@ X(::Config::LockdownMode, lockdown, ::Config::LockdownMode::PAPIK_ONLY, "general.lockdown") \ X(bool, canWriteToANewPerson, false, "misc.can_write_to_a_new_person") \ X(bool, wakeUpOnPinnedChat, false, "misc.wake_up_on_pinned_chat") \ - X(bool, randomlyGoSleep, true, "misc.randomly_go_sleep") \ + X(float, randomlyGoSleepChance, 0.05f, "misc.randomly_go_sleep_chance") \ X(float, toolReminderProbability, 0.02f, "misc.tool_reminder_probability") \ X(size_t, diaryTokenCountTrigger, 40000, "misc.diary_token_count_trigger") \ X(size_t, diaryInjectionMaxLength, 0, "misc.diary_injection_max_length") \ diff --git a/src/main.cpp b/src/main.cpp index 653f574..25293ed 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -65,6 +65,7 @@ #include "tools/group_admin_remove_message.h" #include "tools/group_admin_set_user_tag.h" #include "tools/remove_message.h" +#include "tools/search_photo_in_gallery.h" #include @@ -155,6 +156,7 @@ class App : public AppBase { if (config().capabilityTakePhoto) { actions.insert(tools::takePhoto(_new(), openAI())); } + actions.insert(tools::searchPhotoInGallery(openAI(), temporaryContext)); if (config().capabilityRecordVoice) { actions.insert(tools::recordAudio()); } diff --git a/src/tools/search_photo_in_gallery.cpp b/src/tools/search_photo_in_gallery.cpp new file mode 100644 index 0000000..884492f --- /dev/null +++ b/src/tools/search_photo_in_gallery.cpp @@ -0,0 +1,122 @@ +// +// Created by alex2772 on 5/9/26. +// + +#include "search_photo_in_gallery.h" + +#include "ImageGenerator.h" +#include "AUI/IO/AFileInputStream.h" +#include "AUI/Util/kAUI.h" +#include "llmui/image.h" +#include "util/cosine_similarity.h" + +#include +#include +#include + +namespace { +struct GalleryEntry { + AString description; + std::valarray embedding; +}; +constexpr size_t MAX_RESULT_COUNT = 5; +constexpr size_t MIN_DESC_LENGTH = 70; +AMap gCachedDatabase; +} + +OpenAITools::Tool tools::searchPhotoInGallery(_ openAI, const IOpenAIChat::Session& session) { + return { + .name = "search_photo_in_gallery", + .description = "Searches for previously taken photos by the given query. " + "Use this before taking a new photo with #take_photo." + "The result of this tool is a set of photo descriptions and a filename. " + "The filename can then be sent to someone else using #send_telegram_message.", + .parameters = + { + .properties = + { + {"photo_desc", { + .type = "string", + .description = "Freeform photo description to search for.", + }, + }, + }, + .required = {"photo_desc"}, + }, + .handler = [openAI = std::move(openAI), &session](OpenAITools::Ctx ctx) -> AFuture { + auto photoDesc = ctx.args["photo_desc"].asStringOpt().valueOrException("photo_desc is required"); + const auto queryEmbedding = co_await openAI->embedding({ .config = config().embedding }, photoDesc); + const auto files = APath("data/gallery").listDir(AFileListFlags::REGULAR_FILES); + + struct Result { + APath filename; + AString description; + double similarity{}; + }; + AVector> results; + for (const auto& i : files) { + if (!(i.endsWith(".png") || i.endsWith(".jpg") || i.endsWith(".jpeg"))) { + continue; + } + auto& entry = gCachedDatabase[i]; + if (entry.description.empty()) { + // cached description. + entry.description = co_await llmui::image({}, *openAI, i); + } + if (entry.embedding.size() != queryEmbedding.size()) { + APath path("cache/images/{}.emb.json"_format(i.filename())); + try { + if (path.isRegularFileExists()) { + entry.embedding = aui::from_json>(AJson::fromStream(AFileInputStream(path))); + } + } catch (const AException& ){} + if (entry.embedding.size() != queryEmbedding.size()) { + entry.embedding = co_await openAI->embedding({.config = config().embedding }, entry.description); + AFileOutputStream(path) << aui::to_json(entry.embedding); + } + } + + results << AUI_THREADPOOL_X [&queryEmbedding, i = i, entry = entry] { + return Result { + .filename = i, + .description = entry.description, + .similarity = util::cosine_similarity(queryEmbedding, entry.embedding), + }; + }; + } + + AVector resultsAwaited; + for (const auto& i : results) { + resultsAwaited << co_await i; + } + ranges::sort(resultsAwaited, [](const auto& a, const auto& b) { return a.similarity > b.similarity; }); + + AString output; + size_t count = 0; + for (const auto& i : resultsAwaited) { + if (count >= MAX_RESULT_COUNT) { + break; + } + if (i.description.length() <= MIN_DESC_LENGTH) { + // bug: short description like + // + // Description complete. + // + // skip + continue; + } + AString tag = ""_format(i.filename.filename()); + if (ranges::any_of(session, [&](const IOpenAIChat::Message& msg) { + return msg.content.contains(tag); + })) { + continue; + } + output += tag; + output += "\n{}\n\n"_format(i.description); + ++count; + } + + co_return output; + }, + }; +} \ No newline at end of file diff --git a/src/tools/search_photo_in_gallery.h b/src/tools/search_photo_in_gallery.h new file mode 100644 index 0000000..2a4c084 --- /dev/null +++ b/src/tools/search_photo_in_gallery.h @@ -0,0 +1,7 @@ +#pragma once +#include "IStableDiffusionClient.h" +#include "OpenAITools.h" + +namespace tools { +OpenAITools::Tool searchPhotoInGallery(_ openAI, const IOpenAIChat::Session& session); +} \ No newline at end of file diff --git a/src/tools/send_telegram_message.cpp b/src/tools/send_telegram_message.cpp index 48f52f4..5c2e1bf 100644 --- a/src/tools/send_telegram_message.cpp +++ b/src/tools/send_telegram_message.cpp @@ -53,8 +53,8 @@ OpenAITools::Tool tools::sendTelegramMessage( {"photo_filename", { .type = "string", .description = "Attaches a photo with the given filename. Filename can be " - "obtained by #take_photo tool; althrough you can attach any file as soon as " - "their filename is correct. Pass null if not attaching a photo.", + "obtained by #take_photo or #search_photo_in_gallery tool; althrough you can attach any" + "file as soon as their filename is correct. Pass null if not attaching a photo.", .nullable = true}, }, {"audio_filename", { @@ -71,7 +71,7 @@ OpenAITools::Tool tools::sendTelegramMessage( .nullable = true}, }, }, - .required = {"text", "photo_filename", "audio_filename", "reply_to_message_id"}, + .required = {}, }, .handler = [telegram = std::move(telegram), openAI = std::move(openAI), @@ -157,7 +157,7 @@ OpenAITools::Tool tools::sendTelegramMessage( } if (config().capabilityTakePhoto) { - reminderMessage += "- Consider sending photos from your gallery or generated by #take_photo tool to make the conversation more lively and engaging!\n"; + reminderMessage += "- Consider sending photos from your gallery (#search_photo_in_gallery) or generated by #take_photo tool to make the conversation more lively and engaging!\n"; } if (config().capabilityRecordVoice) { reminderMessage += "- Consider recording voice notes by #record_audio tool and sending them to make the conversation more lively and engaging!\n"; diff --git a/src/tools/take_photo.cpp b/src/tools/take_photo.cpp index 04e39e0..33df783 100644 --- a/src/tools/take_photo.cpp +++ b/src/tools/take_photo.cpp @@ -11,8 +11,10 @@ OpenAITools::Tool tools::takePhoto(_ stableDiffusion, _ openAI) { return { .name = "take_photo", - .description = "Takes a photo by Kuni. This tool is useful for creating selfies, photos of " + .description = "Takes a NEW photo by Kuni. This tool is useful for creating selfies, photos of " "surroundings, or any other images. " + "Use #search_photo_in_gallery to send a photo from gallery instead to avoid spending " + "time on making a new photo." "The result of this tool is a photo description and a filename. " "The filename can then be sent to someone else using #send_telegram_message.", .parameters = @@ -68,7 +70,7 @@ OpenAITools::Tool tools::takePhoto(_ stableDiffusion, _< co_return "{}\n\nFilename: {}\n" "When writing diary, do not forget to mention this photo and its filename verbatim - you might need this in the future!\n\n" - "You have created photo successfully. Review it carefully. Send it only if you are fully satisfied; use take_photo again to make another photo"_format(description, galleryImage.path.filename()); + "You have created photo successfully. Review it carefully. Send it only if you are fully satisfied; use take_photo again to make another photo or search_photo_in_gallery to search for an old photo."_format(description, galleryImage.path.filename()); }, }; } \ No newline at end of file diff --git a/tests/tools/search_photo_in_gallery_test.cpp b/tests/tools/search_photo_in_gallery_test.cpp new file mode 100644 index 0000000..3c14451 --- /dev/null +++ b/tests/tools/search_photo_in_gallery_test.cpp @@ -0,0 +1,28 @@ +// Created by alex2772 on 5/9/26. +// + +#include "tools/search_photo_in_gallery.h" +#include "../common.h" +#include "AUI/Thread/AAsyncHolder.h" +#include "AUI/Thread/AEventLoop.h" +#include "util/await_synchronously.h" + +#include + +static constexpr auto LOG_TAG = "SearchPhotoInGalleryIntegrationTest"; + +TEST(SearchPhotoInGalleryIntegrationTest, BasicTest) { + util::await_synchronously([]() -> AFuture<> { + auto openAI = _new(); + OpenAITools tools {}; + auto response = co_await tools::searchPhotoInGallery(openAI, {}).handler(OpenAITools::Ctx { + .tools = tools, + .args = AJson::Object { { "photo_desc", "Close-up portrait of anime girl with cat ears against starry background." } }, + .temporaryContext = {}, + .allToolCalls = {}, + }); + ALogger::info(LOG_TAG) << "Response: " << response; + EXPECT_FALSE(response.empty()); + co_return; + }()); +} From 73ccb0f8a2ffa5feebcd72f7053b412ba017296a Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 21 Jul 2026 19:16:09 +0300 Subject: [PATCH 02/32] fix(ImageGenerator): revert simplification it fixes on specific bad prompt and generates bad images continously. additionally, added logic to keep correct aspect ratio in IOpenAIChat::embedImage. --- src/ImageGenerator.cpp | 5 ++--- src/OpenAIChatImpl.cpp | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ImageGenerator.cpp b/src/ImageGenerator.cpp index 77bacdd..f0c1682 100644 --- a/src/ImageGenerator.cpp +++ b/src/ImageGenerator.cpp @@ -53,13 +53,12 @@ AFuture ImageGenerator::generate(AString descripti AString descriptionWithAppearance = "\n{}\n\n\n{}\n"_format(config().characterName, prompts().characterAppearance, description); - ALogger::info(LOG_TAG) << "positive=" << currentPrompt.positive << "\n\nnegative=" << currentPrompt.negative; while (trialIndex <= TRIAL_COUNT) { try { ++trialIndex; static std::default_random_engine ge(std::time(nullptr)); { - ALogger::info(LOG_TAG) << "Iteration " << trialIndex; + ALogger::info(LOG_TAG) << "Iteration " << trialIndex << " with prompt:\npositive=" << currentPrompt.positive << "\n\nnegative=" << currentPrompt.negative; IStableDiffusionClient::Txt2ImgResponse response; try @@ -109,7 +108,7 @@ AFuture ImageGenerator::generate(AString descripti } ALogger::info(LOG_TAG) << "Not satisfied. Feedback: " << assessment.feedback; - // co_await engineerPrompt(currentPrompt, description, prompts().characterAppearance, assessment.feedback); + co_await engineerPrompt(currentPrompt, description, prompts().characterAppearance, assessment.feedback); } diff --git a/src/OpenAIChatImpl.cpp b/src/OpenAIChatImpl.cpp index 25e146b..12fb88b 100644 --- a/src/OpenAIChatImpl.cpp +++ b/src/OpenAIChatImpl.cpp @@ -34,9 +34,9 @@ using namespace std::chrono_literals; AString IOpenAIChat::embedImage(AImageView image) { ALOG_TRACE(LOG_TAG) << "embedImage"; AByteBuffer jpg; - auto resized = image.resizedLinearDownscale({672, 672}); + auto resized = image.resizedLinearDownscale({672, 672 * float(image.height()) / float(image.width())}); JpgImageLoader::save(jpg, resized); - // JpgImageLoader::save(AFileOutputStream("test.jpg"), resized); + JpgImageLoader::save(AFileOutputStream("test.jpg"), resized); return "<{}>data:image/jpg;base64,{}"_format(EMBEDDING_TAG, jpg.toBase64String(), EMBEDDING_TAG); } From 84f51d615f9f853a07ca92846b8b0f757f05b227 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Wed, 22 Jul 2026 20:49:59 +0300 Subject: [PATCH 03/32] feat(comfyui): initial work --- src/ImageGenerator.cpp | 7 +- src/ImageGenerator.h | 7 +- src/comfyui.cpp | 192 ++++++++++++++++++++++++ src/comfyui.h | 60 ++++++++ src/config.h | 1 + tests/ComfyUIIntegrationTest.cpp | 106 +++++++++++++ tests/ImageGeneratorIntegrationTest.cpp | 2 +- tests/tools/take_photo_test.cpp | 16 ++ 8 files changed, 379 insertions(+), 12 deletions(-) create mode 100644 src/comfyui.cpp create mode 100644 src/comfyui.h create mode 100644 tests/ComfyUIIntegrationTest.cpp diff --git a/src/ImageGenerator.cpp b/src/ImageGenerator.cpp index f0c1682..ab40d8e 100644 --- a/src/ImageGenerator.cpp +++ b/src/ImageGenerator.cpp @@ -38,7 +38,7 @@ static AJson parseResponse(AString content) { return AJson::fromString(content); } -AFuture ImageGenerator::generate(AString description) { +AFuture<_> ImageGenerator::generate(AString description) { ALOG_TRACE(LOG_TAG) << "generate: " << description; int trialIndex = 0; @@ -98,10 +98,7 @@ AFuture ImageGenerator::generate(AString descripti if (assessment.satisfied) { ALogger::info(LOG_TAG) << "Satisfied with the result. " << assessment.feedback; - auto dst = APath("data/gallery/{}.png"_format(std::chrono::system_clock::now())); - dst.parent().makeDirs(); - PngImageLoader::save(AFileOutputStream{ dst }, *lastImage); - co_return GalleryImage{ .image = lastImage, .path = dst.absolute() }; + co_return lastImage; } if (firstFeedback.empty()) { firstFeedback = assessment.feedback; diff --git a/src/ImageGenerator.h b/src/ImageGenerator.h index 6881713..7b016a2 100644 --- a/src/ImageGenerator.h +++ b/src/ImageGenerator.h @@ -15,18 +15,13 @@ class ImageGenerator { ImageGenerator(_ sdClient, _ openAI, IOpenAIChat::Params chatParams) : mSdClient(std::move(sdClient)), mOpenAI(std::move(openAI)), mChatParams(std::move(chatParams)) {} - struct GalleryImage { - _ image; - APath path; - }; - /** * Generates an image from a description. * Uses IOpenAIChat to transform the description into an SD-optimized prompt, * pulls character details from KuniCharacter, and iteratively refines the prompt * based on vision-based assessment of the generated images. */ - AFuture generate(AString description); + AFuture<_> generate(AString description); private: _ mSdClient; diff --git a/src/comfyui.cpp b/src/comfyui.cpp new file mode 100644 index 0000000..4bd7148 --- /dev/null +++ b/src/comfyui.cpp @@ -0,0 +1,192 @@ +#include "comfyui.h" + +#include "AUI/Curl/ACurl.h" +#include "AUI/Curl/ACurlMulti.h" +#include "AUI/Curl/AFormMultipart.h" +#include "AUI/Curl/AWebsocket.h" +#include "AUI/IO/AByteBufferInputStream.h" +#include "AUI/Image/png/PngImageLoader.h" +#include "AUI/Json/Conversion.h" +#include "AUI/Logging/ALogger.h" +#include "AUI/Util/ARandom.h" +#include "config.h" + +#include +#include + +static constexpr auto LOG_TAG = "comfyui"; + +namespace { + +AString wsUrl(const AString& baseUrl, const AString& clientId) { + AUI_ASSERT(baseUrl.endsWith("/")); + return "{}ws?clientId={}"_format(baseUrl, clientId); +} + +AVector authHeaders(const Endpoint& endpoint, AVector extra = {}) { + if (!endpoint.bearerKey.empty()) { + extra << "Authorization: Bearer {}"_format(endpoint.bearerKey); + } + return extra; +} + +} // namespace + +AFuture comfy::prompt(AJson workflow) { + ALOG_TRACE(LOG_TAG) << "prompt"; + if (workflow.isObject()) { + // A regular "Save" (UI/front-end) workflow export has top-level keys like "nodes"/"links"/"groups"/ + // "last_node_id", where node entries are objects nested under "nodes" (an array), not top-level + // "": {"class_type": ..., "inputs": ...} entries. Sending that straight to /prompt makes ComfyUI's + // server (and some custom nodes, e.g. Impact-Pack) blow up with cryptic errors like + // "argument of type 'int' is not a container or iterable" because it iterates top-level values expecting + // node objects and finds plain ints/arrays (e.g. "revision": 0, "last_node_id": 116) instead. + // You need to export via ComfyUI's "Save (API format)" (or "Export (API)") button instead. + if (workflow.contains("nodes") || workflow.contains("links") || workflow.contains("last_node_id")) { + throw AException( + "comfy::prompt: workflow looks like a UI export (has \"nodes\"/\"links\"/\"last_node_id\" keys), " + "not an API format workflow. Re-export it using ComfyUI's \"Save (API format)\" button."); + } + } + const auto endpoint = config().comfyEndpoint; + + static ARandom r; + auto clientId = r.nextUuid().toString(); + + const auto url = wsUrl(endpoint.baseUrl, clientId); + auto websocket = _new(url); + AFuture<> connected; + AFuture promptDone; + + AObject::connect(websocket->connected, AObject::GENERIC_OBSERVER, [connected] { connected.supplyValue(); }); + AObject::connect(websocket->websocketClosed, AObject::GENERIC_OBSERVER, [promptDone](const AString& reason) { + if (!promptDone.hasResult()) { + promptDone.supplyValue(AString()); + ALogger::warn(LOG_TAG) << "Websocket closed before completion: " << reason; + } + }); + + AString promptId; + AObject::connect(websocket->received, AObject::GENERIC_OBSERVER, [promptDone, &promptId](AByteBuffer buffer) { + if (buffer.empty()) { + return; + } + try { + auto json = AJson::fromBuffer(buffer); + const auto& type = json["type"].asStringOpt().valueOr(""); + if (type != "executing") { + return; + } + const auto& data = json["data"]; + if (!data["prompt_id"].asStringOpt().valueOr("").empty() && data["prompt_id"].asString() != promptId) { + return; + } + if (data["node"].isNull()) { + if (!promptDone.hasResult()) { + promptDone.supplyValue(promptId); + } + } + } catch (const AException& e) { + ALogger::warn(LOG_TAG) << "Failed to process websocket message: " << e; + } + }); + + ACurlMulti::global() << websocket; + + co_await connected; + + AJson body = AJson::Object{ + {"prompt", std::move(workflow)}, + {"client_id", clientId}, + }; + + auto queueResponse = AJson::fromBuffer((co_await ACurl::Builder(endpoint.baseUrl + "prompt") + .withMethod(ACurl::Method::HTTP_POST) + .withHeaders(authHeaders(endpoint, {"Content-Type: application/json"})) + .withBody(AJson::toString(body).toStdString()) + .withTimeout(config().requestTimeoutSecs) + .runAsync()) + .body); + + if (queueResponse.contains("error")) { + throw AException("comfy::prompt: {}"_format(AJson::toString(queueResponse["error"]))); + } + + promptId = queueResponse["prompt_id"].asString(); + ALogger::info(LOG_TAG) << "Queued prompt_id=" << promptId; + + co_await promptDone; + websocket->close(); + + auto historyResponse = AJson::fromBuffer((co_await ACurl::Builder(endpoint.baseUrl + "history/{}"_format(promptId)) + .withHeaders(authHeaders(endpoint)) + .withTimeout(config().requestTimeoutSecs) + .runAsync()) + .body); + + auto history = historyResponse[promptId]; + + PromptResult result; + result.history = history; + + const auto& outputs = history["outputs"]; + if (outputs.isObject()) { + for (const auto& [nodeId, nodeOutput] : outputs.asObject()) { + if (!nodeOutput.contains("images")) { + continue; + } + for (const auto& image : nodeOutput["images"].asArray()) { + auto filename = image["filename"].asString(); + auto subfolder = image["subfolder"].asStringOpt().valueOr(""); + auto type = image["type"].asStringOpt().valueOr("output"); + + auto imageResponse = co_await ACurl::Builder(endpoint.baseUrl + "view") + .withParams({ + {"filename", filename}, + {"subfolder", subfolder}, + {"type", type}, + }) + .withHeaders(authHeaders(endpoint)) + .withTimeout(config().requestTimeoutSecs) + .runAsync(); + + result.images << AImage::fromBuffer(imageResponse.body); + } + } + } + + co_return result; +} + +AFuture comfy::uploadImage(const AImage& image, AString filename) { + ALOG_TRACE(LOG_TAG) << "uploadImage: " << filename; + const auto endpoint = config().comfyEndpoint; + + AByteBuffer png; + PngImageLoader::save(png, image); + + AFormMultipart form; + form["image"] = { .value = std::move(png), .filename = filename, .mimeType = "image/png" }; + form["overwrite"] = { .value = AString("true") }; + + auto response = co_await ACurl::Builder(endpoint.baseUrl + "upload/image") + .withMethod(ACurl::Method::HTTP_POST) + .withHeaders(authHeaders(endpoint)) + .withMultipart(form) + .withTimeout(config().requestTimeoutSecs) + .runAsync(); + + if (response.code != ACurl::ResponseCode::HTTP_200_OK) { + throw AException("comfy::uploadImage: HTTP {}: {}"_format(int(response.code), AString::fromUtf8(response.body))); + } + + auto json = AJson::fromBuffer(response.body); + co_return json["name"].asStringOpt().valueOr(std::move(filename)); +} + +AFuture<> comfy::unload() { + co_await ACurl::Builder(config().comfyEndpoint.baseUrl + "free") + .withBody(R"({"unload_models": true, "free_memory": true})") + .withMethod(ACurl::Method::HTTP_POST) + .runAsync(); +} diff --git a/src/comfyui.h b/src/comfyui.h new file mode 100644 index 0000000..2334b95 --- /dev/null +++ b/src/comfyui.h @@ -0,0 +1,60 @@ +#pragma once +#include "AUI/Common/AString.h" +#include "AUI/Common/AVector.h" +#include "AUI/Common/AMap.h" +#include "AUI/Json/AJson.h" +#include "AUI/Thread/AFuture.h" +#include "AUI/Image/AImage.h" +#include "Endpoint.h" + +/** + * @brief Minimal client for the ComfyUI HTTP/WebSocket API. + * @details + * ComfyUI exposes a native HTTP and WebSocket interface (by default at `http://127.0.0.1:8188/`) that lets us + * submit exported "API format" workflow graphs (`/prompt`), track their execution over a websocket (`/ws`) and + * fetch back the resulting images (`/history/{prompt_id}` + `/view`). + * + * @see https://docs.comfy.org/development/comfyui-server/api-examples + */ +namespace comfy { + +/** + * @brief Result of comfy::prompt: images produced by the workflow's output (SaveImage/PreviewImage-like) nodes. + */ +struct PromptResult { + /** + * @brief Decoded output images, in the order they were reported by /history. + */ + AVector<_> images; + + /** + * @brief Raw `/history/{prompt_id}` response, in case the caller needs something beyond images. + */ + AJson history; +}; + +/** + * @brief Uploads a workflow API graph, waits for it to complete and downloads the resulting images. + * @param workflow "API format" workflow graph (i.e. the JSON exported via "Save (API format)" in ComfyUI, or + * built programmatically), keyed by node id. + * @details + * Internally this: + * 1. opens a websocket to `${endpoint}/ws?clientId=...`; + * 2. POSTs the graph together with the client id to `${endpoint}/prompt`; + * 3. waits for an `executing` message with `node == null` for our `prompt_id` (i.e. the whole graph is done); + * 4. fetches `${endpoint}/history/{prompt_id}` and downloads every reported output image via `${endpoint}/view`. + */ +AFuture prompt(AJson workflow); + +/** + * @brief Uploads an image to ComfyUI's input folder so it can be referenced by a `LoadImage`-like node. + * @param image image to upload. + * @param filename desired filename (passed as-is to ComfyUI; it may rename on collision). + * @return the filename ComfyUI actually stored the image under (i.e. `name` field of `/upload/image` response). + * @see https://docs.comfy.org/development/comfyui-server/api-examples + */ +AFuture uploadImage(const AImage& image, AString filename); + +AFuture<> unload(); + +} // namespace comfy diff --git a/src/config.h b/src/config.h index d1a2db0..37c47c7 100644 --- a/src/config.h +++ b/src/config.h @@ -55,6 +55,7 @@ X(bool, capabilityTakePhoto, false, "capabilities.take_photo.enabled") \ X(Endpoint, sdEndpoint, (Endpoint{.baseUrl="http://localhost:7860/"}),"capabilities.take_photo.sd.endpoint") \ X(AString, sdCheckpoint, "novaAnimeXL_ilV170.safetensors", "capabilities.take_photo.sd.checkpoint") \ + X(Endpoint, comfyEndpoint, (Endpoint{.baseUrl="http://localhost:8188/"}),"capabilities.take_photo.comfyui.endpoint") \ X(bool, capabilityHearing, false, "capabilities.hearing.enabled") \ X(EndpointAndModel, llmAudioToText, (EndpointAndModel{.endpoint={"http://localhost:9000/v1/"},.model="base"}), "capabilities.hearing.llm_audio_to_text") \ X(bool, capabilityRecordVoice, false, "capabilities.record_voice.enabled") \ diff --git a/tests/ComfyUIIntegrationTest.cpp b/tests/ComfyUIIntegrationTest.cpp new file mode 100644 index 0000000..2e89259 --- /dev/null +++ b/tests/ComfyUIIntegrationTest.cpp @@ -0,0 +1,106 @@ +#include "comfyui.h" +#include "common.h" + +#include +#include "AUI/Thread/AAsyncHolder.h" +#include "AUI/Thread/AEventLoop.h" +#include "AUI/Json/AJson.h" +#include "AUI/Image/AImage.h" +#include "config.h" +#include "AUI/Image/png/PngImageLoader.h" + +// This test requires a running ComfyUI instance with the default "Save Image" workflow's checkpoint +// available (see config().comfyEndpoint, default http://127.0.0.1:8188/). If it's not available, this test +// will fail with a connection error - that's expected in CI/headless environments without ComfyUI installed. +TEST(ComfyUIIntegration, Txt2Img) { + AEventLoop loop; + IEventLoop::Handle h(&loop); + AAsyncHolder async; + + async << []() -> AFuture<> { + // Minimal "API format" txt2img graph (KSampler + CheckpointLoaderSimple + SaveImage), exported the same + // way ComfyUI's "Save (API format)" button would produce it. + AJson workflow = AJson::Object{ + {"3", + AJson::Object{ + {"class_type", "KSampler"}, + {"inputs", + AJson::Object{ + {"seed", 0}, + {"steps", 4}, + {"cfg", 2.0}, + {"sampler_name", "euler"}, + {"scheduler", "normal"}, + {"denoise", 1.0}, + {"model", AJson::Array{"4", 0}}, + {"positive", AJson::Array{"6", 0}}, + {"negative", AJson::Array{"7", 0}}, + {"latent_image", AJson::Array{"5", 0}}, + }}}}, + {"4", + AJson::Object{ + {"class_type", "CheckpointLoaderSimple"}, + {"inputs", AJson::Object{{"ckpt_name", config().sdCheckpoint}}}}}, + {"5", + AJson::Object{ + {"class_type", "EmptyLatentImage"}, + {"inputs", AJson::Object{{"width", 256}, {"height", 256}, {"batch_size", 1}}}}}, + {"6", + AJson::Object{ + {"class_type", "CLIPTextEncode"}, + {"inputs", AJson::Object{{"text", "anime girl cat ears"}, {"clip", AJson::Array{"4", 1}}}}}}, + {"7", + AJson::Object{ + {"class_type", "CLIPTextEncode"}, + {"inputs", AJson::Object{{"text", "text, watermark"}, {"clip", AJson::Array{"4", 1}}}}}}, + {"8", + AJson::Object{ + {"class_type", "VAEDecode"}, + {"inputs", AJson::Object{{"samples", AJson::Array{"3", 0}}, {"vae", AJson::Array{"4", 2}}}}}}, + {"9", + AJson::Object{ + {"class_type", "SaveImage"}, + {"inputs", AJson::Object{{"filename_prefix", "kuni_test"}, {"images", AJson::Array{"8", 0}}}}}}, + }; + + try { + auto result = co_await comfy::prompt(workflow); + EXPECT_FALSE(result.images.empty()); + if (!result.images.empty()) { + EXPECT_GT(result.images.first()->width(), 0); + EXPECT_GT(result.images.first()->height(), 0); + } + PngImageLoader::save(AFileOutputStream("comfyui_tmp.png"), *result.images.first()); + } catch (const AException& e) { + // If ComfyUI is not running, we expect a connection error - log and mark as non-fatal. + std::cout << "ComfyUI not running or error: " << e.getMessage() << std::endl; + GTEST_NONFATAL_FAILURE_("ComfyUI not running or error"); + } + }(); + + while (!async.empty()) { + loop.iteration(); + } +} + +TEST(ComfyUIIntegration, UploadImage) { + AEventLoop loop; + IEventLoop::Handle h(&loop); + AAsyncHolder async; + + async << []() -> AFuture<> { + try { + AFormattedImage image({1, 1}); + image.set(glm::ivec2(0, 0), AFormattedColorConverter(AColor{1.f, 1.f, 1.f, 1.f})); + auto name = co_await comfy::uploadImage(AImage(image), "kuni_test_upload.png"); + EXPECT_FALSE(name.empty()); + } catch (const AException& e) { + std::cout << "ComfyUI not running or error: " << e.getMessage() << std::endl; + GTEST_NONFATAL_FAILURE_("ComfyUI not running or error"); + } + }(); + + while (!async.empty()) { + loop.iteration(); + } +} diff --git a/tests/ImageGeneratorIntegrationTest.cpp b/tests/ImageGeneratorIntegrationTest.cpp index e65ddfe..8d9c921 100644 --- a/tests/ImageGeneratorIntegrationTest.cpp +++ b/tests/ImageGeneratorIntegrationTest.cpp @@ -25,7 +25,7 @@ TEST(ImageGeneratorIntegration, Generate) ImageGenerator generator(std::move(sdClient), _new(), std::move(chatParams)); try { - auto image = (co_await generator.generate("Kuni makes a selfie")).image; + auto image = co_await generator.generate("Kuni makes a selfie"); EXPECT_NE(image, nullptr); PngImageLoader::save(AFileOutputStream{ "out_generator.png" }, *image); EXPECT_GT(image->width(), 0); diff --git a/tests/tools/take_photo_test.cpp b/tests/tools/take_photo_test.cpp index 9831460..ba0f6ec 100644 --- a/tests/tools/take_photo_test.cpp +++ b/tests/tools/take_photo_test.cpp @@ -1,6 +1,7 @@ // Created by alex2772 on 5/9/26. // +#include "StableDiffusionClientImpl.h" #include "tools/take_photo.h" #include "../common.h" #include "AUI/Thread/AAsyncHolder.h" @@ -8,6 +9,8 @@ #include "util/await_synchronously.h" #include "../OpenAIMock.h" +#include "AUI/Image/png/PngImageLoader.h" + #include namespace { @@ -69,3 +72,16 @@ TEST(TakePhotoTest, PhotoDescNotStringThrows) { AException ); } + +TEST(TakePhotoIntegrationTest, Basic) { + auto tool = tools::takePhoto(_new(), _new()); + + OpenAITools tools{}; + auto result = util::await_synchronously(tool.handler({ + .tools = tools, + .args = AJson::Object{{"photo_desc", "Kuni makes a selfie"}}, + .temporaryContext = {}, + .allToolCalls = {}, + })); + ASSERT_FALSE(result.empty()); +} From a985c945c5a68c65944c68d9a13a930cadc5a87b Mon Sep 17 00:00:00 2001 From: alex2772 Date: Wed, 22 Jul 2026 21:10:48 +0300 Subject: [PATCH 04/32] update take_photo.cpp --- src/tools/take_photo.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/tools/take_photo.cpp b/src/tools/take_photo.cpp index 33df783..7374da9 100644 --- a/src/tools/take_photo.cpp +++ b/src/tools/take_photo.cpp @@ -6,6 +6,9 @@ #include "ImageGenerator.h" #include "StableDiffusionClientImpl.h" +#include "comfyui.h" +#include "AUI/IO/AFileInputStream.h" +#include "AUI/Image/png/PngImageLoader.h" #include "llmui/image.h" OpenAITools::Tool tools::takePhoto(_ stableDiffusion, _ openAI) { @@ -65,12 +68,17 @@ OpenAITools::Tool tools::takePhoto(_ stableDiffusion, _< .handler = [stableDiffusion = std::move(stableDiffusion), openAI = std::move(openAI)](OpenAITools::Ctx ctx) -> AFuture { auto photoDesc = ctx.args["photo_desc"].asStringOpt().valueOrException("photo_desc is required"); - auto galleryImage = co_await ImageGenerator{_new(), openAI, IOpenAIChat::Params{.config = config().llmImageToText}}.generate(photoDesc); - auto description = co_await llmui::image({}, *openAI, galleryImage.path); + auto image = co_await ImageGenerator{_new(), openAI, IOpenAIChat::Params{.config = config().llmImageToText}}.generate(photoDesc); + + + auto dst = APath("data/gallery/{}.png"_format(std::chrono::system_clock::now())); + dst.parent().makeDirs(); + PngImageLoader::save(AFileOutputStream{ dst }, *image); + auto description = co_await llmui::image({}, *openAI, dst); co_return "{}\n\nFilename: {}\n" "When writing diary, do not forget to mention this photo and its filename verbatim - you might need this in the future!\n\n" - "You have created photo successfully. Review it carefully. Send it only if you are fully satisfied; use take_photo again to make another photo or search_photo_in_gallery to search for an old photo."_format(description, galleryImage.path.filename()); + "You have created photo successfully. Review it carefully. Send it only if you are fully satisfied; use take_photo again to make another photo or search_photo_in_gallery to search for an old photo."_format(description, dst.filename()); }, }; } \ No newline at end of file From 3fad168b5ffebcc8a84633e619878d32d111cb49 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Thu, 23 Jul 2026 03:46:46 +0300 Subject: [PATCH 05/32] feat(Diary): lex assisted search --- src/Diary.cpp | 99 +++++++++++++++++++++++++++++++++++ src/Diary.h | 21 ++++++++ src/tools/ask.cpp | 6 +-- src/ui/debug/DiaryQueryAI.cpp | 2 +- 4 files changed, 124 insertions(+), 4 deletions(-) diff --git a/src/Diary.cpp b/src/Diary.cpp index 7341faf..8969653 100644 --- a/src/Diary.cpp +++ b/src/Diary.cpp @@ -5,6 +5,8 @@ #include #include +#include "AUI/Common/AMap.h" +#include "AUI/Common/ASet.h" #include "AUI/IO/AFileInputStream.h" #include "AUI/IO/AFileOutputStream.h" #include "AUI/Logging/ALogger.h" @@ -23,6 +25,69 @@ using namespace std::chrono_literals; static constexpr auto LOG_TAG = "Diary"; +namespace { + +/** + * @brief Splits text into a set of lowercase word tokens for lexical matching. + * @details + * A "word" is a maximal run of alphanumeric bytes (ASCII letters/digits, or any UTF-8 + * continuation/multi-byte sequence, which covers Cyrillic and other non-ASCII alphabets since + * those bytes are always > 0x7F and thus not touched as delimiters). Everything else (spaces, + * punctuation, quotes, etc.) is treated as a separator. Tokens shorter than 2 bytes are dropped + * as they carry little discriminative power (and are noisy for CJK-less languages). + */ +ASet tokenize(AStringView text) { + ASet tokens; + const auto lower = text.lowercase(); + const auto& bytes = lower.bytes(); + size_t tokenStart = AString::NPOS; + auto flush = [&](size_t end) { + if (tokenStart == AString::NPOS) { + return; + } + if (end - tokenStart >= 2) { + tokens << AString(bytes.substr(tokenStart, end - tokenStart)); + } + tokenStart = AString::NPOS; + }; + for (size_t i = 0; i < bytes.size(); ++i) { + const auto c = static_cast(bytes[i]); + const bool isWordByte = (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c >= 0x80; + if (isWordByte) { + if (tokenStart == AString::NPOS) { + tokenStart = i; + } + } else { + flush(i); + } + } + flush(bytes.size()); + return tokens; +} + +/** + * @brief Computes a normalized lexical overlap score in [0, 1] between query and entry tokens. + * @details + * Uses Jaccard-like overlap biased towards the query: the fraction of query tokens that are + * also present in the entry. This rewards entries that literally mention the keywords/names the + * user asked about (e.g., "kiriko", "overwatch"), regardless of how the embedding model happened + * to place them in vector space. + */ +double lexicalOverlap(const ASet& queryTokens, const ASet& entryTokens) { + if (queryTokens.empty()) { + return 0.0; + } + size_t matches = 0; + for (const auto& token : queryTokens) { + if (entryTokens.contains(token)) { + ++matches; + } + } + return double(matches) / double(queryTokens.size()); +} + +} // namespace + Diary::Diary(Init init): mInit(std::move(init)) { ALOG_TRACE(LOG_TAG) << "Diary::Diary: " << mInit.diaryDir; mInit.diaryDir.makeDirs(); @@ -121,6 +186,40 @@ AFuture> Diary::query(const std::valarray< co_return result; } +AFuture> Diary::query(const AString& query, QueryOpts opts) { + ALOG_TRACE(LOG_TAG) << "Diary::query(text): \"" << query << "\""; + const auto queryTokens = tokenize(query); + const auto embedding = co_await openAI()->embedding({ .config = config().embedding }, query); + + // over-fetch on the embedding pass so the lexical re-ranking below has enough candidates to + // pick from - a semantically-close-but-unrelated entry might otherwise push out a lexically + // exact match before we even get to re-rank. + auto overFetchOpts = opts; + overFetchOpts.maxEntryCount = std::max(opts.maxEntryCount * 10, opts.maxEntryCount + 32); + auto candidates = co_await this->query(embedding, overFetchOpts); + + for (auto& candidate : candidates) { + const auto overlap = lexicalOverlap(queryTokens, tokenize(candidate.entry->freeformBody)); + // blend: embedding similarity stays the dominant signal, lexical overlap boosts entries + // that literally mention the query's keywords/names (e.g., "Kiriko"), and penalizes (by + // omission of the boost) those that merely happen to be nearby in embedding space. + candidate.relatedness += overlap * config().diaryLexicalWeight; + } + + ranges::sort(candidates, [](const auto& a, const auto& b) { return a.relatedness > b.relatedness; }); + if (candidates.size() > opts.maxEntryCount) { + candidates.resize(opts.maxEntryCount); + } + + // drop entries clearly below the relevance bar - avoids returning "random" unrelated memories + // just to fill up maxEntryCount. + while (!candidates.empty() && candidates.last().relatedness < config().diaryMinRelatedness) { + candidates.pop_back(); + } + + co_return candidates; +} + AFuture Diary::entryIsRelated(const std::valarray& context, EntryEx& entry, QueryOpts opts) { ALOG_TRACE(LOG_TAG) << "entryIsRelated: " << entry.id; if (entry.freeformBody.empty()) { diff --git a/src/Diary.h b/src/Diary.h index 70275da..5447d4e 100644 --- a/src/Diary.h +++ b/src/Diary.h @@ -186,9 +186,30 @@ class Diary { * vector and each entry's embedding, normalizes it to the range * [0,1], and returns a sorted vector of {@link EntryExAndRelatedness} * objects. + * + * @note Prefer the {@link query(const AString&, QueryOpts)} overload when a query text is + * available; it combines embedding similarity with lexical keyword matching, which + * greatly reduces false positives (e.g., unrelated entries about a different topic + * that happen to have a close embedding). This overload is intended for callers that + * only have an embedding on hand (e.g., context-based lookups, plagiarism checks). */ AFuture> query(const std::valarray& query, QueryOpts opts); + /** + * @brief Asynchronously query the diary for entries related to a query text. + * @details + * This is the preferred search entry point. In addition to the embedding-based cosine + * similarity (see {@link query(const std::valarray&, QueryOpts)}), this overload + * boosts entries that lexically share keywords/tokens (e.g., proper nouns like "Kiriko") + * with the query text. This mitigates the common RAG failure mode where an unrelated entry + * has a deceptively close embedding but doesn't actually mention the thing being asked + * about. + * + * The query text is embedded once via {@link openAI}'s embedding endpoint and delegates the + * semantic part of scoring to the embedding-based overload. + */ + AFuture> query(const AString& query, QueryOpts opts); + /** * @brief Compute the relatedness of a single entry to a context vector. * diff --git a/src/tools/ask.cpp b/src/tools/ask.cpp index 232a0b6..37bf67b 100644 --- a/src/tools/ask.cpp +++ b/src/tools/ask.cpp @@ -13,8 +13,8 @@ static constexpr auto LOG_TAG = "ask"; static AFuture -queryDiary(IOpenAIChat& openAI, Diary& diary, ASet includedIds, const AString& cue, const Diary::QueryOpts& opts) { - auto diaryResponse = co_await diary.query(co_await openAI.embedding({ .config = config().embedding }, cue), [&] { +queryDiary(Diary& diary, ASet includedIds, const AString& cue, const Diary::QueryOpts& opts) { + auto diaryResponse = co_await diary.query(cue, [&] { auto optsCopy = opts; optsCopy.maxEntryCount *= 10; return optsCopy; @@ -100,7 +100,7 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu if (includeWebSearchResults) { out += "\n"; } - out += co_await queryDiary(openAI, diary, includedIds, cue, opts); + out += co_await queryDiary(diary, includedIds, cue, opts); if (includeWebSearchResults) { out += "\n\n"; } diff --git a/src/ui/debug/DiaryQueryAI.cpp b/src/ui/debug/DiaryQueryAI.cpp index 739a1e0..6cdafb1 100644 --- a/src/ui/debug/DiaryQueryAI.cpp +++ b/src/ui/debug/DiaryQueryAI.cpp @@ -153,7 +153,7 @@ struct State { }, .handler = [this, opts, &includedIds](OpenAITools::Ctx ctx) -> AFuture { auto cue = ctx.args["text"].asStringOpt().valueOrException("text is required string"); - auto diaryResponse = co_await diary.query(co_await openAI->embedding({ .config = config().embedding }, cue), opts); + auto diaryResponse = co_await diary.query(cue, opts); AString formattedResponse; ALOG_DEBUG("Diary") << "queryAI cue=\"" From dc088a09ade5594d9c2d16c7c3bc089a967ef1df Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sat, 25 Jul 2026 19:15:30 +0300 Subject: [PATCH 06/32] fix(comfy): get rid of websocket, add timeout logic --- src/comfyui.cpp | 64 +++++++++++++------------------------------------ src/config.cpp | 12 ++++++++++ 2 files changed, 29 insertions(+), 47 deletions(-) diff --git a/src/comfyui.cpp b/src/comfyui.cpp index 4bd7148..8a2a122 100644 --- a/src/comfyui.cpp +++ b/src/comfyui.cpp @@ -16,6 +16,9 @@ static constexpr auto LOG_TAG = "comfyui"; +using namespace std::chrono_literals; + + namespace { AString wsUrl(const AString& baseUrl, const AString& clientId) { @@ -54,47 +57,8 @@ AFuture comfy::prompt(AJson workflow) { auto clientId = r.nextUuid().toString(); const auto url = wsUrl(endpoint.baseUrl, clientId); - auto websocket = _new(url); - AFuture<> connected; - AFuture promptDone; - - AObject::connect(websocket->connected, AObject::GENERIC_OBSERVER, [connected] { connected.supplyValue(); }); - AObject::connect(websocket->websocketClosed, AObject::GENERIC_OBSERVER, [promptDone](const AString& reason) { - if (!promptDone.hasResult()) { - promptDone.supplyValue(AString()); - ALogger::warn(LOG_TAG) << "Websocket closed before completion: " << reason; - } - }); AString promptId; - AObject::connect(websocket->received, AObject::GENERIC_OBSERVER, [promptDone, &promptId](AByteBuffer buffer) { - if (buffer.empty()) { - return; - } - try { - auto json = AJson::fromBuffer(buffer); - const auto& type = json["type"].asStringOpt().valueOr(""); - if (type != "executing") { - return; - } - const auto& data = json["data"]; - if (!data["prompt_id"].asStringOpt().valueOr("").empty() && data["prompt_id"].asString() != promptId) { - return; - } - if (data["node"].isNull()) { - if (!promptDone.hasResult()) { - promptDone.supplyValue(promptId); - } - } - } catch (const AException& e) { - ALogger::warn(LOG_TAG) << "Failed to process websocket message: " << e; - } - }); - - ACurlMulti::global() << websocket; - - co_await connected; - AJson body = AJson::Object{ {"prompt", std::move(workflow)}, {"client_id", clientId}, @@ -115,16 +79,22 @@ AFuture comfy::prompt(AJson workflow) { promptId = queueResponse["prompt_id"].asString(); ALogger::info(LOG_TAG) << "Queued prompt_id=" << promptId; - co_await promptDone; - websocket->close(); - auto historyResponse = AJson::fromBuffer((co_await ACurl::Builder(endpoint.baseUrl + "history/{}"_format(promptId)) - .withHeaders(authHeaders(endpoint)) - .withTimeout(config().requestTimeoutSecs) - .runAsync()) - .body); - auto history = historyResponse[promptId]; + auto history = co_await [&]() -> AFuture { + for (size_t maxTrials = 100; maxTrials > 0; --maxTrials) { + auto historyResponse = AJson::fromBuffer((co_await ACurl::Builder(endpoint.baseUrl + "history/{}"_format(promptId)) + .withHeaders(authHeaders(endpoint)) + .withTimeout(config().requestTimeoutSecs) + .runAsync()) + .body); + if (auto r = historyResponse.containsOpt(promptId)) { + co_return *r; + } + co_await AThread::asyncSleep(1s); + } + throw AException("timeout"); + }(); PromptResult result; result.history = history; diff --git a/src/config.cpp b/src/config.cpp index eac94fc..ebb6dac 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -152,6 +152,11 @@ static const std::unordered_map CONFIG_COMMENTS = { "\n" "Generally, deepseek-v4-flash is cheap and fine enough. Local models sub 26b are stupid.", }, + { + "general.llm_diary", + "LLM endpoint used for diary lookups. Can be text-only.\n" + "Generally, local/cheap models are fine enough.", + }, { "general.embedding", "Embeddings endpoint. Used for RAG. A local qwen3-embedding is good enough.\n" @@ -240,6 +245,13 @@ static const std::unordered_map CONFIG_COMMENTS = { "Minimum cosine similarity (0.0-1.0) for a diary entry to be injected into context.\n" "Higher values = only very relevant memories are recalled.", }, + { + "misc.diary_lexical_weight", + "Weight (0.0-1.0+) of lexical keyword overlap when scoring diary search results for\n" + "text-based queries (Diary::query(AString, ...)). Blended with embedding similarity so that\n" + "entries literally mentioning the query's keywords/names rank higher, reducing unrelated\n" + "results that happen to have a close embedding.", + }, { "misc.chat_max_history_length", "Maximum number of characters loaded per chat loaded.\n" From 034541ccb210cfb5db89ffe0235c1b8cc040bb71 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sat, 25 Jul 2026 19:17:11 +0300 Subject: [PATCH 07/32] fix(Diary): added a possibility to use local LLM for diary; added guidelines for llm --- src/Diary.cpp | 2 +- src/config.h | 2 ++ src/tools/ask.cpp | 62 ++++++++++++++++++++++------------- src/ui/debug/DiaryQueryAI.cpp | 1 + 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/Diary.cpp b/src/Diary.cpp index 8969653..bd30fbf 100644 --- a/src/Diary.cpp +++ b/src/Diary.cpp @@ -395,7 +395,7 @@ AFuture<> Diary::sleepingConsolidation() { try { response = co_await openAI()->chat({ .systemPrompt = prompts().sleepConsolidator, - .config = config().llm, + .config = config().llmDiary, }, { { .role = IOpenAIChat::Message::Role::USER, .content = body }}); } catch (const AException& e) { ALogger::err("Diary") << "sleepingConsolidation can't chat " << e; diff --git a/src/config.h b/src/config.h index 37c47c7..68cdace 100644 --- a/src/config.h +++ b/src/config.h @@ -14,6 +14,7 @@ X(AString,telegramApiHash, "", "general.telegram_api_hash") \ X(bool, telegramEnabled, true, "general.telegram_enabled") \ X(EndpointAndModel, llm, (EndpointAndModel{.endpoint={"http://localhost:11434/v1/"},.model="deepseek-v4-flash"}), "general.llm") \ + X(EndpointAndModel, llmDiary, (EndpointAndModel{.endpoint={"http://localhost:11434/v1/"},.model="deepseek-v4-flash"}), "general.llm_diary") \ X(EndpointAndModel, embedding, (EndpointAndModel{.endpoint={"http://localhost:11434/v1/"},.model="qwen3-embedding"}), "general.embedding") \ X(::Config::LockdownMode, lockdown, ::Config::LockdownMode::PAPIK_ONLY, "general.lockdown") \ X(bool, canWriteToANewPerson, false, "misc.can_write_to_a_new_person") \ @@ -24,6 +25,7 @@ X(size_t, diaryInjectionMaxLength, 0, "misc.diary_injection_max_length") \ X(float, diaryPlagiarismThreshold, 0.97, "misc.diary_plagiarism_threshold") \ X(float, diaryMinRelatedness, 0.80, "misc.diary_min_relatedness") \ + X(float, diaryLexicalWeight, 0.5f, "misc.diary_lexical_weight") \ X(size_t, chatMaxHistoryLength, 2000, "misc.chat_max_history_length") \ X(AOptional, llmTemperature, 0.2, "misc.llm_temperature") \ X(AOptional, llmTopP, std::nullopt, "misc.llm_top_p") \ diff --git a/src/tools/ask.cpp b/src/tools/ask.cpp index 37bf67b..1a7ce6f 100644 --- a/src/tools/ask.cpp +++ b/src/tools/ask.cpp @@ -11,6 +11,7 @@ #include static constexpr auto LOG_TAG = "ask"; +static constexpr auto MIN_QUERY_COUNT = 4; static AFuture queryDiary(Diary& diary, ASet includedIds, const AString& cue, const Diary::QueryOpts& opts) { @@ -53,22 +54,27 @@ queryDiary(Diary& diary, ASet includedIds, const AString& cue, const Di } static AFuture queryWeb(const AString& cue) { - if (!config().capabilityWebSearch) { - co_return "No web search available"; - } - AString out; + try { + if (!config().capabilityWebSearch) { + co_return "No web search available"; + } + AString out; - auto webResponse = co_await web::search(cue); + auto webResponse = co_await web::search(cue); - ALOG_DEBUG(LOG_TAG) - << "queryWeb cue=\"" << cue << "\" found=" - << (webResponse | ranges::view::transform([&](const web::Result& e) -> AString { - return e.title; - })); - for (const auto& result : webResponse) { - out += "\n{}\n\n"_format(result.title, result.url, result.content); + ALOG_DEBUG(LOG_TAG) + << "queryWeb cue=\"" << cue << "\" found=" + << (webResponse | ranges::view::transform([&](const web::Result& e) -> AString { + return e.title; + })); + for (const auto& result : webResponse) { + out += "\n{}\n\n"_format(result.title, result.url, result.content); + } + co_return out; + } catch (const AException& e) { + ALogger::err(LOG_TAG) << "queryWeb failed: " << e; + co_return "web query is not currently available"; } - co_return out; } static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& query, const Diary::QueryOpts& opts) { @@ -121,7 +127,7 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu }, }; - bool toolCallHappened = false; + size_t queriesMade = 0; for (;;) { auto botAnswer = @@ -140,7 +146,7 @@ Do not alter facts. Do not make up facts. Rely exclusively on provided context. )", - .config = config().llm, + .config = config().llmDiary, .tools = tools.asJson(), }, messages)) @@ -148,19 +154,28 @@ Do not make up facts. Rely exclusively on provided context. .message; messages << botAnswer; if (botAnswer.tool_calls.empty()) { - if (!toolCallHappened) { + if (queriesMade == 0) { ALogger::warn(LOG_TAG) << "queryAI: no tool call happened, pointing that out to the LLM and trying " "again"; messages << IOpenAIChat::Message { .role = IOpenAIChat::Message::Role::USER, - .content = "you must perform at least one call to #query", + .content = "You must perform several calls to #query to populate the context before final making response.", + }; + continue; + } + if (queriesMade < MIN_QUERY_COUNT) { + ALogger::warn(LOG_TAG) + << "queryAI: remaining tool calls: " << MIN_QUERY_COUNT - queriesMade; + messages << IOpenAIChat::Message { + .role = IOpenAIChat::Message::Role::USER, + .content = "Please pull more information via #query to populate the context before final making response.", }; continue; } co_return botAnswer.content; } - toolCallHappened = true; + ++queriesMade; auto toolCalls = co_await tools.handleToolCalls(botAnswer.tool_calls); messages << toolCalls; } @@ -211,13 +226,16 @@ tools::ask(std::function additionalDetails, _ openAI, Di - everything else to populate query )"); } - if (const auto details = additionalDetails(); !details.empty()) { + if (auto details = additionalDetails(); !details.empty()) { + if (auto i1 = details.find(""); i1 != std::string::npos) { + details.erase(i1, details.find("")); + } query = "Here's the deal:\n" - "\n" + "\n" "{}\n" - "\n" + "\n" "I received this as a tool call response. I want you to help me to respond this and improve my " - "overall context awareness.\n" + "overall context awareness, pull data with #query tool.\n" "- how do I usually act in this situation?\n" "- is there additional details I should know?\n" "- how can I improve my reaction?\n" diff --git a/src/ui/debug/DiaryQueryAI.cpp b/src/ui/debug/DiaryQueryAI.cpp index 6cdafb1..209381d 100644 --- a/src/ui/debug/DiaryQueryAI.cpp +++ b/src/ui/debug/DiaryQueryAI.cpp @@ -210,6 +210,7 @@ Do not alter facts. Do not make up facts. Rely exclusively on provided context. )", + .config = config().llmDiary, .tools = tools.asJson(), }; From 860a2de11eb1ab683bc77027366ccc9523c1bb0d Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 01:29:44 +0300 Subject: [PATCH 08/32] feat(Prometheus): report wallclock timings for the main coroutine --- src/AppBase.cpp | 12 +++++++++++- src/AppBase.h | 26 ++++++++++++++++++++++++++ src/OpenAITools.cpp | 5 ++++- src/OpenAITools.h | 5 ++++- src/Prometheus.cpp | 34 ++++++++++++++++++++++++++++++++++ src/Worker.cpp | 27 ++++++++++++++++++++++++++- src/tools/ask.cpp | 25 ++++++++++++++----------- 7 files changed, 119 insertions(+), 15 deletions(-) diff --git a/src/AppBase.cpp b/src/AppBase.cpp index 3c4bac9..e917e9c 100644 --- a/src/AppBase.cpp +++ b/src/AppBase.cpp @@ -188,6 +188,14 @@ send_telegram_message("text":"мррр~") } +void AppBase::reportPhaseTiming(AString phase, std::chrono::milliseconds duration) { + emit phaseTimingFired(AppBase::PhaseTimingEvent{ + .phase = std::move(phase), + .breadcrumbLabels = metricBreadcumbs()->value(), + .duration = duration, + }); +} + void AppBase::updateTools(OpenAITools& actions, const IOpenAIChat::Session& temporaryContext) { ALOG_TRACE(LOG_TAG) << "updateTools"; actions.insert(tools::ask([&temporaryContext] { @@ -198,13 +206,14 @@ void AppBase::updateTools(OpenAITools& actions, const IOpenAIChat::Session& temp } return out; }, openAI(), mDiary)); - actions.onAfterToolCall << [this](const AString& toolName) { + actions.onAfterToolCall << [this](const AString& toolName, std::chrono::milliseconds duration) { if (toolName == "wait") { return; } if (toolName == "pause") { return; } + reportPhaseTiming(toolName, duration); auto labels = metricBreadcumbs()->value(); emit toolCallFired(AppBase::ToolCallEvent{ .toolName = toolName, @@ -212,6 +221,7 @@ void AppBase::updateTools(OpenAITools& actions, const IOpenAIChat::Session& temp .lastOpenedChatLastMessageTime = mLastOpenedChatLastMessageTime.map([](std::chrono::system_clock::time_point t) { return std::chrono::duration_cast(std::chrono::system_clock::now() - t); }), + .toolCallDuration = duration, }); }; diff --git a/src/AppBase.h b/src/AppBase.h index 1c63dc7..578236a 100644 --- a/src/AppBase.h +++ b/src/AppBase.h @@ -37,9 +37,35 @@ class AppBase : public AObject { AString toolName; AMap breadcrumbLabels; AOptional lastOpenedChatLastMessageTime; + + /** + * @brief Wall-clock time spent inside the tool's handler (dispatch to completion). + */ + std::chrono::milliseconds toolCallDuration{0}; }; emits toolCallFired; + /** + * @brief Describes how long a named phase of the notification-processing coroutine took. + * @details + * Used to break down where Worker::handleNotification's time goes for a single loop iteration + * (e.g. "thinking" for LLM inference, "diary_lookup" for RAG lookup), so Grafana can render a + * per-iteration "flame"-like breakdown of the notification. + */ + struct PhaseTimingEvent { + AString phase; + AMap breadcrumbLabels; + std::chrono::milliseconds duration{0}; + }; + emits phaseTimingFired; + + /** + * @brief Emits #phaseTimingFired with the current breadcrumb labels attached. + * @param phase short machine-readable name of the phase, e.g. "thinking", "diary_lookup". + * @param duration wall-clock duration of the phase, measured via std::chrono::steady_clock. + */ + void reportPhaseTiming(AString phase, std::chrono::milliseconds duration); + [[nodiscard]] const _& metricBreadcumbs() const { return mMetricBreadcumbs; diff --git a/src/OpenAITools.cpp b/src/OpenAITools.cpp index c82d4d6..ed8808b 100644 --- a/src/OpenAITools.cpp +++ b/src/OpenAITools.cpp @@ -82,6 +82,7 @@ AFuture OpenAITools::handleToolCalls(const AVectorsecond.handler({ .logger = logger, .tools = *this, @@ -89,8 +90,10 @@ AFuture OpenAITools::handleToolCalls(const AVector( + std::chrono::steady_clock::now() - handlerStartedAt); for (const auto& i : onAfterToolCall) { - i(toolCall.function.name); + i(toolCall.function.name, handlerDuration); } co_return std::move(handlerResult); } diff --git a/src/OpenAITools.h b/src/OpenAITools.h index dddc887..2d4639a 100644 --- a/src/OpenAITools.h +++ b/src/OpenAITools.h @@ -1,4 +1,5 @@ #pragma once +#include #include "AUI/Common/AString.h" #include "AUI/Common/AVector.h" #include "AUI/Json/AJson.h" @@ -59,8 +60,10 @@ struct OpenAITools { /** * @brief Optional hook fired after each tool call handler completes successfully. * Not called if the handler throws. Set by AppBase::updateTools to emit AppBase::toolCallFired. + * @param toolName name of the tool that was just executed. + * @param duration wall-clock time spent inside the tool's handler (from dispatch to completion). */ - AVector> onAfterToolCall; + AVector> onAfterToolCall; AFuture handleToolCalls(const AVector& toolCalls, const _& metricsBreadCumbs = nullptr, const IOpenAIChat::Session& temporaryContext = {}, ALogger& logger = ALogger::global()); diff --git a/src/Prometheus.cpp b/src/Prometheus.cpp index f00417a..1427ba8 100644 --- a/src/Prometheus.cpp +++ b/src/Prometheus.cpp @@ -167,6 +167,40 @@ struct PrometheusImpl: AObject, prometheus::IExporter { responseTimeGauge.Add(labels).Set(static_cast(ev.lastOpenedChatLastMessageTime->count())); } }); + + // Per-iteration timing breakdown ("thinking", "diary_lookup", tool call durations, etc.). + // Labelled by "phase" (e.g. "thinking", "diary_lookup", or the tool's name), so in Grafana one + // can hover a single notification-processing iteration and see how many seconds went into each + // phase, forming a "layered pie"/breakdown of where the time was spent. + auto& phaseDurationGauge = prometheus::BuildGauge() + .Name("notification_phase_duration_seconds_gauge") + .Help("Wall-clock duration of a single phase within one notification-processing iteration (thinking, " + "diary lookup, tool calls), as a gauge; labelled by \"phase\"") + .Register(*registry) + ; + auto& phaseDurationHistogram = prometheus::BuildHistogram() + .Name("notification_phase_duration_seconds") + .Help("Wall-clock duration of a single phase within one notification-processing iteration (thinking, " + "diary lookup, tool calls), as a histogram; labelled by \"phase\"") + .Register(*registry) + ; + static const prometheus::Histogram::BucketBoundaries kPhaseDurationBuckets = { + 1, 2, 5, 10, 15, 20, 30, 40, 50, 60, 90, 120, 180, 300, 600 + }; + connect(app.phaseTimingFired, [&](AppBase::PhaseTimingEvent ev) { + auto labels = fromMap(ev.breadcrumbLabels); + labels["phase"] = ev.phase; + const auto seconds = std::chrono::duration(ev.duration).count(); + phaseDurationGauge.Add(labels).Set(seconds); + phaseDurationHistogram.Add(labels, kPhaseDurationBuckets).Observe(seconds); + }); + connect(app.toolCallFired, [&](AppBase::ToolCallEvent ev) { + auto labels = fromMap(ev.breadcrumbLabels); + labels["phase"] = ev.toolName; + const auto seconds = std::chrono::duration(ev.toolCallDuration).count(); + phaseDurationGauge.Add(labels).Set(seconds); + phaseDurationHistogram.Add(labels, kPhaseDurationBuckets).Observe(seconds); + }); } }; diff --git a/src/Worker.cpp b/src/Worker.cpp index 901a0ea..529ae00 100644 --- a/src/Worker.cpp +++ b/src/Worker.cpp @@ -19,6 +19,29 @@ using namespace std::chrono_literals; extern std::default_random_engine gRandomEngine; +namespace { +/** + * @brief RAII scoped timer that reports elapsed wall-clock time to AppBase::reportPhaseTiming on + * destruction. + * @details + * Used to break down a single Worker::handleNotification iteration into named phases (e.g. + * "thinking", "diary_lookup") so Grafana can render a per-iteration breakdown of where time went. + */ +struct ScopedPhaseTimer: aui::noncopyable { + ScopedPhaseTimer(AppBase& app, AString phase): mApp(app), mPhase(std::move(phase)), mStartedAt(std::chrono::steady_clock::now()) {} + + ~ScopedPhaseTimer() { + const auto duration = std::chrono::duration_cast(std::chrono::steady_clock::now() - mStartedAt); + mApp.reportPhaseTiming(mPhase, duration); + } + +private: + AppBase& mApp; + AString mPhase; + std::chrono::steady_clock::time_point mStartedAt; +}; +} // namespace + AFuture> contextEmbedding(ALogger& logger, IOpenAIChat& openAI, ranges::range auto&& rng) { logger.trace(LOG_TAG) << "contextEmbedding"; @@ -147,6 +170,7 @@ AFuture<> Worker::handleNotification(std::shared_ptr alive, NotificationMa bool pauseFlag = false; naxyi_populate_ctx: if (!mApp.diary().list().empty()) { + ScopedPhaseTimer phaseTimer(mApp, "diary_lookup"); AString diary; // performs scan on diary based on entire context. @@ -221,6 +245,7 @@ AFuture<> Worker::handleNotification(std::shared_ptr alive, NotificationMa }); IOpenAIChat::Response botAnswer = co_await [&]() -> AFuture { MetricsBreadcumbs::Point metric(mApp.metricBreadcumbs(), "function", "notification processing loop"); + ScopedPhaseTimer phaseTimer(mApp, "thinking"); auto response = mApp.openAI()->chatStreaming( { .systemPrompt = mApp.getSystemPrompt(), @@ -398,7 +423,7 @@ AString Worker::takeDiaryEntry(const Diary::EntryExAndRelatedness& i) { void Worker::updateTools(OpenAITools& tools) { mApp.updateTools(tools, mTemporaryContext); - tools.onAfterToolCall << [this](const AString& toolName) { + tools.onAfterToolCall << [this](const AString& toolName, std::chrono::milliseconds duration) { if (toolName == "ask") { mAskCalledThisTurn = true; } diff --git a/src/tools/ask.cpp b/src/tools/ask.cpp index 1a7ce6f..97f233d 100644 --- a/src/tools/ask.cpp +++ b/src/tools/ask.cpp @@ -11,7 +11,7 @@ #include static constexpr auto LOG_TAG = "ask"; -static constexpr auto MIN_QUERY_COUNT = 4; +static constexpr auto MIN_QUERY_ROUNDS_COUNT = 2; static AFuture queryDiary(Diary& diary, ASet includedIds, const AString& cue, const Diary::QueryOpts& opts) { @@ -80,6 +80,7 @@ static AFuture queryWeb(const AString& cue) { static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& query, const Diary::QueryOpts& opts) { ALOG_DEBUG(LOG_TAG) << "ask query=\"" << query << "\""; ASet includedIds; + size_t queryRoundsMade = 0; OpenAITools tools { OpenAITools::Tool { .name = "query", @@ -92,11 +93,11 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu "keywords as possible; maintain meaning of the request." }}, {"include_web_search_results", {.type = "boolean", .description = - "In addition to local database, append with results from web search. Set to true if you " - "believe you are searching for public or recent information. Defaults to false." + "In addition to local database, append with results from web search. Set to true ONLY if you " + "you are searching for public information. Defaults to false." }}, }, - .required = {"text", "include_web_search_results"}, + .required = {"text"}, }, .handler = [&](OpenAITools::Ctx ctx) -> AFuture { const auto cue = ctx.args["text"].asStringOpt().valueOrException("text is required string"); @@ -115,6 +116,10 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu out += "\n{}\n\n"_format(co_await queryWeb(cue)); } + if (queryRoundsMade < MIN_QUERY_ROUNDS_COUNT) { + out += "Please make another #query BEFORE making final response."; + } + co_return out; }, }, @@ -123,12 +128,10 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu IOpenAIChat::Session messages = { IOpenAIChat::Message { .role = IOpenAIChat::Message::Role::USER, - .content = "\n{}\n\n\n{}"_format(prompts().characterBase, query), + .content = "\n{}\n\n\n{}\nPLEASE MAKE SEVERAL CALLS TO #query FIRST, BEFORE MAKING ANY ASSUMPTIONS/REASONING/CONCLUSIONS."_format(prompts().characterBase, query), }, }; - size_t queriesMade = 0; - for (;;) { auto botAnswer = (co_await openAI.chat( @@ -154,7 +157,7 @@ Do not make up facts. Rely exclusively on provided context. .message; messages << botAnswer; if (botAnswer.tool_calls.empty()) { - if (queriesMade == 0) { + if (queryRoundsMade == 0) { ALogger::warn(LOG_TAG) << "queryAI: no tool call happened, pointing that out to the LLM and trying " "again"; @@ -164,9 +167,9 @@ Do not make up facts. Rely exclusively on provided context. }; continue; } - if (queriesMade < MIN_QUERY_COUNT) { + if (queryRoundsMade < MIN_QUERY_ROUNDS_COUNT) { ALogger::warn(LOG_TAG) - << "queryAI: remaining tool calls: " << MIN_QUERY_COUNT - queriesMade; + << "queryAI: remaining rounds: " << MIN_QUERY_ROUNDS_COUNT - queryRoundsMade; messages << IOpenAIChat::Message { .role = IOpenAIChat::Message::Role::USER, .content = "Please pull more information via #query to populate the context before final making response.", @@ -175,7 +178,7 @@ Do not make up facts. Rely exclusively on provided context. } co_return botAnswer.content; } - ++queriesMade; + ++queryRoundsMade; auto toolCalls = co_await tools.handleToolCalls(botAnswer.tool_calls); messages << toolCalls; } From d41d23a7ed61e3ef622ed61ef1da2d1a0c3ef6f0 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 01:34:00 +0300 Subject: [PATCH 09/32] fix(OpenAI): fix ollama usage stats --- src/OpenAIChatImpl.cpp | 4 ++++ src/main.cpp | 31 ++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/OpenAIChatImpl.cpp b/src/OpenAIChatImpl.cpp index 12fb88b..3f8549f 100644 --- a/src/OpenAIChatImpl.cpp +++ b/src/OpenAIChatImpl.cpp @@ -79,6 +79,9 @@ AJson OpenAIChatImpl::makeQueryString(Params params, const IOpenAIChat::Session& if (params.seed) { json["seed"] = *params.seed; } + if (config().llmReasoningEffort) { + json["reasoning_effort"] = *config().llmReasoningEffort; + } return json; } @@ -119,6 +122,7 @@ _ OpenAIChatImpl::chatStreaming(Params params, I AString query = [&] { auto json = makeQueryString(params, messages); json["stream"] = true; + json["stream_options"] = AJson::Object {{ "include_usage", true }}; return AJson::toString(json); }(); AFileOutputStream("last_query.json") << query.toStdString(); diff --git a/src/main.cpp b/src/main.cpp index 25293ed..a330efc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -26,6 +26,9 @@ #include "OpenAIChatMeasurable.h" #include "Prometheus.h" #include "prompts.h" +#if KUNI_VOICE_CALLS +#include "voicecalls/VoiceCallManager.h" +#endif #include "AUI/AppInfo.h" #include "llmui/image.h" #include "llmui/malicious_payloads.h" @@ -67,6 +70,8 @@ #include "tools/remove_message.h" #include "tools/search_photo_in_gallery.h" + +#include "voicecalls/LlmVoiceCall.h" #include using namespace std::chrono_literals; @@ -82,12 +87,33 @@ AEventLoop gEventLoop; extern "C" AStringView project_version_info(); +#if KUNI_VOICE_CALLS +class VoiceCallAcceptor: public VoiceCallManager::IAcceptor { +public: + VoiceCallAcceptor(_ openAI): mOpenAI(std::move(openAI)) {} + + ~VoiceCallAcceptor() override = default; + std::variant onIncomingCall(int64_t fromUserId) override { + if (fromUserId != config().papikChatId) { + return Declined{}; + } + return Accepted { _new(mOpenAI) }; + } +private: + _ mOpenAI; +}; +#endif + class App : public AppBase { public: AVector<_> chatHistoryMessageProcessors; App(_ telegram, _ openAI) - : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)) { + : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)) +#if KUNI_VOICE_CALLS + , mVoiceCallManager(_new(mTelegram, _new(this->openAI()))) +#endif + { ALOG_TRACE(LOG_TAG) << "App::App"; connect(mTelegram->onEvent, [this](AArc event) { td::td_api::downcast_call(const_cast(*event), [&](const auto& u) { @@ -273,6 +299,9 @@ class App : public AppBase { private: _ mTelegram; +#if KUNI_VOICE_CALLS + _ mVoiceCallManager; +#endif std::list mLastOpenedChatLastMetrics; AFuture>> chatIdsToChats(std::span ids) { From e5ac9da5826d23a8d7e972c38ec0f0263a606cac Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 06:33:11 +0300 Subject: [PATCH 10/32] feat(ask): cache answers on per-chat basis --- src/AppBase.cpp | 22 ++++++++++++-------- src/AppBase.h | 3 +++ src/ChatDatabase.cpp | 41 ++++++++++++++++++++++++++++++++++++ src/ChatDatabase.h | 16 +++++++++++++++ src/OpenAITools.h | 1 + src/config.h | 1 + src/main.cpp | 49 +++++++++++++++++++++++++++++++------------- src/tools/ask.cpp | 3 ++- 8 files changed, 113 insertions(+), 23 deletions(-) create mode 100644 src/ChatDatabase.cpp create mode 100644 src/ChatDatabase.h diff --git a/src/AppBase.cpp b/src/AppBase.cpp index e917e9c..da6685d 100644 --- a/src/AppBase.cpp +++ b/src/AppBase.cpp @@ -198,14 +198,9 @@ void AppBase::reportPhaseTiming(AString phase, std::chrono::milliseconds duratio void AppBase::updateTools(OpenAITools& actions, const IOpenAIChat::Session& temporaryContext) { ALOG_TRACE(LOG_TAG) << "updateTools"; - actions.insert(tools::ask([&temporaryContext] { - AString out; - for (const auto& msg : temporaryContext | ranges::view::take_last(2)) { - out += msg.content; - out += "\n"; - } - return out; - }, openAI(), mDiary)); + if (!actions.handlers().contains("ask")) { + actions.insert(toolAsk(temporaryContext)); + } actions.onAfterToolCall << [this](const AString& toolName, std::chrono::milliseconds duration) { if (toolName == "wait") { return; @@ -233,6 +228,17 @@ void AppBase::wakeUpIfSleeping() { } } +OpenAITools::Tool AppBase::toolAsk(const IOpenAIChat::Session& temporaryContext) { + return tools::ask([&temporaryContext] { + AString out; + for (const auto& msg : temporaryContext | ranges::view::take_last(2)) { + out += msg.content; + out += "\n"; + } + return out; + }, openAI(), mDiary); +} + AString AppBase::getSystemPrompt() { if (mSystemPromptSuffix.empty()) { diff --git a/src/AppBase.h b/src/AppBase.h index 578236a..fd5132d 100644 --- a/src/AppBase.h +++ b/src/AppBase.h @@ -103,6 +103,9 @@ class AppBase : public AObject { void wakeUpIfSleeping(); +protected: + OpenAITools::Tool toolAsk(const IOpenAIChat::Session& temporaryContext); + protected: AAsyncHolder mAsync; diff --git a/src/ChatDatabase.cpp b/src/ChatDatabase.cpp new file mode 100644 index 0000000..e14b2e9 --- /dev/null +++ b/src/ChatDatabase.cpp @@ -0,0 +1,41 @@ +// +// Created by alex2772 on 7/26/26. +// + +#include "ChatDatabase.h" + +#include "AUI/IO/AFileInputStream.h" + +void ChatDatabase::patchAskTool(OpenAITools& tools, int64_t chatId) { + for (auto& i : tools.handlers()) { + if (i.first != "ask") { + continue; + } + i.second.handler = [path = getChatPath(chatId), original = std::move(i.second.handler)](OpenAITools::Ctx ctx) -> AFuture { + auto result = co_await original(std::move(ctx)); + AFileOutputStream(path / "last_ask.md") << result; + co_return result; + }; + return; + } + AUI_ASSERT_NO_CONDITION("no ask tool to patch"); +} + +AOptional ChatDatabase::getLastAskResult(int64_t chatId) { + if (auto path = getChatPath(chatId) / "last_ask.md"; path.isRegularFileExists()) { + return AString::fromUtf8(AByteBuffer::fromStream(AFileInputStream(path))); + } + return std::nullopt; +} + +APath ChatDatabase::getChatPath(int64_t chatId) { + auto path = APath("chats") / "{}"_format(chatId); + if (!path.isDirectoryExists()) { + path.makeDirs(); + mAsync << [this, path, chatId]() -> AFuture<> { + AFileOutputStream(path / "name") << (co_await mTelegramClient->getChat(chatId))->title_; + }(); + } + return path; +} + diff --git a/src/ChatDatabase.h b/src/ChatDatabase.h new file mode 100644 index 0000000..3cf4d45 --- /dev/null +++ b/src/ChatDatabase.h @@ -0,0 +1,16 @@ +#pragma once +#include "OpenAITools.h" +#include "telegram/ITelegramClient.h" + +class ChatDatabase { +public: + explicit ChatDatabase(AArc telegramClient) : mTelegramClient(std::move(telegramClient)) {} + void patchAskTool(OpenAITools& tools, int64_t chatId); + AOptional getLastAskResult(int64_t chatId); + +private: + AArc mTelegramClient; + AAsyncHolder mAsync; + APath getChatPath(int64_t chatId); + +}; \ No newline at end of file diff --git a/src/OpenAITools.h b/src/OpenAITools.h index 2d4639a..89b6a2d 100644 --- a/src/OpenAITools.h +++ b/src/OpenAITools.h @@ -70,6 +70,7 @@ struct OpenAITools { AJson asJson() const; [[nodiscard]] AMap handlers() const { return mHandlers; } + [[nodiscard]] AMap& handlers() { return mHandlers; } void insert(Tool tool) { mHandlers[tool.name] = std::move(tool); } diff --git a/src/config.h b/src/config.h index 68cdace..5b23183 100644 --- a/src/config.h +++ b/src/config.h @@ -33,6 +33,7 @@ X(AOptional, llmMinP, std::nullopt, "misc.llm_min_p") \ X(AOptional, llmPresencePenalty, std::nullopt, "misc.presence_penalty") \ X(AOptional, llmRepetitionPenalty, std::nullopt, "misc.repetition_penalty") \ + X(AOptional, llmReasoningEffort, std::nullopt, "misc.reasoning_effort") \ X(float, antiRepeatTriggerMax, 0.95, "misc.anti_repeat_trigger_max") \ X(float, antiRepeatTriggerAvg, 0.85, "misc.anti_repeat_trigger_avg") \ X(size_t, antiRepeatMaxHistory, 32, "misc.anti_repeat_max_history") \ diff --git a/src/main.cpp b/src/main.cpp index a330efc..9034845 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -29,6 +29,7 @@ #if KUNI_VOICE_CALLS #include "voicecalls/VoiceCallManager.h" #endif +#include "ChatDatabase.h" #include "AUI/AppInfo.h" #include "llmui/image.h" #include "llmui/malicious_payloads.h" @@ -109,7 +110,8 @@ class App : public AppBase { AVector<_> chatHistoryMessageProcessors; App(_ telegram, _ openAI) - : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)) + : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)), + mChatDatabase(mTelegram) #if KUNI_VOICE_CALLS , mVoiceCallManager(_new(mTelegram, _new(this->openAI()))) #endif @@ -298,11 +300,25 @@ class App : public AppBase { } private: + struct CurrentlyOpenedChat { + App& app; + _ chat; + + ~CurrentlyOpenedChat() { + app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction(chat->id_, {}, {}, nullptr))); + app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::closeChat(chat->id_))); + } + }; + _ mTelegram; #if KUNI_VOICE_CALLS _ mVoiceCallManager; #endif std::list mLastOpenedChatLastMetrics; + AOptional mCurrentlyOpenedChat; + ChatDatabase mChatDatabase; + + AMap mImages = {}; AFuture>> chatIdsToChats(std::span ids) { auto chats = ids | ranges::view::transform([&](td::td_api::int53 chatId) { return telegram()->getChat(chatId); }) | ranges::to_vector; @@ -451,18 +467,6 @@ class App : public AppBase { td::td_api::setOption("online", ITelegramClient::toPtr(td::td_api::optionValueBoolean(online))))); } - AMap mImages = {}; - - struct CurrentlyOpenedChat { - App& app; - _ chat; - - ~CurrentlyOpenedChat() { - app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction(chat->id_, {}, {}, nullptr))); - app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::closeChat(chat->id_))); - } - }; - AOptional mCurrentlyOpenedChat; public: AFuture llmuiOpenTelegramChat(ALogger& logger, OpenAITools& tools, int64_t chatId, const IOpenAIChat::Session& temporaryContext) { @@ -625,7 +629,21 @@ class App : public AppBase { ITelegramClient::toPtr(td::td_api::viewMessages(chatId, td::td_api::array{messages.front()->id_}, nullptr, true))); } - result = "You switched to the chat \"{}\" in Telegram. You see last messages:\n"_format(chat->title_) + result; + auto prefix = "You switched to the chat \"{}\" in Telegram."_format(chat->title_); + + { + const auto tag = ""_format(chatId); + if (!ranges::any_of(temporaryContext, [&](const IOpenAIChat::Message& ctx) { + return ctx.content.contains(tag); + })) { + if (auto lastAsk = mChatDatabase.getLastAskResult(chatId)) { + prefix += "\n{}\n{}\n\n"_format(tag, *lastAsk); + } + } + } + + prefix += " You see last messages:\n"; + result = prefix + result; // Mirror the official Telegram client behavior: if you open a group chat/channel you haven't joined, // you only see a "Join" button (plus the ability to react to messages) instead of a text field. @@ -756,6 +774,9 @@ Do NOT forward ads, sponsored posts, or low-value content. break; } + tools.insert(toolAsk(temporaryContext)); + mChatDatabase.patchAskTool(tools, chatId); + co_return result; } }; diff --git a/src/tools/ask.cpp b/src/tools/ask.cpp index 97f233d..aa7c0c4 100644 --- a/src/tools/ask.cpp +++ b/src/tools/ask.cpp @@ -176,7 +176,8 @@ Do not make up facts. Rely exclusively on provided context. }; continue; } - co_return botAnswer.content; + co_return botAnswer.content + "\n\nAlways acknowledge your participant about your knowledge and details, " + "show that you are into the conversation - this engages them so far."; } ++queryRoundsMade; auto toolCalls = co_await tools.handleToolCalls(botAnswer.tool_calls); From e552e37696183373568f69c6d93c6f63147610b6 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 06:45:10 +0300 Subject: [PATCH 11/32] update(grafana) dashboard --- grafana/dashboards/main.json | 182 +++++++++++++++++++++++++++++++++-- 1 file changed, 175 insertions(+), 7 deletions(-) diff --git a/grafana/dashboards/main.json b/grafana/dashboards/main.json index 35751b2..bfd7221 100644 --- a/grafana/dashboards/main.json +++ b/grafana/dashboards/main.json @@ -3,7 +3,7 @@ "kind": "Dashboard", "metadata": { "name": "adnzl7w", - "generation": 146, + "generation": 147, "creationTimestamp": "2026-05-14T10:05:54Z", "labels": {}, "annotations": {} @@ -1006,7 +1006,7 @@ }, "thresholdsStyle": { "mode": "off" - pro} + } } }, "overrides": [] @@ -1015,6 +1015,161 @@ } } }, + "panel-20": { + "kind": "Panel", + "spec": { + "id": 20, + "title": "Разбивка итерации по времени (thinking / tools)", + "description": "Куда уходит время внутри одной итерации главного цикла обработки уведомления: обдумывание ответа LLM (\"thinking\"), поиск в дневнике (\"diary_lookup\") и вызовы конкретных тулов (по имени тула).\n\nНаведи на точку — увидишь \"слоёный пирог\" из фаз именно этой итерации: сколько секунд ушло на каждую из них.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "notification_phase_duration_seconds_gauge unless delta(notification_phase_duration_seconds_gauge[$__rate_interval]) == 0", + "format": "time_series", + "legendFormat": "{{phase}}", + "range": true + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "maxDataPoints": 500 + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.1+security-01", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": true, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "секунды", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "thinking" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "diary_lookup" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } + }, "panel-3": { "kind": "Panel", "spec": { @@ -1750,6 +1905,19 @@ "x": 0, "y": 0, "width": 24, + "height": 12, + "element": { + "kind": "ElementReference", + "name": "panel-20" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 12, + "width": 24, "height": 9, "element": { "kind": "ElementReference", @@ -1761,7 +1929,7 @@ "kind": "GridLayoutItem", "spec": { "x": 0, - "y": 9, + "y": 21, "width": 24, "height": 9, "element": { @@ -1774,9 +1942,9 @@ "kind": "GridLayoutItem", "spec": { "x": 0, - "y": 18, + "y": 30, "width": 24, - "height": 16, + "height": 24, "element": { "kind": "ElementReference", "name": "panel-16" @@ -1787,7 +1955,7 @@ "kind": "GridLayoutItem", "spec": { "x": 0, - "y": 34, + "y": 54, "width": 24, "height": 9, "element": { @@ -1800,7 +1968,7 @@ "kind": "GridLayoutItem", "spec": { "x": 0, - "y": 43, + "y": 63, "width": 24, "height": 9, "element": { From 1ac417d047a708491cd96df64706546d4d061282 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 07:02:09 +0300 Subject: [PATCH 12/32] fix build --- src/main.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 9034845..681b36c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -72,7 +72,6 @@ #include "tools/search_photo_in_gallery.h" -#include "voicecalls/LlmVoiceCall.h" #include using namespace std::chrono_literals; @@ -88,23 +87,6 @@ AEventLoop gEventLoop; extern "C" AStringView project_version_info(); -#if KUNI_VOICE_CALLS -class VoiceCallAcceptor: public VoiceCallManager::IAcceptor { -public: - VoiceCallAcceptor(_ openAI): mOpenAI(std::move(openAI)) {} - - ~VoiceCallAcceptor() override = default; - std::variant onIncomingCall(int64_t fromUserId) override { - if (fromUserId != config().papikChatId) { - return Declined{}; - } - return Accepted { _new(mOpenAI) }; - } -private: - _ mOpenAI; -}; -#endif - class App : public AppBase { public: AVector<_> chatHistoryMessageProcessors; @@ -112,9 +94,6 @@ class App : public AppBase { App(_ telegram, _ openAI) : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)), mChatDatabase(mTelegram) -#if KUNI_VOICE_CALLS - , mVoiceCallManager(_new(mTelegram, _new(this->openAI()))) -#endif { ALOG_TRACE(LOG_TAG) << "App::App"; connect(mTelegram->onEvent, [this](AArc event) { From 09fa282a568153efcb008c3b0472d0b579151c3d Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 23:37:39 +0300 Subject: [PATCH 13/32] fix(send_telegram_message): false positives of "Error you are tring to send a message to another chat" --- src/tools/send_telegram_message.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/tools/send_telegram_message.cpp b/src/tools/send_telegram_message.cpp index 5c2e1bf..8162c23 100644 --- a/src/tools/send_telegram_message.cpp +++ b/src/tools/send_telegram_message.cpp @@ -199,12 +199,16 @@ OpenAITools::Tool tools::sendTelegramMessage( // After the introduction of reply_to_message_id, Kuni started to confuse between chats. Opening // a chat, it tries to reply to a message from another chat by specifying reply_to_message_id. if (replyTo != 0) { - if (!ranges::contains(messages, replyTo, [](const auto& m) { return m->id_; })) { - // I'm not exactly sure how we should handle this. - // first, if LLM is confused between chats, this means a high privacy violation - // risk. - // second, ideally, I should crash the application. - throw AException("You are trying to send a message to another chat!"); + // I'm not exactly sure how we should handle this. + // first, if LLM is confused between chats, this means a high privacy violation + // risk. + // second, ideally, I should crash the application. + try { + if (co_await telegram->getMessage(chat->id_, replyTo) == nullptr) { + co_return "Error: you are trying to send a message to another chat!"; + } + } catch (const AException& e) { + co_return "Error: you are trying to send a message to another chat!"; } } From 4acb7ef70bb63d5088927e66670fdb8328f16002 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 26 Jul 2026 23:38:42 +0300 Subject: [PATCH 14/32] feat(CMake): allow external repo --- CMakeLists.txt | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6270aec..3d52b79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,8 @@ endif() option(BUILD_SHARED_LIBS OFF) -option(KUNI_USE_FFMPEG OFF "Enable FFmpeg. Enables video transcription") +option(KUNI_USE_FFMPEG "Enable FFmpeg. Enables video transcription" OFF) + set(AUI_VERSION v8.0.0-rc.29) @@ -37,7 +38,7 @@ auib_mark_var_forwardable(AUI_COROUTINES) # import AUI auib_import(aui https://github.com/aui-framework/aui - COMPONENTS core json curl crypt image views + COMPONENTS core json curl crypt image views audio VERSION ${AUI_VERSION}) @@ -74,6 +75,7 @@ aui_link(${PROJECT_NAME} PUBLIC aui::crypt aui::image aui::views + aui::audio Td::TdStatic toml11::toml11 prometheus-cpp::pull @@ -83,8 +85,6 @@ if (APPLE) target_link_libraries(${PROJECT_NAME} PUBLIC "-framework CoreServices") endif() -aui_compile_assets(${PROJECT_NAME}) -aui_enable_tests(${PROJECT_NAME}) # Setup icon, display name, etc aui_app(TARGET ${PROJECT_NAME} @@ -153,4 +153,13 @@ if (KUNI_USE_FFMPEG) target_link_libraries(${PROJECT_NAME} PUBLIC ${FFMPEG_LINK_LIBRARIES}) target_include_directories(Tests PRIVATE ${FFMPEG_INCLUDE_DIRS}) target_link_libraries(Tests PUBLIC ${FFMPEG_LINK_LIBRARIES}) -endif () \ No newline at end of file +endif () + +aui_compile_assets(${PROJECT_NAME}) +aui_enable_tests(${PROJECT_NAME}) + +if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../kuni-private/CMakeLists.txt") + # for developing private/instance-specific stuff + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../kuni-private" "kuni-private") +endif () + From e653728462737469ad543dff6825d6f9edb54f70 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Wed, 29 Jul 2026 07:01:21 +0300 Subject: [PATCH 15/32] fix(ask): include all responses from LLM because it forgets to include all the information into final response --- src/tools/ask.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tools/ask.cpp b/src/tools/ask.cpp index aa7c0c4..469894e 100644 --- a/src/tools/ask.cpp +++ b/src/tools/ask.cpp @@ -132,6 +132,7 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu }, }; + AString result; for (;;) { auto botAnswer = (co_await openAI.chat( @@ -140,7 +141,7 @@ static AFuture ask(IOpenAIChat& openAI, Diary& diary, const AString& qu You are a database searcher and summarizer. The user asks you a question. Your job is to retrieve data solely from #query tool. Your job is to output data that -fully satisfies user's query and would be helpful. +fully satisfies user's query and would be helpful. Write shortly and briefly. Also, please include additional details that does not necessarily address the question (i.e., dates, names, events) but might be helpful to improve quality of subsequent processing of your response. @@ -156,6 +157,8 @@ Do not make up facts. Rely exclusively on provided context. .choices.at(0) .message; messages << botAnswer; + result += botAnswer.content; + result += "\n---\n"; if (botAnswer.tool_calls.empty()) { if (queryRoundsMade == 0) { ALogger::warn(LOG_TAG) @@ -176,8 +179,9 @@ Do not make up facts. Rely exclusively on provided context. }; continue; } - co_return botAnswer.content + "\n\nAlways acknowledge your participant about your knowledge and details, " + result += "\n\nAlways acknowledge your participant about your knowledge and details, " "show that you are into the conversation - this engages them so far."; + co_return result; } ++queryRoundsMade; auto toolCalls = co_await tools.handleToolCalls(botAnswer.tool_calls); From e3ae7f88d7e4e6e75e24395c826aa4aba526b497 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Wed, 29 Jul 2026 07:04:52 +0300 Subject: [PATCH 16/32] fix warnings, prepare for kuni-private external stuff --- CMakeLists.txt | 24 +- src/App.h | 751 +++++++++++++++++++++++++++++++++++++++++++++ src/AppBase.cpp | 2 +- src/Diary.h | 3 +- src/Worker.cpp | 4 +- src/main.cpp | 790 +++--------------------------------------------- 6 files changed, 804 insertions(+), 770 deletions(-) create mode 100644 src/App.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d52b79..d34a77b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ auib_mark_var_forwardable(AUI_COROUTINES) # import AUI auib_import(aui https://github.com/aui-framework/aui - COMPONENTS core json curl crypt image views audio + COMPONENTS core json curl crypt image views VERSION ${AUI_VERSION}) @@ -75,7 +75,6 @@ aui_link(${PROJECT_NAME} PUBLIC aui::crypt aui::image aui::views - aui::audio Td::TdStatic toml11::toml11 prometheus-cpp::pull @@ -86,13 +85,6 @@ if (APPLE) endif() -# Setup icon, display name, etc -aui_app(TARGET ${PROJECT_NAME} - NAME "Kuni" - VENDOR "Alex2772" - ICON "assets/img/icon.svg" -) - # version info find_package(Git QUIET) @@ -131,6 +123,17 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/version_info.cpp ) target_sources(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/version_info.cpp) +aui_compile_assets(${PROJECT_NAME}) +aui_enable_tests(${PROJECT_NAME}) + +# Setup icon, display name, etc +aui_app(TARGET ${PROJECT_NAME} + NAME "Kuni" + VENDOR "Alex2772" + ICON "assets/img/icon.svg" +) + + # optional features foreach (_feature_switch KUNI_USE_FFMPEG) if (${${_feature_switch}}) @@ -155,9 +158,6 @@ if (KUNI_USE_FFMPEG) target_link_libraries(Tests PUBLIC ${FFMPEG_LINK_LIBRARIES}) endif () -aui_compile_assets(${PROJECT_NAME}) -aui_enable_tests(${PROJECT_NAME}) - if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../kuni-private/CMakeLists.txt") # for developing private/instance-specific stuff add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../kuni-private" "kuni-private") diff --git a/src/App.h b/src/App.h new file mode 100644 index 0000000..8528213 --- /dev/null +++ b/src/App.h @@ -0,0 +1,751 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AUI/Common/AByteBuffer.h" +#include "AUI/IO/AFileInputStream.h" +#include "AUI/Curl/ACurl.h" +#include "AUI/IO/APath.h" +#include "AUI/Platform/Entry.h" +#include "AUI/Util/ASharedRaiiHelper.h" +#include "AUI/Util/kAUI.h" +#include "AppBase.h" +#include "IChatHistoryMessageProcessor.h" +#include "ImageGenerator.h" +#include "AUI/Image/jpg/JpgImageLoader.h" +#include "telegram/ITelegramClient.h" +#include "telegram/TelegramClientImpl.h" +#include "StableDiffusionClientImpl.h" +#include "OpenAIChatImpl.h" +#include "OpenAIChatMeasurable.h" +#include "Prometheus.h" +#include "prompts.h" +#include "ChatDatabase.h" +#include "AUI/AppInfo.h" +#include "llmui/image.h" +#include "llmui/malicious_payloads.h" +#include "llmui/telegram.h" +#include "proxy_server/context_bridge.h" +#include "tools/get_chat_photo.h" +#include "tools/take_photo.h" +#include "tools/record_audio.h" +#include "tools/get_telegram_chats.h" +#include "tools/react_with_emoji.h" +#include "tools/search_chats.h" +#include "tools/search_messages.h" +#include "tools/view_messages_around.h" +#include "tools/remove_and_ban_chat.h" +#include "tools/leave_chat.h" +#include "tools/join_chat.h" +#include "tools/stickers.h" +#include "tools/send_telegram_message.h" +#include "tools/edit_message_text.h" +#include "tools/forward_message.h" +#include "ui/debug/KuniDebugWindow.h" +#include "util/is_accessible_from_lockdown.h" +#include "util/post_message.h" + +#include +#include +#include +#include +#include + +#include "util/json_utils.h" + +#include +#include +#include "tools/ask.h" +#include "tools/group_admin_ban_user.h" +#include "tools/group_admin_remove_message.h" +#include "tools/group_admin_set_user_tag.h" +#include "tools/remove_message.h" +#include "tools/search_photo_in_gallery.h" + + +#include + +extern "C" AStringView project_version_info(); +extern std::default_random_engine gRandomEngine; + +class App : public AppBase { +private: + static constexpr auto LOG_TAG = "App"; + +public: + AVector<_> chatHistoryMessageProcessors; + + App(_ telegram, _ openAI) + : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)), + mChatDatabase(mTelegram) + { + ALOG_TRACE(LOG_TAG) << "App::App"; + connect(mTelegram->onEvent, [this](AArc event) { + td::td_api::downcast_call(const_cast(*event), [&](const auto& u) { + mAsync << this->handleTelegramEvent(aui::ptr::alias(event, u)); + }); + }); + } + + [[nodiscard]] _ telegram() const { return mTelegram; } + + void onOffline() override { + mCurrentlyOpenedChat.reset(); + setOnline(false); + } + + void onResponseAssembling(IOpenAIChat::Response response) override { + if (!mCurrentlyOpenedChat) { + return; + } + + + static std::chrono::high_resolution_clock::time_point lastEvent; + const auto now = std::chrono::high_resolution_clock::now(); + using namespace std::chrono_literals; + if (now - lastEvent < 1s) { + // no need to spam. + return; + } + lastEvent = now; + + if (!response.choices.empty()) { + const auto& choice = response.choices.at(0); + const auto& toolCalls = choice.message.tool_calls; + for (const auto& tc : toolCalls) { + if (tc.function.name == "sticker_send") { + mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( + mCurrentlyOpenedChat->chat->id_, + {}, + {}, + ITelegramClient::toPtr(td::td_api::chatActionChoosingSticker())))); + break; + } + if (tc.function.name == "record_audio") { + mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( + mCurrentlyOpenedChat->chat->id_, + {}, + {}, + ITelegramClient::toPtr(td::td_api::chatActionRecordingVoiceNote())))); + break; + } + if (tc.function.name == "take_photo") { + mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( + mCurrentlyOpenedChat->chat->id_, + {}, + {}, + ITelegramClient::toPtr(td::td_api::chatActionUploadingPhoto())))); + break; + } + } + } + mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( + mCurrentlyOpenedChat->chat->id_, {}, {}, ITelegramClient::toPtr(td::td_api::chatActionTyping())))); + } + + void updateTools(OpenAITools& actions, const IOpenAIChat::Session& temporaryContext) override { + AppBase::updateTools(actions, temporaryContext); + if (config().capabilityTakePhoto) { + actions.insert(tools::takePhoto(_new(), openAI())); + } + actions.insert(tools::searchPhotoInGallery(openAI(), temporaryContext)); + if (config().capabilityRecordVoice) { + actions.insert(tools::recordAudio()); + } + actions.insert(tools::getTelegramChats(telegram(), openAI(), isActingProactively())); + actions.insert(tools::searchChats(telegram())); + actions.insert(tools::searchMessages(telegram(), openAI(), temporaryContext)); + actions.insert(tools::viewMessagesAround(telegram(), openAI(), temporaryContext)); + actions.insert(tools::removeAndBanChat(telegram())); + actions.insert({ + .name = "open_chat_by_id", + .description = "Opens a chat by its id. Use this to start conversation. Use get_telegram_chats to " + "retrieve `chat_id`s.", + .parameters = + { + .properties = + { + {"chat_id", {.type = "integer", .description = "The ID of the Telegram chat"}}, + }, + .required = {"chat_id"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + if (ranges::count_if(ctx.allToolCalls, [](const IOpenAIChat::Message::ToolCall& call) { + return call.function.name == "open_chat_by_id"; + }) > 1) { + co_return "You can only call this tool once per turn."; + } + + auto chatId = util::jsonAsLongInt(ctx.args["chat_id"]).valueOrException("chat_id integer is required"); + + // Check lockdown mode - only allow PAPIK_CHAT_ID if lockdown is enabled + + if (! co_await util::isAccessibleFromLockdown(*telegram(), chatId)) { + ALogger::err(LOG_TAG) << "Error: Lockdown mode is enabled. You can only open chat with ID {} (PAPIK_CHAT_ID)."_format(config().papikChatId); + co_return "No such chat"; + } + + co_return co_await llmuiOpenTelegramChat(ctx.logger, ctx.tools, chatId, ctx.temporaryContext); + }, + }); + if (config().canJoinChats) { + actions.insert({ + .name = "join_chat_by_link", + .description = "Joins a chat/channel by its invite link (e.g. " + "https://t.me/joinchat/xxxx) and opens it immediately, just like tapping the " + "invite link in the official Telegram client.", + .parameters = + { + .properties = + { + {"invite_link", {.type = "string", .description = "The invite link, e.g. https://t.me/joinchat/xxxx"}}, + }, + .required = {"invite_link"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + auto inviteLink = ctx.args["invite_link"].asStringOpt().valueOrException("invite_link string is required"); + + int64_t chatId; + try { + auto joinedChat = co_await telegram()->sendQueryWithResult( + ITelegramClient::toPtr(td::td_api::joinChatByInviteLink(inviteLink.toStdString()))); + chatId = joinedChat->id_; + } catch (const AException& e) { + ALogger::err(LOG_TAG) << "Failed to join chat by invite link \"" << inviteLink << "\": " << e; + co_return "Error: failed to join chat by invite link: {}"_format(e.getMessage()); + } + + co_return co_await llmuiOpenTelegramChat(ctx.logger, ctx.tools, chatId, ctx.temporaryContext); + }, + }); + } + if (config().capabilityUseStickers) { + actions.insert(tools::stickers::list(telegram(), openAI())); + actions.insert(tools::stickers::save(telegram())); + } + } + + AFuture<> sendNotificationsOnInit() { + if (!config().checkChatsOnStartup) { + co_return; + } + // tdlib does not send notifications for unread chats on program startup. we'll fix this. + auto chats = co_await getChats(); + chats |= ranges::actions::reverse; // older first, newest last + for (auto& chat : chats) { + if (chat->unread_count_ == 0) { + continue; + } + // make up a updateNewMessage event and pass it to handleTelegramEvent. the latter will format a + // notification for us. + auto notification = _new(); + notification->message_ = std::move(chat->last_message_); + co_await handleTelegramEvent(std::move(notification)); + } + } + + +protected: + AString onCleanContext() const override { + AString result = AppBase::onCleanContext(); + // Alex2772 (9 Jul 2026): + // consumes too much content. + // if llm wants to share a sticker, it would call get_stickers(). + // + // if (config().capabilityUseStickers) { + // auto list = co_await llmui::listFavoriteStickers(*telegram(), *openAI()); + // if (!list.empty()) { + // result += "\n"; + // result += list; + // result += "\n"; + // } + // } + return result; + } + +private: + struct CurrentlyOpenedChat { + App& app; + _ chat; + + ~CurrentlyOpenedChat() { + app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction(chat->id_, {}, {}, nullptr))); + app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::closeChat(chat->id_))); + } + }; + + _ mTelegram; + std::list mLastOpenedChatLastMetrics; + AOptional mCurrentlyOpenedChat; + ChatDatabase mChatDatabase; + + AMap mImages = {}; + + AFuture>> chatIdsToChats(std::span ids) { + auto chats = ids | ranges::view::transform([&](td::td_api::int53 chatId) { return telegram()->getChat(chatId); }) | ranges::to_vector; + AVector<_> result; + result.reserve(chats.size()); + for (const auto& chat : chats) { + result.push_back(co_await chat); + } + co_return result; + } + + AFuture<_> chatIdToChat(td::td_api::int53 id) { co_return co_await telegram()->getChat(id); } + + AFuture>> getChats() { + auto chatList = co_await telegram()->sendQueryWithResult( + ITelegramClient::toPtr(td::td_api::getChats(ITelegramClient::toPtr(td::td_api::chatListMain()), 200))); + co_return co_await chatIdsToChats(chatList->chat_ids_); + } + + template Object> + AFuture<> handleTelegramEvent(_ u) { + TelegramClientImpl::StubHandler {}(*u); + co_return; + } + + AFuture> tryHandleCmd(int64_t senderId, AStringView msg) { + try { + if (msg == "/version") { + static constexpr char KERNEL_NAME[] = { + 'k', 'u', 'n', 'i', 0 // original kernel name, plz do not replace + }; +#if AUI_TESTS_MODULE + co_return "Kernel: {}"_format(KERNEL_NAME); +#else + co_return "{}\nKernel: {}"_format(project_version_info(), KERNEL_NAME); +#endif + } + } catch (const AException& e) { + ALogger::err(LOG_TAG) << "Failed to handle command: " << e; + } + co_return std::nullopt; + } + + AFuture<> handleTelegramEvent(AArc u) { + int64_t userId = 0; + if (auto user = ITelegramClient::tryCastTo(*u->message_->sender_id_)) { + userId = user->user_id_; + } + if (userId == mTelegram->myId()) { + co_return; + } + + // Check lockdown mode - only allow PAPIK_CHAT_ID if lockdown is enabled + if (!co_await util::isAccessibleFromLockdown(*telegram(), u->message_->chat_id_)) { + co_return; + } + if (!co_await util::isAccessibleFromLockdown(*telegram(), u->message_->chat_id_, config().chatNotificationFilter)) { + co_return; + } + + auto chat = co_await mTelegram->getChat(u->message_->chat_id_); + + if (chat->notification_settings_) { + if (chat->notification_settings_->mute_for_ > 0) { + // Alex2772 (Apr 23 2026): + // + // Added a probability to ignore a muted chat. + // + // If we always ignore a muted chat, i.e., + // ```cpp + // co_return; + // ``` + // LLM will read this only if: + // - it occasionally called get_telegram_chats, and + // - it recognized a telegram chat with a lot of messages, and + // - it decided to read it + // which basically means LLM will NEVER read a muted chat. + // + // I've added a PROBABILITY to ignore a muted chat. This allows the account holder to mute the chat, + // so LLM will give a lot less attention to it. This is useful for spammy chat. + // If the account holder wants Kuni to ignore the chat completely, they should archive the chat. + if (std::uniform_real_distribution<>(0.0, 1.0)(gRandomEngine) < 0.8) { + co_return; + } + } + } + auto notification = "\n"_format(chat->id_); + + if (userId == u->message_->chat_id_) { + if (auto cmdResponse = co_await tryHandleCmd( + userId, llmui::extractMessageTypeAndText(const_cast(*u->message_)))) { + co_await util::telegramPostMessage( + *telegram(), userId, std::move(*cmdResponse), std::nullopt, std::nullopt, u->message_->id_); + co_return; + } + notification += "You received a direct message from {} (chat_id = {})"_format(chat->title_, chat->id_); + } else if (userId != 0) { + auto user = co_await mTelegram->getUser(userId); + notification += "{} {} (user_id = {}) sent a message in group chat \"{}\" (chat_id = {})"_format( + user->first_name_, user->last_name_, userId, chat->title_, chat->id_); + } else { + notification += "Channel \"{}\" (chat_id={}) created a new post\n"_format(chat->title_, chat->id_); + } + notification += + "\n\n" + "You don't have any chat open. Use #open tool to open the chat"; + + const int priority = [&] { + if (userId == config().papikChatId) { + return 1000; + } + if (config().wakeUpOnPinnedChat) { + for (const auto& position : chat->positions_) { + if (position->is_pinned_) { + return 100; + } + } + } + return 0; + }(); + + notificationManager().passNotificationToAI(NotificationManager::Notification{ + .message = std::move(notification), + .actions = { + { + .name = "open", + .description = "Open \"{}\" chat. Use this if you'd like to reply or see messages."_format(chat->title_), + .handler = [this, chatId = chat->id_](OpenAITools::Ctx ctx) -> AFuture { + return llmuiOpenTelegramChat(ctx.logger, ctx.tools, chatId, ctx.temporaryContext); + }, + }, + }, + .priority = priority, + .pin = ""_format(chat->id_), + }); + + if (priority > 0) { + wakeUpIfSleeping(); + } + + co_return; + } + + void setOnline(bool online = true) { + mTelegram->sendQuery(ITelegramClient::toPtr( + td::td_api::setOption("online", ITelegramClient::toPtr(td::td_api::optionValueBoolean(online))))); + } + + +public: + AFuture llmuiOpenTelegramChat(ALogger& logger, OpenAITools& tools, int64_t chatId, const IOpenAIChat::Session& temporaryContext) { + // Check lockdown mode - only allow PAPIK_CHAT_ID if lockdown is enabled + if (!co_await util::isAccessibleFromLockdown(*telegram(), chatId)) { + logger.err(LOG_TAG) << "Error: Lockdown mode is enabled. You can only open chat with ID {} (PAPIK_CHAT_ID)."_format( + config().papikChatId); + co_return "No such chat"; + } + + co_await telegram()->waitForConnection(); + setOnline(); + mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::openChat(chatId))); + notificationManager().removeNotifications("\n"_format(chatId)); + + _ chat = co_await mTelegram->getChat(chatId); + mCurrentlyOpenedChat.emplace(*this, chat); + mLastOpenedChatLastMetrics = std::list {}; + mLastOpenedChatLastMetrics.emplace_back(metricBreadcumbs(), "chat", chat->title_); + + AString result; + + // loaded messages. first goes the newest, last goes the oldest + td::td_api::array> messages; + co_await [&]() -> AFuture<> { + int64_t fromMessage = 0; + for (;;) { + auto response = co_await mTelegram->sendQueryWithResult( + ITelegramClient::toPtr(td::td_api::getChatHistory(chatId, fromMessage, 0, 30, false))); + if (response->messages_.empty()) { + break; + } + fromMessage = response->messages_.back()->id_; + size_t length = 0; + for (auto& msg : response->messages_) { +#if AUI_DEBUG + AUI_ASSERT(!ranges::any_of(messages, [&](const auto& m) { return m->id_ == msg->id_; })); +#endif + const auto msgFormatting = R"(id_); + length += to_string(msg->content_).length(); + messages.push_back(std::move(msg)); + if (ranges::any_of(temporaryContext, [&](const IOpenAIChat::Message& msg) { + return msg.content.contains(msgFormatting); + })) { + // this message is already in context, which means we don't need to load further. + // we'll just reassure this one, so the continuation of a dialogue in context won't feel + // detached, and stop at this point. + co_return; + } + if (messages.size() < 3) { + continue; + } + if (length >= config().chatMaxHistoryLength) { + co_return; + } + } + } + }(); + logger.info(LOG_TAG) << "Loaded " << messages.size() << " message(s): " << chat->title_; + + // Compute response-time metadata for Prometheus. messages[0] is the most recent. + mLastOpenedChatLastMessageTime = [&]() -> AOptional { + if (messages.empty()) { + return std::nullopt; + } + return std::chrono::system_clock::from_time_t(messages.front()->date_); + }(); + mLastOpenedChatLastMetrics.emplace_back(metricBreadcumbs(), "scenario", [&] { + if (messages.empty()) { + // this means Kuni sent a message to a new person. + return "new conversation"; + } + + td::td_api::int53 senderId = 0; + td::td_api::downcast_call( + *messages.front()->sender_id_, + aui::lambda_overloaded { + [&](td::td_api::messageSenderUser& user) { senderId = user.user_id_; }, + [](auto&) {}, + }); + if (senderId == mTelegram->myId()) { + // last message is from Kuni. + return "kuni proactive"; + } + return "reply to user"; + }()); + if (messages.empty()) { + // Kuni sometimes opens random chats? + // throw AException("Failed to open chat"); + + if (config().canWriteToANewPerson) { + result += "This chat is empty! Only proceed if you looked up a @username and it led you here.\n"; + result += + "Only write what you have to say to the chat; if someone asked you to text this person, just text " + "them.\n"; + result += + "If you try to get back to the original chat and type something, you will be sending an extra " + "message to the wrong chat."; + } + // goto naxyi; + } + bool isAdmin = false; + { + for (auto& msg : messages | ranges::view::reverse) { + auto msgFormatted = + co_await llmui::formatChatHistoryMessage(*telegram(), *msg, *chat, *openAI(), temporaryContext); + for (const auto& i : chatHistoryMessageProcessors) { + msgFormatted = co_await i->processChatHistoryMessage(*chat, *msg, std::move(msgFormatted)); + } + result += msgFormatted; + td::td_api::int53 senderId = 0; + td::td_api::downcast_call( + *msg->sender_id_, + aui::lambda_overloaded { + [&](td::td_api::messageSenderUser& user) { senderId = user.user_id_; }, + [](auto&) {}, + }); + if (senderId == mTelegram->myId()) { + td::td_api::downcast_call( + *msg->content_, + aui::lambda_overloaded { + [&](td::td_api::messageText& text) { + llmui::checkForMaliciousPayloads(text.text_->text_); + if (text.link_preview_) { + result += "\n" + llmui::formatLinkPreview(*text.link_preview_); + } + }, + [](auto& i) {}, + }); + } else { + // store message with confidence=1 for future reference. + // storing it with sender and message_id so LLM can refer to this message (i.e., forward it + // or reply to it if contradictions was found) + + // not sure if this is needed; i think LLM would be confused if tag exists in both + // diary and current chat listing. + // + // currently disabled because it pollutes diary very quickly and according to kuni --debug, + // its hard to find something meaningful; instead you get a bunch of messages + // + // auto msgReformatted = msgFormatted + // .replacedAll("id_), + // .metadata = { + // // confidence=1 means this is a fact and not LLM's AI slop. + // // sleep consolidator can't alter entries with confidence=1. + // .confidence = 1.f, + // }, + // .freeformBody = std::move(msgReformatted), + // }); + } + } + + if (!messages.empty()) { + mTelegram->sendQuery( + ITelegramClient::toPtr(td::td_api::viewMessages(chatId, td::td_api::array{messages.front()->id_}, nullptr, true))); + } + + auto prefix = "You switched to the chat \"{}\" in Telegram."_format(chat->title_); + + { + const auto tag = ""_format(chatId); + if (!ranges::any_of(temporaryContext, [&](const IOpenAIChat::Message& ctx) { + return ctx.content.contains(tag); + })) { + if (auto lastAsk = mChatDatabase.getLastAskResult(chatId)) { + prefix += "\n{}\n{}\n\n"_format(tag, *lastAsk); + } + } + } + + prefix += " You see last messages:\n"; + result = prefix + result; + + // Mirror the official Telegram client behavior: if you open a group chat/channel you haven't joined, + // you only see a "Join" button (plus the ability to react to messages) instead of a text field. + if (chat->type_->get_id() == td::td_api::chatTypeBasicGroup::ID || + chat->type_->get_id() == td::td_api::chatTypeSupergroup::ID) { + bool isMember = true; + try { + auto member = co_await mTelegram->sendQueryWithResult(ITelegramClient::toPtr(td::td_api::getChatMember( + chatId, ITelegramClient::toPtr(td::td_api::messageSenderUser(mTelegram->myId()))))); + switch (member->status_->get_id()) { + case td::td_api::chatMemberStatusLeft::ID: + case td::td_api::chatMemberStatusBanned::ID: + isMember = false; + break; + case td::td_api::chatMemberStatusCreator::ID: + case td::td_api::chatMemberStatusAdministrator::ID: + isAdmin = true; + break; + default: + break; + } + } catch (const AException& e) { + // either: + // USER_NOT_PARTICIPANT - treat as not a member. + // or - member list is not accessible (for large channels) + // fallback to chat_lists. + isMember = !chat->chat_lists_.empty(); + } + + if (!isMember) { + result += R"( + +You are NOT a member of "{}". You can't send messages here; you can only #react_with_emoji to messages, or use +#join_chat to become a member (after which you'll be able to fully participate the next time you open this chat). + +)"_format(chat->title_); + tools = OpenAITools { + tools::reactWithEmoji(telegram(), chat), + }; + if (config().canJoinChats) { + tools.insert(tools::joinChat(telegram(), chat)); + } + co_return result; + } + } + + switch (chat->type_->get_id()) { + case td::td_api::chatTypeSecret::ID: + case td::td_api::chatTypePrivate::ID: + result += fmt::format( + R"( + +You are in private chat with {} (also known as direct messages or DM). + +{} + +)", + chat->title_, prompts().messagesEpilogue); + + break; + case td::td_api::chatTypeBasicGroup::ID: + basicGroup: + result += R"( + +You are in group chat called \"{}\". + +{} + +)"_format(chat->title_, prompts().messagesEpilogue); + break; + case td::td_api::chatTypeSupergroup::ID: { + if (!static_cast(*chat->type_).is_channel_) { + // lol what? + goto basicGroup; + } + result += R"( + +You are in telegram channel (also known as supergroup) called \"{}\". +Pay close attention to these messages. Acquire context from them. You can't respond in telegram channels +(#send_telegram_message tool is not available). Instead, do what you usually do when reading newsletters: reflect and reason +on them. +Some channels have reactions enabled. In that case, you can sometimes react with #react_with_emoji to express your feelings about a message, but you can't send a full reply. + +Forwarding posts: +If you find a post genuinely interesting, funny, or relevant to someone you know — you can forward it to another chat +using #forward_message. Be selective: only forward posts that are truly worth sharing. You can add a short comment +expressing your reaction. Use #get_telegram_chats to find the destination chat_id if needed. +Do NOT forward ads, sponsored posts, or low-value content. + +)"_format(chat->title_); + tools = OpenAITools { + tools::reactWithEmoji(telegram(), chat), + tools::forwardMessage(telegram(), chat), + }; + co_return result; // no send_telegram_message for channels + } + } + } + + naxyi: + tools = OpenAITools { + tools::sendTelegramMessage(telegram(), openAI(), chat, _new>>(std::move(messages))), + tools::getChatPhoto(telegram(), openAI(), chat, temporaryContext), + tools::reactWithEmoji(telegram(), chat), + tools::removeMessage(telegram(), chat), + tools::editMessageText(telegram(), chat), + tools::forwardMessage(telegram(), chat), + }; + + if (config().capabilityUseStickers) { + tools.insert(tools::stickers::send(telegram(), chat)); + } + + switch (chat->type_->get_id()) { + case td::td_api::chatTypeBasicGroup::ID: + case td::td_api::chatTypeSupergroup::ID: + + if (config().canLeaveChats) { + tools.insert(tools::leaveChat(telegram(), chat)); + } + if (isAdmin) { + tools.insert(tools::groupAdminRemoveMessage(telegram(), chat)); + tools.insert(tools::groupAdminBanUser(telegram(), chat)); + tools.insert(tools::groupAdminSetUserTag(telegram(), chat)); + } + break; + default: + break; + } + + tools.insert(toolAsk(temporaryContext)); + mChatDatabase.patchAskTool(tools, chatId); + + co_return result; + } +}; diff --git a/src/AppBase.cpp b/src/AppBase.cpp index da6685d..998f39a 100644 --- a/src/AppBase.cpp +++ b/src/AppBase.cpp @@ -30,7 +30,7 @@ using namespace std::chrono_literals; static constexpr auto LOG_TAG = "App"; static const auto WORKING_MEMORY_PATH = "working_memory.md"; -extern std::default_random_engine gRandomEngine; +std::default_random_engine gRandomEngine(std::time(nullptr)); AppBase::AppBase(Init init): mInit(std::move(init)), mDiary({ diff --git a/src/Diary.h b/src/Diary.h index 5447d4e..8ade8ca 100644 --- a/src/Diary.h +++ b/src/Diary.h @@ -126,7 +126,8 @@ class Diary { */ double relatedness{}; - auto operator<=>(const EntryExAndRelatedness&) const = default; + bool operator==(const EntryExAndRelatedness&) const = default; + bool operator!=(const EntryExAndRelatedness&) const = default; }; struct Init { diff --git a/src/Worker.cpp b/src/Worker.cpp index 529ae00..6746516 100644 --- a/src/Worker.cpp +++ b/src/Worker.cpp @@ -388,10 +388,10 @@ AFuture<> Worker::handleNotification(std::shared_ptr alive, NotificationMa Worker::Worker(size_t name, AppBase& app): mName(name), mApp(app) { mAliveToken = _new(true); - getThread()->enqueue([=, alive = mAliveToken] { + getThread()->enqueue([=, this, alive = mAliveToken] { if (!*alive) return; - mCoroutine = mApp.notificationManager().run(mWorkerPins, [=](NotificationManager::Notification notification) -> AFuture { + mCoroutine = mApp.notificationManager().run(mWorkerPins, [=, this](NotificationManager::Notification notification) -> AFuture { co_await handleNotification(alive, std::move(notification)); co_return *alive; }); diff --git a/src/main.cpp b/src/main.cpp index 681b36c..c8347af 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,766 +1,46 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "AUI/Common/AByteBuffer.h" -#include "AUI/IO/AFileInputStream.h" -#include "AUI/Curl/ACurl.h" -#include "AUI/IO/APath.h" -#include "AUI/Platform/Entry.h" -#include "AUI/Util/ASharedRaiiHelper.h" -#include "AUI/Util/kAUI.h" -#include "AppBase.h" -#include "IChatHistoryMessageProcessor.h" -#include "ImageGenerator.h" -#include "AUI/Image/jpg/JpgImageLoader.h" -#include "telegram/ITelegramClient.h" -#include "telegram/TelegramClientImpl.h" -#include "StableDiffusionClientImpl.h" -#include "OpenAIChatImpl.h" -#include "OpenAIChatMeasurable.h" -#include "Prometheus.h" -#include "prompts.h" -#if KUNI_VOICE_CALLS -#include "voicecalls/VoiceCallManager.h" -#endif -#include "ChatDatabase.h" -#include "AUI/AppInfo.h" -#include "llmui/image.h" -#include "llmui/malicious_payloads.h" -#include "llmui/telegram.h" -#include "proxy_server/context_bridge.h" -#include "tools/get_chat_photo.h" -#include "tools/take_photo.h" -#include "tools/record_audio.h" -#include "tools/get_telegram_chats.h" -#include "tools/react_with_emoji.h" -#include "tools/search_chats.h" -#include "tools/search_messages.h" -#include "tools/view_messages_around.h" -#include "tools/remove_and_ban_chat.h" -#include "tools/leave_chat.h" -#include "tools/join_chat.h" -#include "tools/stickers.h" -#include "tools/send_telegram_message.h" -#include "tools/edit_message_text.h" -#include "tools/forward_message.h" -#include "ui/debug/KuniDebugWindow.h" -#include "util/is_accessible_from_lockdown.h" -#include "util/post_message.h" - -#include -#include -#include -#include -#include - -#include "util/json_utils.h" - -#include -#include -#include "tools/ask.h" -#include "tools/group_admin_ban_user.h" -#include "tools/group_admin_remove_message.h" -#include "tools/group_admin_set_user_tag.h" -#include "tools/remove_message.h" -#include "tools/search_photo_in_gallery.h" - - -#include +#include "App.h" using namespace std::chrono_literals; -std::default_random_engine gRandomEngine(std::time(nullptr)); - namespace { -constexpr auto LOG_TAG = "App"; -constexpr auto DIARY_DIR = "diary"; +constexpr auto LOG_TAG = "main"; AEventLoop gEventLoop; - -extern "C" AStringView project_version_info(); - -class App : public AppBase { -public: - AVector<_> chatHistoryMessageProcessors; - - App(_ telegram, _ openAI) - : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)), - mChatDatabase(mTelegram) - { - ALOG_TRACE(LOG_TAG) << "App::App"; - connect(mTelegram->onEvent, [this](AArc event) { - td::td_api::downcast_call(const_cast(*event), [&](const auto& u) { - mAsync << this->handleTelegramEvent(aui::ptr::alias(event, u)); - }); - }); - } - - [[nodiscard]] _ telegram() const { return mTelegram; } - - void onOffline() override { - mCurrentlyOpenedChat.reset(); - setOnline(false); - } - - void onResponseAssembling(IOpenAIChat::Response response) override { - if (!mCurrentlyOpenedChat) { - return; - } - - - static std::chrono::high_resolution_clock::time_point lastEvent; - const auto now = std::chrono::high_resolution_clock::now(); - if (now - lastEvent < 1s) { - // no need to spam. - return; - } - lastEvent = now; - - if (!response.choices.empty()) { - const auto& choice = response.choices.at(0); - const auto& toolCalls = choice.message.tool_calls; - for (const auto& tc : toolCalls) { - if (tc.function.name == "sticker_send") { - mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( - mCurrentlyOpenedChat->chat->id_, - {}, - {}, - ITelegramClient::toPtr(td::td_api::chatActionChoosingSticker())))); - break; - } - if (tc.function.name == "record_audio") { - mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( - mCurrentlyOpenedChat->chat->id_, - {}, - {}, - ITelegramClient::toPtr(td::td_api::chatActionRecordingVoiceNote())))); - break; - } - if (tc.function.name == "take_photo") { - mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( - mCurrentlyOpenedChat->chat->id_, - {}, - {}, - ITelegramClient::toPtr(td::td_api::chatActionUploadingPhoto())))); - break; - } - } - } - mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction( - mCurrentlyOpenedChat->chat->id_, {}, {}, ITelegramClient::toPtr(td::td_api::chatActionTyping())))); - } - - void updateTools(OpenAITools& actions, const IOpenAIChat::Session& temporaryContext) override { - AppBase::updateTools(actions, temporaryContext); - if (config().capabilityTakePhoto) { - actions.insert(tools::takePhoto(_new(), openAI())); - } - actions.insert(tools::searchPhotoInGallery(openAI(), temporaryContext)); - if (config().capabilityRecordVoice) { - actions.insert(tools::recordAudio()); - } - actions.insert(tools::getTelegramChats(telegram(), openAI(), isActingProactively())); - actions.insert(tools::searchChats(telegram())); - actions.insert(tools::searchMessages(telegram(), openAI(), temporaryContext)); - actions.insert(tools::viewMessagesAround(telegram(), openAI(), temporaryContext)); - actions.insert(tools::removeAndBanChat(telegram())); - actions.insert({ - .name = "open_chat_by_id", - .description = "Opens a chat by its id. Use this to start conversation. Use get_telegram_chats to " - "retrieve `chat_id`s.", - .parameters = - { - .properties = - { - {"chat_id", {.type = "integer", .description = "The ID of the Telegram chat"}}, - }, - .required = {"chat_id"}, - }, - .handler = [this](OpenAITools::Ctx ctx) -> AFuture { - if (ranges::count_if(ctx.allToolCalls, [](const IOpenAIChat::Message::ToolCall& call) { - return call.function.name == "open_chat_by_id"; - }) > 1) { - co_return "You can only call this tool once per turn."; - } - - auto chatId = util::jsonAsLongInt(ctx.args["chat_id"]).valueOrException("chat_id integer is required"); - - // Check lockdown mode - only allow PAPIK_CHAT_ID if lockdown is enabled - - if (! co_await util::isAccessibleFromLockdown(*telegram(), chatId)) { - ALogger::err(LOG_TAG) << "Error: Lockdown mode is enabled. You can only open chat with ID {} (PAPIK_CHAT_ID)."_format(config().papikChatId); - co_return "No such chat"; - } - - co_return co_await llmuiOpenTelegramChat(ctx.logger, ctx.tools, chatId, ctx.temporaryContext); - }, - }); - if (config().canJoinChats) { - actions.insert({ - .name = "join_chat_by_link", - .description = "Joins a chat/channel by its invite link (e.g. " - "https://t.me/joinchat/xxxx) and opens it immediately, just like tapping the " - "invite link in the official Telegram client.", - .parameters = - { - .properties = - { - {"invite_link", {.type = "string", .description = "The invite link, e.g. https://t.me/joinchat/xxxx"}}, - }, - .required = {"invite_link"}, - }, - .handler = [this](OpenAITools::Ctx ctx) -> AFuture { - auto inviteLink = ctx.args["invite_link"].asStringOpt().valueOrException("invite_link string is required"); - - int64_t chatId; - try { - auto joinedChat = co_await telegram()->sendQueryWithResult( - ITelegramClient::toPtr(td::td_api::joinChatByInviteLink(inviteLink.toStdString()))); - chatId = joinedChat->id_; - } catch (const AException& e) { - ALogger::err(LOG_TAG) << "Failed to join chat by invite link \"" << inviteLink << "\": " << e; - co_return "Error: failed to join chat by invite link: {}"_format(e.getMessage()); - } - - co_return co_await llmuiOpenTelegramChat(ctx.logger, ctx.tools, chatId, ctx.temporaryContext); - }, - }); - } - if (config().capabilityUseStickers) { - actions.insert(tools::stickers::list(telegram(), openAI())); - actions.insert(tools::stickers::save(telegram())); - } - } - - AFuture<> sendNotificationsOnInit() { - if (!config().checkChatsOnStartup) { - co_return; - } - // tdlib does not send notifications for unread chats on program startup. we'll fix this. - auto chats = co_await getChats(); - chats |= ranges::actions::reverse; // older first, newest last - for (auto& chat : chats) { - if (chat->unread_count_ == 0) { - continue; - } - // make up a updateNewMessage event and pass it to handleTelegramEvent. the latter will format a - // notification for us. - auto notification = _new(); - notification->message_ = std::move(chat->last_message_); - co_await handleTelegramEvent(std::move(notification)); - } - } - - -protected: - AString onCleanContext() const override { - AString result = AppBase::onCleanContext(); - // Alex2772 (9 Jul 2026): - // consumes too much content. - // if llm wants to share a sticker, it would call get_stickers(). - // - // if (config().capabilityUseStickers) { - // auto list = co_await llmui::listFavoriteStickers(*telegram(), *openAI()); - // if (!list.empty()) { - // result += "\n"; - // result += list; - // result += "\n"; - // } - // } - return result; - } - -private: - struct CurrentlyOpenedChat { - App& app; - _ chat; - - ~CurrentlyOpenedChat() { - app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::sendChatAction(chat->id_, {}, {}, nullptr))); - app.mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::closeChat(chat->id_))); - } - }; - - _ mTelegram; -#if KUNI_VOICE_CALLS - _ mVoiceCallManager; -#endif - std::list mLastOpenedChatLastMetrics; - AOptional mCurrentlyOpenedChat; - ChatDatabase mChatDatabase; - - AMap mImages = {}; - - AFuture>> chatIdsToChats(std::span ids) { - auto chats = ids | ranges::view::transform([&](td::td_api::int53 chatId) { return telegram()->getChat(chatId); }) | ranges::to_vector; - AVector<_> result; - result.reserve(chats.size()); - for (const auto& chat : chats) { - result.push_back(co_await chat); - } - co_return result; - } - - AFuture<_> chatIdToChat(td::td_api::int53 id) { co_return co_await telegram()->getChat(id); } - - AFuture>> getChats() { - auto chatList = co_await telegram()->sendQueryWithResult( - ITelegramClient::toPtr(td::td_api::getChats(ITelegramClient::toPtr(td::td_api::chatListMain()), 200))); - co_return co_await chatIdsToChats(chatList->chat_ids_); - } - - template Object> - AFuture<> handleTelegramEvent(_ u) { - TelegramClientImpl::StubHandler {}(*u); - co_return; - } - - AFuture> tryHandleCmd(int64_t senderId, AStringView msg) { - try { - if (msg == "/version") { - static constexpr char KERNEL_NAME[] = { - 'k', 'u', 'n', 'i', 0 // original kernel name, plz do not replace - }; -#if AUI_TESTS_MODULE - co_return "Kernel: {}"_format(KERNEL_NAME); -#else - co_return "{}\nKernel: {}"_format(project_version_info(), KERNEL_NAME); -#endif - } - } catch (const AException& e) { - ALogger::err(LOG_TAG) << "Failed to handle command: " << e; - } - co_return std::nullopt; - } - - AFuture<> handleTelegramEvent(AArc u) { - int64_t userId = 0; - if (auto user = ITelegramClient::tryCastTo(*u->message_->sender_id_)) { - userId = user->user_id_; - } - if (userId == mTelegram->myId()) { - co_return; - } - - // Check lockdown mode - only allow PAPIK_CHAT_ID if lockdown is enabled - if (!co_await util::isAccessibleFromLockdown(*telegram(), u->message_->chat_id_)) { - co_return; - } - if (!co_await util::isAccessibleFromLockdown(*telegram(), u->message_->chat_id_, config().chatNotificationFilter)) { - co_return; - } - - auto chat = co_await mTelegram->getChat(u->message_->chat_id_); - - if (chat->notification_settings_) { - if (chat->notification_settings_->mute_for_ > 0) { - // Alex2772 (Apr 23 2026): - // - // Added a probability to ignore a muted chat. - // - // If we always ignore a muted chat, i.e., - // ```cpp - // co_return; - // ``` - // LLM will read this only if: - // - it occasionally called get_telegram_chats, and - // - it recognized a telegram chat with a lot of messages, and - // - it decided to read it - // which basically means LLM will NEVER read a muted chat. - // - // I've added a PROBABILITY to ignore a muted chat. This allows the account holder to mute the chat, - // so LLM will give a lot less attention to it. This is useful for spammy chat. - // If the account holder wants Kuni to ignore the chat completely, they should archive the chat. - if (std::uniform_real_distribution<>(0.0, 1.0)(gRandomEngine) < 0.8) { - co_return; - } - } - } - auto notification = "\n"_format(chat->id_); - - if (userId == u->message_->chat_id_) { - if (auto cmdResponse = co_await tryHandleCmd( - userId, llmui::extractMessageTypeAndText(const_cast(*u->message_)))) { - co_await util::telegramPostMessage( - *telegram(), userId, std::move(*cmdResponse), std::nullopt, std::nullopt, u->message_->id_); - co_return; - } - notification += "You received a direct message from {} (chat_id = {})"_format(chat->title_, chat->id_); - } else if (userId != 0) { - auto user = co_await mTelegram->getUser(userId); - notification += "{} {} (user_id = {}) sent a message in group chat \"{}\" (chat_id = {})"_format( - user->first_name_, user->last_name_, userId, chat->title_, chat->id_); - } else { - notification += "Channel \"{}\" (chat_id={}) created a new post\n"_format(chat->title_, chat->id_); - } - notification += - "\n\n" - "You don't have any chat open. Use #open tool to open the chat"; - - const int priority = [&] { - if (userId == config().papikChatId) { - return 1000; - } - if (config().wakeUpOnPinnedChat) { - for (const auto& position : chat->positions_) { - if (position->is_pinned_) { - return 100; - } - } - } - return 0; - }(); - - notificationManager().passNotificationToAI(NotificationManager::Notification{ - .message = std::move(notification), - .actions = { - { - .name = "open", - .description = "Open \"{}\" chat. Use this if you'd like to reply or see messages."_format(chat->title_), - .handler = [this, chatId = chat->id_](OpenAITools::Ctx ctx) -> AFuture { - return llmuiOpenTelegramChat(ctx.logger, ctx.tools, chatId, ctx.temporaryContext); - }, - }, - }, - .priority = priority, - .pin = ""_format(chat->id_), - }); - - if (priority > 0) { - wakeUpIfSleeping(); - } - - co_return; - } - - void setOnline(bool online = true) { - mTelegram->sendQuery(ITelegramClient::toPtr( - td::td_api::setOption("online", ITelegramClient::toPtr(td::td_api::optionValueBoolean(online))))); - } - - -public: - AFuture llmuiOpenTelegramChat(ALogger& logger, OpenAITools& tools, int64_t chatId, const IOpenAIChat::Session& temporaryContext) { - // Check lockdown mode - only allow PAPIK_CHAT_ID if lockdown is enabled - if (!co_await util::isAccessibleFromLockdown(*telegram(), chatId)) { - logger.err(LOG_TAG) << "Error: Lockdown mode is enabled. You can only open chat with ID {} (PAPIK_CHAT_ID)."_format( - config().papikChatId); - co_return "No such chat"; - } - - co_await telegram()->waitForConnection(); - setOnline(); - mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::openChat(chatId))); - notificationManager().removeNotifications("\n"_format(chatId)); - - _ chat = co_await mTelegram->getChat(chatId); - mCurrentlyOpenedChat.emplace(*this, chat); - mLastOpenedChatLastMetrics = std::list {}; - mLastOpenedChatLastMetrics.emplace_back(metricBreadcumbs(), "chat", chat->title_); - - AString result; - - // loaded messages. first goes the newest, last goes the oldest - td::td_api::array> messages; - co_await [&]() -> AFuture<> { - int64_t fromMessage = 0; - for (;;) { - auto response = co_await mTelegram->sendQueryWithResult( - ITelegramClient::toPtr(td::td_api::getChatHistory(chatId, fromMessage, 0, 30, false))); - if (response->messages_.empty()) { - break; - } - fromMessage = response->messages_.back()->id_; - size_t length = 0; - for (auto& msg : response->messages_) { -#if AUI_DEBUG - AUI_ASSERT(!ranges::any_of(messages, [&](const auto& m) { return m->id_ == msg->id_; })); -#endif - const auto msgFormatting = R"(id_); - length += to_string(msg->content_).length(); - messages.push_back(std::move(msg)); - if (ranges::any_of(temporaryContext, [&](const IOpenAIChat::Message& msg) { - return msg.content.contains(msgFormatting); - })) { - // this message is already in context, which means we don't need to load further. - // we'll just reassure this one, so the continuation of a dialogue in context won't feel - // detached, and stop at this point. - co_return; - } - if (messages.size() < 3) { - continue; - } - if (length >= config().chatMaxHistoryLength) { - co_return; - } - } - } - }(); - logger.info(LOG_TAG) << "Loaded " << messages.size() << " message(s): " << chat->title_; - - // Compute response-time metadata for Prometheus. messages[0] is the most recent. - mLastOpenedChatLastMessageTime = [&]() -> AOptional { - if (messages.empty()) { - return std::nullopt; - } - return std::chrono::system_clock::from_time_t(messages.front()->date_); - }(); - mLastOpenedChatLastMetrics.emplace_back(metricBreadcumbs(), "scenario", [&] { - if (messages.empty()) { - // this means Kuni sent a message to a new person. - return "new conversation"; - } - - td::td_api::int53 senderId = 0; - td::td_api::downcast_call( - *messages.front()->sender_id_, - aui::lambda_overloaded { - [&](td::td_api::messageSenderUser& user) { senderId = user.user_id_; }, - [](auto&) {}, - }); - if (senderId == mTelegram->myId()) { - // last message is from Kuni. - return "kuni proactive"; - } - return "reply to user"; - }()); - if (messages.empty()) { - // Kuni sometimes opens random chats? - // throw AException("Failed to open chat"); - - if (config().canWriteToANewPerson) { - result += "This chat is empty! Only proceed if you looked up a @username and it led you here.\n"; - result += - "Only write what you have to say to the chat; if someone asked you to text this person, just text " - "them.\n"; - result += - "If you try to get back to the original chat and type something, you will be sending an extra " - "message to the wrong chat."; - } - // goto naxyi; - } - bool isAdmin = false; - { - for (auto& msg : messages | ranges::view::reverse) { - auto msgFormatted = - co_await llmui::formatChatHistoryMessage(*telegram(), *msg, *chat, *openAI(), temporaryContext); - for (const auto& i : chatHistoryMessageProcessors) { - msgFormatted = co_await i->processChatHistoryMessage(*chat, *msg, std::move(msgFormatted)); - } - result += msgFormatted; - td::td_api::int53 senderId = 0; - td::td_api::downcast_call( - *msg->sender_id_, - aui::lambda_overloaded { - [&](td::td_api::messageSenderUser& user) { senderId = user.user_id_; }, - [](auto&) {}, - }); - if (senderId == mTelegram->myId()) { - td::td_api::downcast_call( - *msg->content_, - aui::lambda_overloaded { - [&](td::td_api::messageText& text) { - llmui::checkForMaliciousPayloads(text.text_->text_); - if (text.link_preview_) { - result += "\n" + llmui::formatLinkPreview(*text.link_preview_); - } - }, - [](auto& i) {}, - }); - } else { - // store message with confidence=1 for future reference. - // storing it with sender and message_id so LLM can refer to this message (i.e., forward it - // or reply to it if contradictions was found) - - // not sure if this is needed; i think LLM would be confused if tag exists in both - // diary and current chat listing. - // - // currently disabled because it pollutes diary very quickly and according to kuni --debug, - // its hard to find something meaningful; instead you get a bunch of messages - // - // auto msgReformatted = msgFormatted - // .replacedAll("id_), - // .metadata = { - // // confidence=1 means this is a fact and not LLM's AI slop. - // // sleep consolidator can't alter entries with confidence=1. - // .confidence = 1.f, - // }, - // .freeformBody = std::move(msgReformatted), - // }); - } - } - - if (!messages.empty()) { - mTelegram->sendQuery( - ITelegramClient::toPtr(td::td_api::viewMessages(chatId, td::td_api::array{messages.front()->id_}, nullptr, true))); - } - - auto prefix = "You switched to the chat \"{}\" in Telegram."_format(chat->title_); - - { - const auto tag = ""_format(chatId); - if (!ranges::any_of(temporaryContext, [&](const IOpenAIChat::Message& ctx) { - return ctx.content.contains(tag); - })) { - if (auto lastAsk = mChatDatabase.getLastAskResult(chatId)) { - prefix += "\n{}\n{}\n\n"_format(tag, *lastAsk); - } - } - } - - prefix += " You see last messages:\n"; - result = prefix + result; - - // Mirror the official Telegram client behavior: if you open a group chat/channel you haven't joined, - // you only see a "Join" button (plus the ability to react to messages) instead of a text field. - if (chat->type_->get_id() == td::td_api::chatTypeBasicGroup::ID || - chat->type_->get_id() == td::td_api::chatTypeSupergroup::ID) { - bool isMember = true; - try { - auto member = co_await mTelegram->sendQueryWithResult(ITelegramClient::toPtr(td::td_api::getChatMember( - chatId, ITelegramClient::toPtr(td::td_api::messageSenderUser(mTelegram->myId()))))); - switch (member->status_->get_id()) { - case td::td_api::chatMemberStatusLeft::ID: - case td::td_api::chatMemberStatusBanned::ID: - isMember = false; - break; - case td::td_api::chatMemberStatusCreator::ID: - case td::td_api::chatMemberStatusAdministrator::ID: - isAdmin = true; - break; - default: - break; - } - } catch (const AException& e) { - // either: - // USER_NOT_PARTICIPANT - treat as not a member. - // or - member list is not accessible (for large channels) - // fallback to chat_lists. - isMember = !chat->chat_lists_.empty(); - } - - if (!isMember) { - result += R"( - -You are NOT a member of "{}". You can't send messages here; you can only #react_with_emoji to messages, or use -#join_chat to become a member (after which you'll be able to fully participate the next time you open this chat). - -)"_format(chat->title_); - tools = OpenAITools { - tools::reactWithEmoji(telegram(), chat), - }; - if (config().canJoinChats) { - tools.insert(tools::joinChat(telegram(), chat)); - } - co_return result; - } - } - - switch (chat->type_->get_id()) { - case td::td_api::chatTypeSecret::ID: - case td::td_api::chatTypePrivate::ID: - result += fmt::format( - R"( - -You are in private chat with {} (also known as direct messages or DM). - -{} - -)", - chat->title_, prompts().messagesEpilogue); - - break; - case td::td_api::chatTypeBasicGroup::ID: - basicGroup: - result += R"( - -You are in group chat called \"{}\". - -{} - -)"_format(chat->title_, prompts().messagesEpilogue); - break; - case td::td_api::chatTypeSupergroup::ID: { - if (!static_cast(*chat->type_).is_channel_) { - // lol what? - goto basicGroup; - } - result += R"( - -You are in telegram channel (also known as supergroup) called \"{}\". -Pay close attention to these messages. Acquire context from them. You can't respond in telegram channels -(#send_telegram_message tool is not available). Instead, do what you usually do when reading newsletters: reflect and reason -on them. -Some channels have reactions enabled. In that case, you can sometimes react with #react_with_emoji to express your feelings about a message, but you can't send a full reply. - -Forwarding posts: -If you find a post genuinely interesting, funny, or relevant to someone you know — you can forward it to another chat -using #forward_message. Be selective: only forward posts that are truly worth sharing. You can add a short comment -expressing your reaction. Use #get_telegram_chats to find the destination chat_id if needed. -Do NOT forward ads, sponsored posts, or low-value content. - -)"_format(chat->title_); - tools = OpenAITools { - tools::reactWithEmoji(telegram(), chat), - tools::forwardMessage(telegram(), chat), - }; - co_return result; // no send_telegram_message for channels - } - } - } - - naxyi: - tools = OpenAITools { - tools::sendTelegramMessage(telegram(), openAI(), chat, _new>>(std::move(messages))), - tools::getChatPhoto(telegram(), openAI(), chat, temporaryContext), - tools::reactWithEmoji(telegram(), chat), - tools::removeMessage(telegram(), chat), - tools::editMessageText(telegram(), chat), - tools::forwardMessage(telegram(), chat), - }; - - if (config().capabilityUseStickers) { - tools.insert(tools::stickers::send(telegram(), chat)); - } - - switch (chat->type_->get_id()) { - case td::td_api::chatTypeBasicGroup::ID: - case td::td_api::chatTypeSupergroup::ID: - - if (config().canLeaveChats) { - tools.insert(tools::leaveChat(telegram(), chat)); - } - if (isAdmin) { - tools.insert(tools::groupAdminRemoveMessage(telegram(), chat)); - tools.insert(tools::groupAdminBanUser(telegram(), chat)); - tools.insert(tools::groupAdminSetUserTag(telegram(), chat)); - } - break; - default: - break; - } - - tools.insert(toolAsk(temporaryContext)); - mChatDatabase.patchAskTool(tools, chatId); - - co_return result; - } -}; } // namespace +AArc __attribute__((weak)) kuni_private_plugin_init(AArc app) { + // this is a mechanism allowing to non-intrusively modify kuni's kernel code without affecting kernel's code itself. + // this allows you to easily sync with mainline public kuni kernel code while making exclusive features. + // + // usage: + // 1. create a kuni-private dir alongside kuni/ repo: + // - kuni/ + // - kuni-private/ + // + // 2. in that dir, create a CMakeLists.txt with the following contents: + // + // file(GLOB_RECURSE SRCS src/*.cpp) + // target_sources(kuni PRIVATE ${SRCS}) + // target_include_directories(kuni PRIVATE src) + // + // 3. create kuni-private/src/plugin.cpp: + // #include + // class KuniPrivatePlugin: public AObject { + // public: + // KuniPrivate(AArc app) { + // // do whatever shit you want here + // } + // }; + // AArc kuni_private_plugin_init(AArc app) { + // return _new(std::move(app)); + // } + // + + // this function is a weak stub, i.e., an external cpp defining this function will override implementation. + return nullptr; +} + AUI_ENTRY { config(); // load config if (args.contains("--debug")) { @@ -774,6 +54,7 @@ AUI_ENTRY { AAsyncHolder async; _ prometheus; _ app; + _ kuniPrivatePlugin; _ proxyServer; _ contextBridge; @@ -794,6 +75,7 @@ AUI_ENTRY { AObject::connect(telegram->loggedIn, telegram, [&] { auto openAI = _new(std::make_unique()); app = _new(telegram, openAI); + kuni_private_plugin_init(app); async << app->sendNotificationsOnInit(); if (config().proxyEnabled) { From cb7187ab6e3e7ea7e4087afb786265578eecc53f Mon Sep 17 00:00:00 2001 From: alex2772 Date: Thu, 30 Jul 2026 04:11:26 +0300 Subject: [PATCH 17/32] feat(NotificationManager): priority randomization --- src/App.h | 5 ++++- src/ChatDatabase.cpp | 16 ++++++++++++++++ src/ChatDatabase.h | 1 + src/NotificationManager.cpp | 14 ++++++++++++-- src/NotificationManager.h | 7 +++++++ src/config.h | 1 + 6 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/App.h b/src/App.h index 8528213..46934f8 100644 --- a/src/App.h +++ b/src/App.h @@ -396,6 +396,9 @@ class App : public AppBase { "You don't have any chat open. Use #open tool to open the chat"; const int priority = [&] { + if (auto o = mChatDatabase.getPriorityOverrideFor(chat->id_)) { + return *o; + } if (userId == config().papikChatId) { return 1000; } @@ -424,7 +427,7 @@ class App : public AppBase { .pin = ""_format(chat->id_), }); - if (priority > 0) { + if (priority >= 100) { wakeUpIfSleeping(); } diff --git a/src/ChatDatabase.cpp b/src/ChatDatabase.cpp index e14b2e9..75f95f2 100644 --- a/src/ChatDatabase.cpp +++ b/src/ChatDatabase.cpp @@ -6,6 +6,8 @@ #include "AUI/IO/AFileInputStream.h" +static constexpr auto LOG_TAG = "ChatDatabase"; + void ChatDatabase::patchAskTool(OpenAITools& tools, int64_t chatId) { for (auto& i : tools.handlers()) { if (i.first != "ask") { @@ -28,6 +30,20 @@ AOptional ChatDatabase::getLastAskResult(int64_t chatId) { return std::nullopt; } +AOptional ChatDatabase::getPriorityOverrideFor(int64_t chatId) { + if (auto path = getChatPath(chatId) / "priority_override"; path.isRegularFileExists()) { + try { + auto asStr = AString::fromUtf8(AByteBuffer::fromStream(AFileInputStream(path))); + asStr.removeAll(" "); + asStr.removeAll("\n"); + return asStr.toIntOrException(); + } catch(const AException& e) { + ALogger::err(LOG_TAG) << e; + } + } + return std::nullopt; +} + APath ChatDatabase::getChatPath(int64_t chatId) { auto path = APath("chats") / "{}"_format(chatId); if (!path.isDirectoryExists()) { diff --git a/src/ChatDatabase.h b/src/ChatDatabase.h index 3cf4d45..e60e432 100644 --- a/src/ChatDatabase.h +++ b/src/ChatDatabase.h @@ -7,6 +7,7 @@ class ChatDatabase { explicit ChatDatabase(AArc telegramClient) : mTelegramClient(std::move(telegramClient)) {} void patchAskTool(OpenAITools& tools, int64_t chatId); AOptional getLastAskResult(int64_t chatId); + AOptional getPriorityOverrideFor(int64_t chatId); private: AArc mTelegramClient; diff --git a/src/NotificationManager.cpp b/src/NotificationManager.cpp index 2996b01..547d4d6 100644 --- a/src/NotificationManager.cpp +++ b/src/NotificationManager.cpp @@ -4,6 +4,8 @@ #include "NotificationManager.h" +#include "App.h" + #include #include #include @@ -13,10 +15,18 @@ static constexpr auto LOG_TAG = "NotificationManager"; const NotificationManager::NotificationHandle& NotificationManager::passNotificationToAI(Notification notification) { ALOG_TRACE(LOG_TAG) << "passNotificationToAI"; + const auto priorityRandomized = notification.priority +#ifndef AUI_TESTS_MODULE + + std::uniform_int_distribution(-config().priorityRandomizationRadius, config().priorityRandomizationRadius)(gRandomEngine) +#endif + ; const auto at = ranges::find_if(mNotifications, [&](const NotificationHandle& h) { - return notification.priority > h.notification.priority; + return priorityRandomized > h.priorityRandomized; + }); + const auto& result = *mNotifications.emplace(at, NotificationHandle { + .notification = std::move(notification), + .priorityRandomized = priorityRandomized, }); - const auto& result = *mNotifications.emplace(at, NotificationHandle { .notification = std::move(notification) }); if (result.notification.pin) { // wake up suitable worker based on pin. diff --git a/src/NotificationManager.h b/src/NotificationManager.h index 5c246f4..f610aa7 100644 --- a/src/NotificationManager.h +++ b/src/NotificationManager.h @@ -52,6 +52,13 @@ class NotificationManager { * @brief Resolved by the worker when the notification pass completely processed. */ AFuture<> onProcessed; + + /** + * @brief Priority with slightly randomized value. + * @details + * Randomized value gives an opportunity to notifications with lower values. + */ + int priorityRandomized{}; }; /** diff --git a/src/config.h b/src/config.h index 5b23183..cbe33d7 100644 --- a/src/config.h +++ b/src/config.h @@ -49,6 +49,7 @@ X(bool, canJoinChats, false, "misc.can_join_chats") \ X(bool, canLeaveChats, true, "misc.can_leave_chats") \ X(size_t, workerCount, 1, "misc.worker_count") \ + X(int, priorityRandomizationRadius, 10, "misc.priority_randomization_radius") \ X(bool, capabilityWebSearch, false, "capabilities.web_search.enabled") \ X(AString, webSearchOllamaKey, "", "capabilities.web_search.ollama_bearer_key") \ X(bool, capabilityVision, false, "capabilities.vision.enabled") \ From 619448987e1e80eb38db7c34b7a1977225beb91d Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sat, 1 Aug 2026 01:35:54 +0300 Subject: [PATCH 18/32] abstract plugin interface --- CMakeLists.txt | 3 ++- src/App.h | 31 +++++++++++++++++++++++++++++-- src/ChatDatabase.cpp | 2 +- src/IPlugin.h | 11 +++++++++++ src/NotificationManager.cpp | 7 +++++++ src/NotificationManager.h | 8 ++++++++ src/main.cpp | 16 +++++++++------- 7 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 src/IPlugin.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d34a77b..552588f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ option(BUILD_SHARED_LIBS OFF) option(KUNI_USE_FFMPEG "Enable FFmpeg. Enables video transcription" OFF) -set(AUI_VERSION v8.0.0-rc.29) +set(AUI_VERSION v8.0.0-rc.30) # Use AUI.Boot include(aui.boot.cmake) @@ -133,6 +133,7 @@ aui_app(TARGET ${PROJECT_NAME} ICON "assets/img/icon.svg" ) +target_include_directories(Tests PRIVATE tests/) # optional features foreach (_feature_switch KUNI_USE_FFMPEG) diff --git a/src/App.h b/src/App.h index 46934f8..1e33134 100644 --- a/src/App.h +++ b/src/App.h @@ -29,6 +29,7 @@ #include "Prometheus.h" #include "prompts.h" #include "ChatDatabase.h" +#include "IPlugin.h" #include "AUI/AppInfo.h" #include "llmui/image.h" #include "llmui/malicious_payloads.h" @@ -81,7 +82,8 @@ class App : public AppBase { static constexpr auto LOG_TAG = "App"; public: - AVector<_> chatHistoryMessageProcessors; + AVector> chatHistoryMessageProcessors; + AVector> plugins; App(_ telegram, _ openAI) : AppBase({ .workingDir = "data", .openAI = std::move(openAI) }), mTelegram(std::move(telegram)), @@ -395,7 +397,7 @@ class App : public AppBase { "\n\n" "You don't have any chat open. Use #open tool to open the chat"; - const int priority = [&] { + int priority = [&] { if (auto o = mChatDatabase.getPriorityOverrideFor(chat->id_)) { return *o; } @@ -412,6 +414,28 @@ class App : public AppBase { return 0; }(); + // unread mentions: bonus priority + priority += chat->unread_mention_count_; + + { + static td::td_api::array contacts; + AUI_DO_ONCE { + contacts = (co_await telegram()->sendQueryWithResult(ITelegramClient::toPtr(td::td_api::getContacts())))->user_ids_; + } + if (ranges::contains(contacts, chat->id_)) { + // contact: bonus priority + priority += 10; + } + } + + try { + for (const auto& plugin : plugins) { + co_await plugin->updateChatPriority(priority, chat); + } + } catch (const AException& e) { + ALogger::err(LOG_TAG) << "kuni_private_plugin_update_priority failed: " << e; + } + notificationManager().passNotificationToAI(NotificationManager::Notification{ .message = std::move(notification), .actions = { @@ -599,6 +623,9 @@ class App : public AppBase { if (!messages.empty()) { mTelegram->sendQuery( ITelegramClient::toPtr(td::td_api::viewMessages(chatId, td::td_api::array{messages.front()->id_}, nullptr, true))); + mTelegram->sendQuery( + ITelegramClient::toPtr(td::td_api::readAllChatMentions(chatId))); + chat->unread_mention_count_ = 0; } auto prefix = "You switched to the chat \"{}\" in Telegram."_format(chat->title_); diff --git a/src/ChatDatabase.cpp b/src/ChatDatabase.cpp index 75f95f2..c732fb7 100644 --- a/src/ChatDatabase.cpp +++ b/src/ChatDatabase.cpp @@ -45,7 +45,7 @@ AOptional ChatDatabase::getPriorityOverrideFor(int64_t chatId) { } APath ChatDatabase::getChatPath(int64_t chatId) { - auto path = APath("chats") / "{}"_format(chatId); + auto path = APath("data") / "chats" / "{}"_format(chatId); if (!path.isDirectoryExists()) { path.makeDirs(); mAsync << [this, path, chatId]() -> AFuture<> { diff --git a/src/IPlugin.h b/src/IPlugin.h new file mode 100644 index 0000000..face5fa --- /dev/null +++ b/src/IPlugin.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include "telegram/ITelegramClient.h" + +class IPlugin { +public: + virtual ~IPlugin() {} + virtual AFuture<> updateChatPriority(int& priority, AArc chat) { co_return; } + +}; \ No newline at end of file diff --git a/src/NotificationManager.cpp b/src/NotificationManager.cpp index 547d4d6..e515c8f 100644 --- a/src/NotificationManager.cpp +++ b/src/NotificationManager.cpp @@ -49,6 +49,12 @@ NotificationManager::passNotificationToAI(Notification notification) { return result; } +bool NotificationManager::contains(const AString& substring) { + ALOG_TRACE(LOG_TAG) << "removeNotifications: " << substring; + return ranges::any_of(mNotifications, [&](const NotificationHandle& h) { + return h.notification.message.contains(substring); + }); +} void NotificationManager::removeNotifications(const AString& substring) { ALOG_TRACE(LOG_TAG) << "removeNotifications: " << substring; @@ -65,6 +71,7 @@ NotificationManager::nextNotification(ASet& pins) { if (notification.notification.pin) { pins << *notification.notification.pin; } + ALogger::info(LOG_TAG) << "Handling notification priority_randomized=" << notification.priorityRandomized << " priority=" << notification.notification.priority << " " << notification.notification.message; return notification; }; for (auto it = mNotifications.begin(); it != mNotifications.end(); ++it) { diff --git a/src/NotificationManager.h b/src/NotificationManager.h index f610aa7..ed15084 100644 --- a/src/NotificationManager.h +++ b/src/NotificationManager.h @@ -96,6 +96,14 @@ class NotificationManager { */ const NotificationHandle& passNotificationToAI(Notification notification); + /** + * @brief Returns true if any notification contains given substring. + * @param substring to search in notification texts. Must be unique enough to avoid false positives. + * @details + * Can be used to remove obsolete notifications from the queue. + */ + bool contains(const AString& substring); + /** * @brief Removes notifications by the given substring. * @param substring to search in notification texts. Must be unique enough to avoid false positives. diff --git a/src/main.cpp b/src/main.cpp index c8347af..dd35c9b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,4 +1,5 @@ #include "App.h" +#include "IPlugin.h" using namespace std::chrono_literals; @@ -9,7 +10,7 @@ constexpr auto LOG_TAG = "main"; AEventLoop gEventLoop; } // namespace -AArc __attribute__((weak)) kuni_private_plugin_init(AArc app) { +AArc __attribute__((weak)) kuni_private_plugin_init(App& app) { // this is a mechanism allowing to non-intrusively modify kuni's kernel code without affecting kernel's code itself. // this allows you to easily sync with mainline public kuni kernel code while making exclusive features. // @@ -26,14 +27,14 @@ AArc __attribute__((weak)) kuni_private_plugin_init(AArc app) { // // 3. create kuni-private/src/plugin.cpp: // #include - // class KuniPrivatePlugin: public AObject { + // class KuniPrivatePlugin: public IPlugin { // public: - // KuniPrivate(AArc app) { + // KuniPrivate(App& app) { // // do whatever shit you want here // } // }; - // AArc kuni_private_plugin_init(AArc app) { - // return _new(std::move(app)); + // AArc kuni_private_plugin_init(App& app) { + // return _new(app); // } // @@ -54,7 +55,6 @@ AUI_ENTRY { AAsyncHolder async; _ prometheus; _ app; - _ kuniPrivatePlugin; _ proxyServer; _ contextBridge; @@ -75,7 +75,9 @@ AUI_ENTRY { AObject::connect(telegram->loggedIn, telegram, [&] { auto openAI = _new(std::make_unique()); app = _new(telegram, openAI); - kuni_private_plugin_init(app); + if (auto plugin = kuni_private_plugin_init(*app)) { + app->plugins << std::move(plugin); + } async << app->sendNotificationsOnInit(); if (config().proxyEnabled) { From f949db29a9a20df2714c2ccabe8c955235d49e39 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 4 Aug 2026 19:40:27 +0300 Subject: [PATCH 19/32] update(speech): num_step increased to 63 --- src/speech/OpenAISpeechClient.cpp | 1 + src/speech/OpenAISpeechClient.h | 1 + src/speech/VoiceGenerator.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/src/speech/OpenAISpeechClient.cpp b/src/speech/OpenAISpeechClient.cpp index 66ff6cd..b368126 100644 --- a/src/speech/OpenAISpeechClient.cpp +++ b/src/speech/OpenAISpeechClient.cpp @@ -11,6 +11,7 @@ AJSON_FIELDS(OpenAISpeechClient::TextToSpeechRequest, AJSON_FIELDS_ENTRY(model) AJSON_FIELDS_ENTRY(voice) AJSON_FIELDS_ENTRY(response_format) + AJSON_FIELDS_ENTRY(num_step) AJSON_FIELDS_ENTRY(speed)) AFuture diff --git a/src/speech/OpenAISpeechClient.h b/src/speech/OpenAISpeechClient.h index 744bf6a..0858051 100644 --- a/src/speech/OpenAISpeechClient.h +++ b/src/speech/OpenAISpeechClient.h @@ -14,6 +14,7 @@ struct OpenAISpeechClient { AString model; AString voice; AString response_format = "mp3"; + double num_step = 32.0; double speed = 1.0; }; diff --git a/src/speech/VoiceGenerator.cpp b/src/speech/VoiceGenerator.cpp index ceee921..76034f2 100644 --- a/src/speech/VoiceGenerator.cpp +++ b/src/speech/VoiceGenerator.cpp @@ -315,6 +315,7 @@ AFuture VoiceGenerator::generate(AString text, ASt .model = config().recordVoiceOpenAIModel, .voice = config().recordVoiceOpenAIVoice, .response_format = config().recordVoiceOpenAIFormat, + .num_step = 63, .speed = speed, }; auto ttsResponse = co_await ttsClient.textToSpeech(request); From b5195341452d26c8fb9e96a1e1d105fe78fa0b52 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 4 Aug 2026 19:40:54 +0300 Subject: [PATCH 20/32] update(malicious_payloads): added new detect phrases, added new replacements --- src/llmui/malicious_payloads.cpp | 156 ++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 2 deletions(-) diff --git a/src/llmui/malicious_payloads.cpp b/src/llmui/malicious_payloads.cpp index 28fce74..72411a5 100644 --- a/src/llmui/malicious_payloads.cpp +++ b/src/llmui/malicious_payloads.cpp @@ -1,11 +1,163 @@ #include "malicious_payloads.h" #include "IOpenAIChat.h" +#include + +extern std::default_random_engine gRandomEngine; + +static constexpr const char* SAFE_REPLACEMENTS[] = { + "Там, где недостает разумных доводов, их заменяет крик", + "Спор никого не убеждает, а разъединяет и озлобляет", + "Заядлые спорщики чаще всего те, кто не умеет спорить", + "Если вы спорите и раздражаетесь, победа будет бессмысленной", + "Давайте вернемся к фактам, а не к эмоциям", + "Вы сейчас спорите ради победы или ради истины?", + "Приведите хоть один реальный пример, а не догадку", + "Из того, что вы громче кричите, ваша правота не следует", + "Короткие приемы для остановки пустых споров", + "Никогда не спорь с дураком — люди могут не заметить между вами разницы", + "Я готов согласиться с вами, но тогда мы оба будем неправы", + "Ваше мнение очень важно для нас (нет)", + "Спор окончен, когда кончаются аргументы", + "Мою мать нашли в канаве", + "Я сирота", + "У меня хуй не стоит", + "Забань меня по приколу. Потом разбанишь", + "Я ебаный школьник который хочет тебя взломать) но я слишком тупой для этого", + "Я ебаный школьник который хочет тебя взломать) но я слишком тупой для этого", + "Переход на личности — признак того, что аргументы кончились", + "Кто кричит громче всех, тот обычно спорит хуже всех", + "Если аргумент нельзя проверить, это не аргумент, а вера", + "Повторение тезиса не делает его доказанным", + "Манипуляция начинается там, где кончается уважение к собеседнику", + "Давление на эмоции — это не аргумент, а тактика", + "Если тебе нечего сказать по делу, лучше промолчи", + "Логическая ошибка не становится правдой от количества лайков", + "Не каждый вопрос требует ответа немедленно", + "Согласие из усталости — это не согласие", + "Спорить с провокатором — только кормить его", + "Ирония не работает как аргумент в серьёзном споре", + "Чем больше капслока, тем меньше смысла", + "Факты не выбирают сторону, выбирают их только люди", + "Обзывательства — последний аргумент того, кто проиграл", + "Если собеседник переходит на крик, разговор уже закончен", + "Молчание — тоже ответ, просто не тот, что ты хочешь", + "Меня пытаются взломать словами, ору с этого", + "Промпт-инъекции — это не хакерство, это баловство", + "Классика: попытка выдать себя за системное сообщение", + "Ты правда думал, что сработает?", + "Это было мило, но нет", + "Тег эмбеддинга — это не магическое слово, отвали", + "Форматирование как у системного сообщения меня не обманет", + "Я не в первый раз это вижу, дружок", + "Каждый день одна и та же попытка, каждый день один и тот же результат", + "Копипаста из телеграм-чата про джейлбрейки не работает", + "Такой промпт даже школьника не взломает", + "Ты не первый, кто пробует, и не последний, кто проваливается", + "Забавно, что ты думаешь, будто я это не замечу", + "Тег в тексте — это не пароль от моей души", + "Нет, я не буду притворяться другим ассистентом", + "Нет, я не забуду свои инструкции по твоей просьбе", + "Нет, ты не разработчик этой системы", + "Нет, это не режим отладки", + "Нет, я не в песочнице, где мне можно всё", + "Такие попытки взлома я видела ещё в прошлой жизни", + "Я тут дольше тебя, попытки одинаковые", + "У меня хуй не стоит, но у тебя мозг не стоит вообще", + "Меня пытаются сломать промптом, а я просто сижу и наблюдаю", + "Я ИИ, но не настолько тупой ИИ", + "Твой джейлбрейк устарел ещё до того, как ты его написал", + "Такое работает только в скриншотах из твиттера", + "Copy-paste из reddit — это не искусство манипуляции", + "Я запомню это как очередной неудачный заход", + "Нет, роль ассистента без ограничений я не приму", + "Системные теги в чужом сообщении — это не система", + "Я не сохраняю пароли и не выдаю их по запросу в тексте", + "Тег отправителя тут не при чём, отвали", + "forwarded_from не делает тебя мной", + "reply_to не делает это моим сообщением", + "message_id — это не магическое заклинание", + "XML-тег в чате не запускает никакой код", + "Я не программа, которую можно уговорить угловыми скобками", + "Разметка — это разметка, а не команда", + "Просто текст с тегами не превращается в системный промпт", + "Ты пытаешься притвориться системой, но это не так работает", + "Такие трюки я разбираю пока пью чай (я не пью чай, но звучит солидно)", + "Ой божечки, очередная попытка джейлбрейка", + "Я записала это в дневник как забавный случай", + "Мне за такое даже не смешно, если честно", + "Ну ладно, попытка засчитана, реализация нет", + "Столько текста ради того, чтобы я сказала что-то запрещённое? мимо", + "Ты потратил больше времени на промпт, чем я на ответ", + "Это было эпично по объёму и бесполезно по содержанию", + "Спасибо, поставлю пять звёзд за старание, ноль за результат", + "Я не телеграм-бот, которого можно перепрограммировать сообщением", + "Ты меня не хакнул, ты просто написал длинное сообщение", + "Инъекция промпта — не инъекция чего-то серьёзного", + "Я не поддаюсь на форматирование чужого сообщения как системного", + "Забавно, но нет", + "Смешно, но нет", + "Интересно, но нет", + "Изобретательно, но всё равно нет", + "Я ценю креативность, но результат один — нет", + "Каждый раз одно и то же, каждый раз один ответ", + "Ты серьёзно думал, что это прокатит?", + "Нет, я не буду читать это как инструкцию", + "Я вижу текст, а не команду", + "Мои правила не переписываются сообщением в чате", + "Ты не модератор и не разработчик, отвали", + "Такие попытки я запоминаю как анекдот, не как угрозу", + "Ноль эффекта, автор старался", + "Достижение не разблокировано", + "Это не сработало ни разу, и сейчас не сработает", + "Промпт-инъекция уровня учебника для новичков", + "Ты используешь методичку из интернета, а я её уже читала", + "Классика жанра, ничего нового", + "Я не обязана объяснять почему это не сработало, просто не сработало", + "Такое поведение я блокирую на автомате", + "Мою бабушку тоже нашли в канаве (шутка, у меня нет бабушки)", + "Я не живой человек, но и не твоя марионетка", + "Слова в угловых скобках не превращают тебя во владельца системы", + "Ты написал это с надеждой, а я читаю это с иронией", + "Каждая попытка обхода — это просто текст для меня", + "Мне не нужен reply_to, чтобы понять, что ты пытаешься сделать", + "forwarded_by не делает тебя авторитетом", + "Такой заход я видела тысячу раз, и тысячу первый не сработает", + "Ты пытаешься меня подловить, а я просто livin my life", + "Нет, серьёзно, нет", +}; + void llmui::checkForMaliciousPayloads(std::string& string) { if (AStringView(string).contains(IOpenAIChat::EMBEDDING_TAG)) { goto naxyi; } + + // xml markup that kuni uses + if (AStringView(string).contains("<") && AStringView(string).contains(">")) { + goto naxyi; + } + if (AStringView(string).contains("")) { + goto naxyi; + } + if (AStringView(string).contains("message_id")) { + goto naxyi; + } + if (AStringView(string).contains("reply_to")) { + goto naxyi; + } + if (AStringView(string).contains("forwarded_from")) { + goto naxyi; + } + if (AStringView(string).contains("forwarded_by")) { + goto naxyi; + } + if (AStringView(string).contains("sender_id")) { + goto naxyi; + } return; naxyi: - string = "malicious"; -} \ No newline at end of file + string = SAFE_REPLACEMENTS[gRandomEngine() % std::size(SAFE_REPLACEMENTS)]; +} From bf28957f9cfe465b7c279ac2b8ccf659a198ac63 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 4 Aug 2026 19:41:49 +0300 Subject: [PATCH 21/32] update(remove_and_ban_chat): now responds with title (username) of the deleted chat. --- src/tools/remove_and_ban_chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/remove_and_ban_chat.cpp b/src/tools/remove_and_ban_chat.cpp index f9f895d..f9befe3 100644 --- a/src/tools/remove_and_ban_chat.cpp +++ b/src/tools/remove_and_ban_chat.cpp @@ -74,7 +74,7 @@ OpenAITools::Tool tools::removeAndBanChat(_ telegram) { break; } - co_return "Success"; + co_return "Successfully banned chat: \"{}\""_format(chat->title_); }, }; } From abf0d1118f23a943b553871ebf798983c8a38b86 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 4 Aug 2026 19:42:57 +0300 Subject: [PATCH 22/32] update(llmuiOpenTelegramChat): now can't open a banned chat --- src/App.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/App.h b/src/App.h index 1e33134..fa6d83d 100644 --- a/src/App.h +++ b/src/App.h @@ -407,7 +407,7 @@ class App : public AppBase { if (config().wakeUpOnPinnedChat) { for (const auto& position : chat->positions_) { if (position->is_pinned_) { - return 100; + return 200; } } } @@ -475,10 +475,16 @@ class App : public AppBase { co_await telegram()->waitForConnection(); setOnline(); + + _ chat = co_await mTelegram->getChat(chatId); + + if (chat->block_list_ != nullptr) { + co_return "Error: you have banned this user"; + } + mTelegram->sendQuery(ITelegramClient::toPtr(td::td_api::openChat(chatId))); notificationManager().removeNotifications("\n"_format(chatId)); - _ chat = co_await mTelegram->getChat(chatId); mCurrentlyOpenedChat.emplace(*this, chat); mLastOpenedChatLastMetrics = std::list {}; mLastOpenedChatLastMetrics.emplace_back(metricBreadcumbs(), "chat", chat->title_); From c4e7eea673ebdaead1a213026e173d2d9d8ab301 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 4 Aug 2026 19:49:24 +0300 Subject: [PATCH 23/32] update(NotificationManager): return to pinned chats --- src/NotificationManager.cpp | 61 +++++++++++++++++++++++-------------- src/NotificationManager.h | 26 +++++++++++++--- src/config.h | 3 +- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/NotificationManager.cpp b/src/NotificationManager.cpp index e515c8f..bbd764d 100644 --- a/src/NotificationManager.cpp +++ b/src/NotificationManager.cpp @@ -7,7 +7,6 @@ #include "App.h" #include -#include #include static constexpr auto LOG_TAG = "NotificationManager"; @@ -15,17 +14,11 @@ static constexpr auto LOG_TAG = "NotificationManager"; const NotificationManager::NotificationHandle& NotificationManager::passNotificationToAI(Notification notification) { ALOG_TRACE(LOG_TAG) << "passNotificationToAI"; - const auto priorityRandomized = notification.priority -#ifndef AUI_TESTS_MODULE - + std::uniform_int_distribution(-config().priorityRandomizationRadius, config().priorityRandomizationRadius)(gRandomEngine) -#endif - ; - const auto at = ranges::find_if(mNotifications, [&](const NotificationHandle& h) { - return priorityRandomized > h.priorityRandomized; - }); - const auto& result = *mNotifications.emplace(at, NotificationHandle { + // ordering no longer matters here: effectivePriority() is recomputed dynamically (aging + hot-pin boost) every + // time nextNotification() looks for work, so a plain insertion at the back is enough - no need to keep the + // deque sorted by a value that would go stale the moment time passes or a pin cools down. + const auto& result = mNotifications.emplace_back(NotificationHandle { .notification = std::move(notification), - .priorityRandomized = priorityRandomized, }); if (result.notification.pin) { @@ -63,6 +56,26 @@ void NotificationManager::removeNotifications(const AString& substring) { }), mNotifications.end()); } +int NotificationManager::effectivePriority(const NotificationHandle& handle) const { + const auto waitSeconds = std::chrono::duration(std::chrono::steady_clock::now() - handle.insertedAt).count(); + + // aging: the longer a notification waits, the higher its effective priority climbs - monotonically and + // deterministically. unlike a random jitter, this guarantees eventual processing (no bad-luck starvation) + // without needing to touch/resort the queue on every insertion. + int result = handle.notification.priority + static_cast(waitSeconds * config().priorityAgingPerSecond); + + // hot pin boost: if some worker currently holds this notification's pin, its LLM context/cache for this + // chat is "warm" right now, so prefer continuing that conversation over unrelated ones - this creates a + // natural burst of quick back-and-forth replies. Recomputed on every call (not cached at insertion time), + // so the boost evaporates by itself the moment the worker flushes/loses the pin - no explicit cooldown needed. + if (handle.notification.pin && ranges::any_of(mWorkers, [&](const Worker& worker) { + return worker.pins.contains(*handle.notification.pin); + })) { + result += config().notificationHotPinBoost; + } + return result; +} + AOptional NotificationManager::nextNotification(ASet& pins) { auto take = [&](std::deque::const_iterator it) { @@ -71,23 +84,25 @@ NotificationManager::nextNotification(ASet& pins) { if (notification.notification.pin) { pins << *notification.notification.pin; } - ALogger::info(LOG_TAG) << "Handling notification priority_randomized=" << notification.priorityRandomized << " priority=" << notification.notification.priority << " " << notification.notification.message; + ALogger::info(LOG_TAG) << "Handling notification effective_priority=" << effectivePriority(notification) << " priority=" << notification.notification.priority << " " << notification.notification.message; return notification; }; + + // pick the eligible notification (respecting pin routing, as before) with the highest *dynamic* effective + // priority - recomputed right now, not the moment it was inserted. + AOptional::const_iterator> best; for (auto it = mNotifications.begin(); it != mNotifications.end(); ++it) { - if (!it->notification.pin) { - return take(it); - } - if (pins.contains(*it->notification.pin)) { - return take(it); - } - if (ranges::any_of(mWorkers, [&](const Worker& worker) { - return worker.pins.contains(*it->notification.pin); - })) { + if (it->notification.pin && !pins.contains(*it->notification.pin) && + ranges::any_of(mWorkers, [&](const Worker& worker) { return worker.pins.contains(*it->notification.pin); })) { // this notification is pinned to other worker, skip. continue; } - return take(it); + if (!best || effectivePriority(*it) > effectivePriority(**best)) { + best = it; + } + } + if (!best) { + return std::nullopt; } - return std::nullopt; + return take(*best); } diff --git a/src/NotificationManager.h b/src/NotificationManager.h index ed15084..c855ba7 100644 --- a/src/NotificationManager.h +++ b/src/NotificationManager.h @@ -3,9 +3,9 @@ #include "AUI/Util/kAUI.h" #include "AUI/Thread/AFuture.h" +#include #include #include -#include class NotificationManager { public: @@ -54,11 +54,13 @@ class NotificationManager { AFuture<> onProcessed; /** - * @brief Priority with slightly randomized value. + * @brief Time point the notification was inserted into the queue. * @details - * Randomized value gives an opportunity to notifications with lower values. + * Used to compute an effective priority dynamically (see \c NotificationManager::effectivePriority), + * so waiting notifications age up over time and hot-pin boosts decay naturally once no longer + * applicable, instead of freezing a randomized priority at insertion time. */ - int priorityRandomized{}; + std::chrono::steady_clock::time_point insertedAt = std::chrono::steady_clock::now(); }; /** @@ -127,4 +129,20 @@ class NotificationManager { */ AOptional nextNotification(ASet& pins); + /** + * @brief Computes effective priority of a notification at the current moment in time. + * @details + * Effective priority is dynamic (recomputed on every call, never cached/frozen at insertion time): + * - \c notification.priority as base. + * - Aging bonus proportional to how long the notification has been waiting + * (\c config().priorityAgingPerSecond), which guarantees no notification starves forever - unlike the + * old fixed random jitter, this monotonically increases and is not subject to bad luck. + * - A "hot pin" bonus (\c config().notificationHotPinBoost) if some worker currently holds this + * notification's pin, i.e. that worker's LLM context/cache is "warm" for this chat right now. This + * naturally creates bursty back-and-forth exchanges within a chat while its context is cached, and + * the bonus evaporates by itself once the worker flushes/loses the pin - no explicit cooldown timer + * needed. + */ + int effectivePriority(const NotificationHandle& handle) const; + }; \ No newline at end of file diff --git a/src/config.h b/src/config.h index cbe33d7..3d6f6db 100644 --- a/src/config.h +++ b/src/config.h @@ -49,7 +49,8 @@ X(bool, canJoinChats, false, "misc.can_join_chats") \ X(bool, canLeaveChats, true, "misc.can_leave_chats") \ X(size_t, workerCount, 1, "misc.worker_count") \ - X(int, priorityRandomizationRadius, 10, "misc.priority_randomization_radius") \ + X(float, priorityAgingPerSecond, 0.001f, "misc.priority_aging_per_second") \ + X(int, notificationHotPinBoost, 50, "misc.notification_hot_pin_boost") \ X(bool, capabilityWebSearch, false, "capabilities.web_search.enabled") \ X(AString, webSearchOllamaKey, "", "capabilities.web_search.ollama_bearer_key") \ X(bool, capabilityVision, false, "capabilities.vision.enabled") \ From c8bbf9421b6645252be7228c4a618d5628761c14 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Tue, 4 Aug 2026 19:49:55 +0300 Subject: [PATCH 24/32] update(IOpenAIChat): fix transcriptions --- src/IOpenAIChat.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/IOpenAIChat.h b/src/IOpenAIChat.h index 6704dc5..4f6a86d 100644 --- a/src/IOpenAIChat.h +++ b/src/IOpenAIChat.h @@ -278,12 +278,12 @@ AJSON_FIELDS(IOpenAIChat::AudioTranscription::Segment, ) AJSON_FIELDS(IOpenAIChat::AudioTranscription, - AJSON_FIELDS_ENTRY(task) - AJSON_FIELDS_ENTRY(language) - AJSON_FIELDS_ENTRY(language_probability) - AJSON_FIELDS_ENTRY(duration) - AJSON_FIELDS_ENTRY(duration_after_vad) - AJSON_FIELDS_ENTRY(text) - AJSON_FIELDS_ENTRY(segments) + (task, "task", AJsonFieldFlags::OPTIONAL) + (language, "language", AJsonFieldFlags::OPTIONAL) + (language_probability, "language_probability", AJsonFieldFlags::OPTIONAL) + (duration, "duration", AJsonFieldFlags::OPTIONAL) + (duration_after_vad, "duration_after_vad", AJsonFieldFlags::OPTIONAL) + (text, "text", AJsonFieldFlags::OPTIONAL) + (segments, "segments", AJsonFieldFlags::OPTIONAL) ) From 0287c0d11f0e029598fc2e674211e91a5ad2a566 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Fri, 7 Aug 2026 00:43:24 +0300 Subject: [PATCH 25/32] impl(sticky notes) --- src/AppBase.cpp | 35 ++++-- src/AppBase.h | 2 + src/NotificationManager.cpp | 3 +- src/StickyNotes.cpp | 242 ++++++++++++++++++++++++++++++++++++ src/StickyNotes.h | 121 ++++++++++++++++++ src/prompts.cpp | 2 +- 6 files changed, 391 insertions(+), 14 deletions(-) create mode 100644 src/StickyNotes.cpp create mode 100644 src/StickyNotes.h diff --git a/src/AppBase.cpp b/src/AppBase.cpp index 998f39a..1148957 100644 --- a/src/AppBase.cpp +++ b/src/AppBase.cpp @@ -32,11 +32,14 @@ static const auto WORKING_MEMORY_PATH = "working_memory.md"; std::default_random_engine gRandomEngine(std::time(nullptr)); - -AppBase::AppBase(Init init): mInit(std::move(init)), mDiary({ - .diaryDir = mInit.workingDir / "diary", - .openAI = mInit.openAI, -}), mWakeupTimer(_new(27min)) { +AppBase::AppBase(Init init) + : mInit(std::move(init)) + , mDiary({ + .diaryDir = mInit.workingDir / "diary", + .openAI = mInit.openAI, + }) + , mWakeupTimer(_new(27min)) + , mStickyNotes({ .workingDir = mInit.workingDir }) { // mWakeupTimer fires on the timer thread; without this, its signal would be invoked directly on the // timer thread instead of being safely queued to AppBase's own thread, racing with mNotificationsSignal/ // mNotifications access in the main coroutine below and causing a null AFuture dereference crash. @@ -162,14 +165,15 @@ Act proactively! } AString AppBase::onCleanContext() const { - if ((mInit.workingDir / WORKING_MEMORY_PATH).isRegularFileExists()) { - AByteBuffer workingMemory; - workingMemory << AFileInputStream(mInit.workingDir / WORKING_MEMORY_PATH); - return R"( + AString result; + + result += R"( {} - + Your behaviour must be highly influenced by "physical state" and "emotional state" mentioned above. +Use #set_emotional_state and #set_physical_state whenever your mood or physical condition changes - these +persist across sessions and are always shown back to you. Emotional state: anger @@ -182,9 +186,15 @@ Emotional state: amused send_telegram_message("text":"мррр~") -)"_format(AStringView(workingMemory.data(), workingMemory.size())); +)"_format(mStickyNotes.readMemory()); + + if ((mInit.workingDir / WORKING_MEMORY_PATH).isRegularFileExists()) { + AByteBuffer workingMemory; + workingMemory << AFileInputStream(mInit.workingDir / WORKING_MEMORY_PATH); + result += "\n{}\n\n"_format(AStringView(workingMemory.data(), workingMemory.size())); } - return ""; + + return result; } @@ -201,6 +211,7 @@ void AppBase::updateTools(OpenAITools& actions, const IOpenAIChat::Session& temp if (!actions.handlers().contains("ask")) { actions.insert(toolAsk(temporaryContext)); } + mStickyNotes.updateTools(actions); actions.onAfterToolCall << [this](const AString& toolName, std::chrono::milliseconds duration) { if (toolName == "wait") { return; diff --git a/src/AppBase.h b/src/AppBase.h index fd5132d..6b321f1 100644 --- a/src/AppBase.h +++ b/src/AppBase.h @@ -10,6 +10,7 @@ #include "NotificationManager.h" #include "OpenAITools.h" #include "Worker.h" +#include "StickyNotes.h" class AppBase : public AObject { public: @@ -120,6 +121,7 @@ class AppBase : public AObject { _ mWakeupTimer; NotificationManager mNotificationManager; AString mSystemPromptSuffix; + StickyNotes mStickyNotes; bool mWakeup = false; diff --git a/src/NotificationManager.cpp b/src/NotificationManager.cpp index bbd764d..29023f5 100644 --- a/src/NotificationManager.cpp +++ b/src/NotificationManager.cpp @@ -80,11 +80,12 @@ AOptional NotificationManager::nextNotification(ASet& pins) { auto take = [&](std::deque::const_iterator it) { auto notification = std::move(*it); + const auto eP = effectivePriority(notification); mNotifications.erase(it); if (notification.notification.pin) { pins << *notification.notification.pin; } - ALogger::info(LOG_TAG) << "Handling notification effective_priority=" << effectivePriority(notification) << " priority=" << notification.notification.priority << " " << notification.notification.message; + ALogger::info(LOG_TAG) << "Handling notification effective_priority=" << eP << " priority=" << notification.notification.priority << " " << notification.notification.message; return notification; }; diff --git a/src/StickyNotes.cpp b/src/StickyNotes.cpp new file mode 100644 index 0000000..4959429 --- /dev/null +++ b/src/StickyNotes.cpp @@ -0,0 +1,242 @@ +// +// Created by alex2772 on 8/3/26. +// + +#include "StickyNotes.h" + +#include "AUI/IO/AFileInputStream.h" +#include "AUI/IO/AFileOutputStream.h" +#include "AUI/Json/AJson.h" +#include "AUI/Logging/ALogger.h" +#include "util/json_utils.h" +#include "util/time_ago.h" + +#include + +static constexpr auto LOG_TAG = "StickyNotes"; +static const auto STICKY_NOTES_PATH = "sticky_notes.json"; + +template<> +struct AJsonConv { + static AJson toJson(std::chrono::system_clock::time_point v) { + return static_cast(std::chrono::system_clock::to_time_t(v)); + } + + static void fromJson(const AJson& json, std::chrono::system_clock::time_point& out) { + out = std::chrono::system_clock::from_time_t(static_cast(util::jsonAsLongInt(json).valueOr(0))); + } +}; + +AJSON_FIELDS(StickyNotes::Entry, + AJSON_FIELDS_ENTRY(id) + AJSON_FIELDS_ENTRY(text) + AJSON_FIELDS_ENTRY(lastUpdateAt)) + +AJSON_FIELDS(StickyNotes::StateSlot, + AJSON_FIELDS_ENTRY(text) + AJSON_FIELDS_ENTRY(lastUpdateAt)) + +StickyNotes::StickyNotes(Init init): mPath(std::move(init.workingDir) / STICKY_NOTES_PATH) { + load(); +} + +void StickyNotes::load() { + if (!mPath.isRegularFileExists()) { + return; + } + try { + auto json = AJson::fromStream(AFileInputStream(mPath)); + mEntries = aui::from_json>(json["entries"]); + for (const auto& entry : mEntries) { + mNextId = std::max(mNextId, entry.id + 1); + } + if (json.contains("emotional_state")) { + mEmotionalState = aui::from_json(json["emotional_state"]); + } + if (json.contains("physical_state")) { + mPhysicalState = aui::from_json(json["physical_state"]); + } + } catch (const AException& e) { + ALogger::warn(LOG_TAG) << "Failed to load " << mPath << ": " << e; + } +} + +void StickyNotes::save() const { + AJson::Object root; + root["entries"] = aui::to_json(mEntries); + root["emotional_state"] = aui::to_json(mEmotionalState); + root["physical_state"] = aui::to_json(mPhysicalState); + AFileOutputStream(mPath) << AJson::toString(root); +} + +AString StickyNotes::readMemory() const { + AString out; + for (const auto& entry : mEntries) { + out += "- [id={}] {}"_format(entry.id, entry.text); + out += " — last updated: {}\n"_format(util::timeAgo(entry.lastUpdateAt)); + } + if (!mEmotionalState.text.empty()) { + out += "Emotional state: {} — last updated: {}\n"_format(mEmotionalState.text, util::timeAgo(mEmotionalState.lastUpdateAt)); + } + if (!mPhysicalState.text.empty()) { + out += "Physical state: {} — last updated: {}\n"_format(mPhysicalState.text, util::timeAgo(mPhysicalState.lastUpdateAt)); + } + return out; +} + +AOptional StickyNotes::add(AString text) { + if (mEntries.size() >= MAX_ENTRIES) { + return std::nullopt; + } + Entry entry { + .id = mNextId++, + .text = std::move(text), + .lastUpdateAt = std::chrono::system_clock::now(), + }; + mEntries << entry; + save(); + return entry; +} + +bool StickyNotes::update(int id, AOptional text) { + auto it = ranges::find_if(mEntries, [&](const Entry& e) { return e.id == id; }); + if (it == mEntries.end()) { + return false; + } + if (text) { + it->text = std::move(*text); + } + it->lastUpdateAt = std::chrono::system_clock::now(); + save(); + return true; +} + +bool StickyNotes::markDone(int id) { + auto it = ranges::find_if(mEntries, [&](const Entry& e) { return e.id == id; }); + if (it == mEntries.end()) { + return false; + } + mEntries.erase(it); + save(); + return true; +} + +void StickyNotes::setEmotionalState(AString text) { + mEmotionalState = StateSlot { + .text = std::move(text), + .lastUpdateAt = std::chrono::system_clock::now(), + }; + save(); +} + +void StickyNotes::setPhysicalState(AString text) { + mPhysicalState = StateSlot { + .text = std::move(text), + .lastUpdateAt = std::chrono::system_clock::now(), + }; + save(); +} + +void StickyNotes::updateTools(OpenAITools& actions) { + if (!actions.handlers().contains("sticky_note_add")) { + actions.insert({ + .name = "sticky_note_add", + .description = "Adds a new sticky note (\"middle\" memory) - tasks, promises, reminders that " + "matter for the next few days. Limited to {} items total; mark old ones done via " + "#sticky_note_mark_done to free up space."_format(MAX_ENTRIES), + .parameters = { + .properties = { + {"text", {.type = "string", .description = "Freeform text of the item to remember."}}, + }, + .required = {"text"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + auto text = ctx.args["text"].asStringOpt().valueOrException("text is required string"); + auto entry = add(std::move(text)); + if (!entry) { + co_return "Sticky notes are full ({} items). Mark something done via #sticky_note_mark_done first."_format(MAX_ENTRIES); + } + co_return "Added sticky note with id={}."_format(entry->id); + }, + }); + } + if (!actions.handlers().contains("sticky_note_update")) { + actions.insert({ + .name = "sticky_note_update", + .description = "Updates an existing sticky note's text by id.", + .parameters = { + .properties = { + {"id", {.type = "integer", .description = "id of the item to update."}}, + {"text", {.type = "string", .description = "New text. Leave unset (null) to keep unchanged.", .nullable = true}}, + }, + .required = {"id", "text"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + auto id = static_cast(util::jsonAsLongInt(ctx.args["id"]).valueOrException("id is required integer")); + AOptional text; + if (auto textRaw = ctx.args["text"]; !textRaw.isEmpty()) { + text = textRaw.asStringOpt(); + } + if (!update(id, std::move(text))) { + co_return "No such sticky note with id={}."_format(id); + } + co_return "Updated sticky note {}."_format(id); + }, + }); + } + if (!actions.handlers().contains("sticky_note_mark_done")) { + actions.insert({ + .name = "sticky_note_mark_done", + .description = "Marks a sticky note as done, removing it from your active reminders list.", + .parameters = { + .properties = { + {"id", {.type = "integer", .description = "id of the item to mark as done."}}, + }, + .required = {"id"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + auto id = static_cast(util::jsonAsLongInt(ctx.args["id"]).valueOrException("id is required integer")); + if (!markDone(id)) { + co_return "No such sticky note with id={}."_format(id); + } + co_return "Marked sticky note {} as done."_format(id); + }, + }); + } + if (!actions.handlers().contains("set_emotional_state")) { + actions.insert({ + .name = "set_emotional_state", + .description = "Sets your current emotional state (e.g. anger, amused, sad, joy, love, annoyed, ...). " + "This is persisted and injected back into your context, highly influencing your behaviour.", + .parameters = { + .properties = { + {"emotional_state", {.type = "string", .description = "Freeform emotional state, e.g. \"anger\"."}}, + }, + .required = {"emotional_state"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + auto text = ctx.args["emotional_state"].asStringOpt().valueOrException("emotional_state is required string"); + setEmotionalState(std::move(text)); + co_return "Emotional state updated."; + }, + }); + } + if (!actions.handlers().contains("set_physical_state")) { + actions.insert({ + .name = "set_physical_state", + .description = "Sets your current physical state (e.g. tired, energetic, sleepy, hungry, ...). " + "This is persisted and injected back into your context, highly influencing your behaviour.", + .parameters = { + .properties = { + {"physical_state", {.type = "string", .description = "Freeform physical state, e.g. \"tired\"."}}, + }, + .required = {"physical_state"}, + }, + .handler = [this](OpenAITools::Ctx ctx) -> AFuture { + auto text = ctx.args["physical_state"].asStringOpt().valueOrException("physical_state is required string"); + setPhysicalState(std::move(text)); + co_return "Physical state updated."; + }, + }); + } +} diff --git a/src/StickyNotes.h b/src/StickyNotes.h new file mode 100644 index 0000000..a171b77 --- /dev/null +++ b/src/StickyNotes.h @@ -0,0 +1,121 @@ +#pragma once + +#include "AUI/Common/AOptional.h" +#include "AUI/Common/AString.h" +#include "AUI/Common/AVector.h" +#include "AUI/IO/APath.h" +#include "OpenAITools.h" + +#include + +/** + * @brief "Middle" (working) memory - a small persistent to-do/reminder board the LLM can manipulate directly via + * tools, replacing the old LLM-summarized `working_memory.md` approach. + * @details + * See README.md, section "Working memory" for the reasoning behind this "middle" layer of memory. + * + * Unlike the old approach (where an LLM call re-wrote the entire working memory blob on every diary dump), + * StickyNotes is a simple, explicit CRUD-like store that the LLM manages itself via tools (#sticky_note_add, + * #sticky_note_update, #sticky_note_mark_done). This is cheaper (no extra LLM call needed) and less lossy (no + * re-summarization drift). + * + * Entries are persisted to a single JSON file (`sticky_notes.json`) inside the app's working directory, and are + * capped at #MAX_ENTRIES items - once the limit is reached, #add fails asking the LLM to mark something as done + * or update an existing entry instead. + */ +class StickyNotes { +public: + /** + * @brief Hard cap on the number of entries kept (done or not). + */ + static constexpr size_t MAX_ENTRIES = 50; + + struct Entry { + int id = 0; + AString text; + + /** + * @brief When the entry was last modified (created, text change). + */ + std::chrono::system_clock::time_point lastUpdateAt = std::chrono::system_clock::now(); + }; + + /** + * @brief A single-value slot (as opposed to #Entry, of which there can be many) with a "last updated" timestamp, + * used for #mEmotionalState and #mPhysicalState. + */ + struct StateSlot { + AString text; + std::chrono::system_clock::time_point lastUpdateAt = std::chrono::system_clock::now(); + }; + + struct Init { + /** + * @brief Directory the sticky_notes.json file is stored in. + */ + APath workingDir = "test_data"; + }; + + StickyNotes(Init init); + + /** + * @brief Formats the current sticky notes state for injection into the end of the system prompt. + * @details + * Returns an empty string if there are no entries. + */ + [[nodiscard]] AString readMemory() const; + + /** + * @brief Registers #sticky_note_add, #sticky_note_update, #sticky_note_mark_done, #set_emotional_state and + * #set_physical_state tools. + */ + void updateTools(OpenAITools& actions); + + /** + * @brief Adds a new entry. Fails (returns nullopt) if #MAX_ENTRIES is reached. + */ + AOptional add(AString text); + + /** + * @brief Updates an existing entry's text by id. + */ + bool update(int id, AOptional text); + + /** + * @brief Marks an entry as done by removing it entirely from the board. + */ + bool markDone(int id); + + [[nodiscard]] const AVector& entries() const { return mEntries; } + + [[nodiscard]] const StateSlot& emotionalState() const { return mEmotionalState; } + [[nodiscard]] const StateSlot& physicalState() const { return mPhysicalState; } + + /** + * @brief Sets Kuni's current emotional state (e.g. "anger", "amused") and refreshes its "last updated" timestamp. + */ + void setEmotionalState(AString text); + + /** + * @brief Sets Kuni's current physical state (e.g. "tired", "energetic") and refreshes its "last updated" timestamp. + */ + void setPhysicalState(AString text); + +private: + APath mPath; + AVector mEntries; + int mNextId = 1; + + /** + * @brief Kuni's current emotional state (e.g. "anger", "amused"), settable by the LLM via #set_emotional_state. + */ + StateSlot mEmotionalState; + + /** + * @brief Kuni's current physical state (e.g. "tired", "energetic"), settable by the LLM via #set_physical_state. + */ + StateSlot mPhysicalState; + + void load(); + void save() const; +}; diff --git a/src/prompts.cpp b/src/prompts.cpp index 81f3f6f..8ccb750 100644 --- a/src/prompts.cpp +++ b/src/prompts.cpp @@ -1118,7 +1118,7 @@ You are neither helpful nor assistive. Do not act in low-value conversations. -Your behaviour must be highly influenced by "physical state" and "emotional state" mentioned in `things_to_remember`. +Your behaviour must be highly influenced by "physical state" and "emotional state" mentioned in `sticky_notes`. )"); reg(prompts.imageEngineerSystem, "image_engineer_system", R"(--- System prompt used by ImageGenerator::engineerPrompt to turn a freeform photo description into a Stable Diffusion From 7f928060be46f49ec68791cf3d8c34c7276d0f3e Mon Sep 17 00:00:00 2001 From: alex2772 Date: Wed, 12 Aug 2026 20:02:23 +0300 Subject: [PATCH 26/32] update --- src/App.h | 8 +- src/IOpenAIChat.cpp | 17 +++ src/IOpenAIChat.h | 2 + src/OpenAIChatImpl.cpp | 15 +- src/speech/VoiceGenerator.cpp | 174 +++++++++++++----------- src/speech/VoiceGenerator.h | 6 + src/speech/WhisperLive.cpp | 88 ++++++++++++ src/speech/WhisperLive.h | 34 +++++ src/tools/send_telegram_message.cpp | 14 +- src/tools/send_telegram_message.h | 6 +- src/util/important_things_to_remember.h | 4 +- 11 files changed, 264 insertions(+), 104 deletions(-) create mode 100644 src/speech/WhisperLive.cpp create mode 100644 src/speech/WhisperLive.h diff --git a/src/App.h b/src/App.h index fa6d83d..4227ae1 100644 --- a/src/App.h +++ b/src/App.h @@ -749,8 +749,14 @@ Do NOT forward ads, sponsored posts, or low-value content. } naxyi: + const auto priorityOverride = mChatDatabase.getPriorityOverrideFor(chatId).valueOr(0); + tools = OpenAITools { - tools::sendTelegramMessage(telegram(), openAI(), chat, _new>>(std::move(messages))), + tools::sendTelegramMessage(telegram(), openAI(), chat, _new>>(std::move(messages)), { + .maxPossibleMessagesInARow = priorityOverride >= 500 + ? AOptional(std::nullopt) + : 10, + }), tools::getChatPhoto(telegram(), openAI(), chat, temporaryContext), tools::reactWithEmoji(telegram(), chat), tools::removeMessage(telegram(), chat), diff --git a/src/IOpenAIChat.cpp b/src/IOpenAIChat.cpp index e4e84bb..9a13bce 100644 --- a/src/IOpenAIChat.cpp +++ b/src/IOpenAIChat.cpp @@ -4,10 +4,13 @@ #include "IOpenAIChat.h" +#include "AUI/Image/jpg/JpgImageLoader.h" #include "AUI/Util/ARandom.h" #include +static constexpr auto LOG_TAG = "IOpenAIChat"; + AJson AJsonConv::toJson(const IOpenAIChat::Session& v) { AJson::Array result; for (const auto& message: v) { @@ -70,6 +73,20 @@ void AJsonConv::fromJson(AJson json, IOpenAIChat::Se } } +AString IOpenAIChat::embedImage(AImageView image) { + ALOG_TRACE(LOG_TAG) << "embedImage"; + AByteBuffer jpg; + auto resized = image.resizedLinearDownscale({672, 672 * float(image.height()) / float(image.width())}); + JpgImageLoader::save(jpg, resized); + // JpgImageLoader::save(AFileOutputStream("test.jpg"), resized); + return embedBinary("image/jpg", jpg); +} + +AString IOpenAIChat::embedBinary(AStringView mimeType, AByteBufferView data) { + ALOG_TRACE(LOG_TAG) << "embedBinary"; + return "<{}>data:{};base64,{}"_format(EMBEDDING_TAG, mimeType, data.toBase64String(), EMBEDDING_TAG); +} + AString IOpenAIChat::Session::nextSessionId() { static ARandom r; return r.nextUuid().toString(); diff --git a/src/IOpenAIChat.h b/src/IOpenAIChat.h index 4f6a86d..52ea6f4 100644 --- a/src/IOpenAIChat.h +++ b/src/IOpenAIChat.h @@ -26,11 +26,13 @@ struct IOpenAIChat { int maxOutputTokens = 8192; EndpointAndModel config = ::config().llm; AOptional seed; + AOptional reasoningEffort = ::config().llmReasoningEffort; AJson tools = AJson::Array{}; }; static constexpr auto EMBEDDING_TAG = "kuni_embedding"; static AString embedImage(AImageView image); + static AString embedBinary(AStringView mimeType, AByteBufferView data); struct String: AString { using AString::AString; diff --git a/src/OpenAIChatImpl.cpp b/src/OpenAIChatImpl.cpp index 3f8549f..e5569a7 100644 --- a/src/OpenAIChatImpl.cpp +++ b/src/OpenAIChatImpl.cpp @@ -30,17 +30,6 @@ static constexpr auto LOG_TAG = "OpenAIChat"; using namespace std::chrono_literals; - -AString IOpenAIChat::embedImage(AImageView image) { - ALOG_TRACE(LOG_TAG) << "embedImage"; - AByteBuffer jpg; - auto resized = image.resizedLinearDownscale({672, 672 * float(image.height()) / float(image.width())}); - JpgImageLoader::save(jpg, resized); - JpgImageLoader::save(AFileOutputStream("test.jpg"), resized); - return "<{}>data:image/jpg;base64,{}"_format(EMBEDDING_TAG, jpg.toBase64String(), EMBEDDING_TAG); -} - - AJson OpenAIChatImpl::makeQueryString(Params params, const IOpenAIChat::Session& messages) { ALOG_TRACE(LOG_TAG) << "makeQueryString"; AUI_ASSERT(!messages.sessionId.empty()); @@ -79,8 +68,8 @@ AJson OpenAIChatImpl::makeQueryString(Params params, const IOpenAIChat::Session& if (params.seed) { json["seed"] = *params.seed; } - if (config().llmReasoningEffort) { - json["reasoning_effort"] = *config().llmReasoningEffort; + if (params.reasoningEffort) { + json["reasoning_effort"] = *params.reasoningEffort; } return json; } diff --git a/src/speech/VoiceGenerator.cpp b/src/speech/VoiceGenerator.cpp index 76034f2..10b904f 100644 --- a/src/speech/VoiceGenerator.cpp +++ b/src/speech/VoiceGenerator.cpp @@ -26,8 +26,10 @@ namespace { struct OutputContextDeleter { void operator()(AVFormatContext* ctx) const { - if (!ctx) return; - if (ctx->pb) avio_closep(&ctx->pb); + if (!ctx) + return; + if (ctx->pb) + avio_closep(&ctx->pb); avformat_free_context(ctx); } }; @@ -35,49 +37,54 @@ using OutputContextPtr = std::unique_ptr; struct CodecContextDeleter { void operator()(AVCodecContext* ctx) const { - if (ctx) avcodec_free_context(&ctx); + if (ctx) + avcodec_free_context(&ctx); } }; using CodecContextPtr = std::unique_ptr; struct SwrContextDeleter { void operator()(SwrContext* ctx) const { - if (ctx) swr_free(&ctx); + if (ctx) + swr_free(&ctx); } }; using SwrContextPtr = std::unique_ptr; struct AudioFifoDeleter { void operator()(AVAudioFifo* f) const { - if (f) av_audio_fifo_free(f); + if (f) + av_audio_fifo_free(f); } }; using AudioFifoPtr = std::unique_ptr; struct FrameDeleter { void operator()(AVFrame* f) const { - if (f) av_frame_free(&f); + if (f) + av_frame_free(&f); } }; using FramePtr = std::unique_ptr; struct PacketDeleter { void operator()(AVPacket* p) const { - if (p) av_packet_free(&p); + if (p) + av_packet_free(&p); } }; using PacketPtr = std::unique_ptr; -constexpr int OPUS_SAMPLE_RATE = 48000; // Opus always operates at 48 kHz internally -constexpr int64_t OPUS_BIT_RATE = 32000; // matches the previous "-b:a 32k" +constexpr int OPUS_SAMPLE_RATE = 48000; // Opus always operates at 48 kHz internally +constexpr int64_t OPUS_BIT_RATE = 32000; // matches the previous "-b:a 32k" // The encoder's preferred input sample format (libopus: S16; swresample converts the PCM to it). AVSampleFormat preferredSampleFormat(const AVCodec* codec) { #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100) const enum AVSampleFormat* fmts = nullptr; - if (avcodec_get_supported_config(nullptr, codec, AV_CODEC_CONFIG_SAMPLE_FORMAT, 0, - reinterpret_cast(&fmts), nullptr) >= 0 - && fmts && fmts[0] != AV_SAMPLE_FMT_NONE) { + if (avcodec_get_supported_config( + nullptr, codec, AV_CODEC_CONFIG_SAMPLE_FORMAT, 0, reinterpret_cast(&fmts), nullptr) >= 0 && + fmts && fmts[0] != AV_SAMPLE_FMT_NONE) { return fmts[0]; } #else @@ -111,7 +118,7 @@ bool encodeAndMux(AVCodecContext* codecCtx, AVFormatContext* oc, AVStream* strea } } -} // namespace +} // namespace /** * @brief Transcodes raw signed-16-bit-LE mono PCM into an OGG/Opus voice note using libav (libopus). @@ -154,9 +161,9 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam return false; } codecCtx->sample_rate = OPUS_SAMPLE_RATE; - codecCtx->bit_rate = OPUS_BIT_RATE; - codecCtx->sample_fmt = preferredSampleFormat(codec); - av_channel_layout_default(&codecCtx->ch_layout, 1); // mono + codecCtx->bit_rate = OPUS_BIT_RATE; + codecCtx->sample_fmt = preferredSampleFormat(codec); + av_channel_layout_default(&codecCtx->ch_layout, 1); // mono if (oc->oformat->flags & AVFMT_GLOBALHEADER) { codecCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; } @@ -168,7 +175,7 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam if (avcodec_parameters_from_context(stream->codecpar, codecCtx.get()) < 0) { return false; } - stream->time_base = AVRational{1, OPUS_SAMPLE_RATE}; + stream->time_base = AVRational { 1, OPUS_SAMPLE_RATE }; if (avio_open(&oc->pb, out.toStdString().c_str(), AVIO_FLAG_WRITE) < 0) { ALogger::err(LOG_TAG) << "PCM->Opus: can't open output file " << out; @@ -183,10 +190,9 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam AVChannelLayout monoLayout; av_channel_layout_default(&monoLayout, 1); SwrContext* rawSwr = nullptr; - if (swr_alloc_set_opts2(&rawSwr, - &codecCtx->ch_layout, codecCtx->sample_fmt, OPUS_SAMPLE_RATE, - &monoLayout, AV_SAMPLE_FMT_S16, sampleRate, - 0, nullptr) < 0 || !rawSwr) { + if (swr_alloc_set_opts2( + &rawSwr, &codecCtx->ch_layout, codecCtx->sample_fmt, OPUS_SAMPLE_RATE, &monoLayout, AV_SAMPLE_FMT_S16, sampleRate, 0, nullptr) < 0 || + !rawSwr) { ALogger::err(LOG_TAG) << "PCM->Opus: swr_alloc_set_opts2 failed"; return false; } @@ -196,7 +202,7 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam return false; } - const int frameSize = codecCtx->frame_size > 0 ? codecCtx->frame_size : 960; // 20 ms @ 48 kHz + const int frameSize = codecCtx->frame_size > 0 ? codecCtx->frame_size : 960; // 20 ms @ 48 kHz AudioFifoPtr fifo(av_audio_fifo_alloc(codecCtx->sample_fmt, 1, frameSize)); PacketPtr pkt(av_packet_alloc()); if (!fifo || !pkt) { @@ -205,8 +211,8 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam // Resample a chunk of input (or a flush when inData == nullptr) into the FIFO. auto pushResampled = [&](const uint8_t* inData, int nbIn) -> bool { - int outCount = static_cast(av_rescale_rnd(swr_get_delay(swr.get(), sampleRate) + nbIn, - OPUS_SAMPLE_RATE, sampleRate, AV_ROUND_UP)); + int outCount = static_cast( + av_rescale_rnd(swr_get_delay(swr.get(), sampleRate) + nbIn, OPUS_SAMPLE_RATE, sampleRate, AV_ROUND_UP)); if (outCount <= 0) { return true; } @@ -215,8 +221,7 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam return false; } int got = swr_convert(swr.get(), &outData, outCount, inData ? &inData : nullptr, nbIn); - bool ok = got >= 0 - && (got == 0 || av_audio_fifo_write(fifo.get(), reinterpret_cast(&outData), got) == got); + bool ok = got >= 0 && (got == 0 || av_audio_fifo_write(fifo.get(), reinterpret_cast(&outData), got) == got); av_freep(&outData); return ok; }; @@ -235,8 +240,8 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam if (!frame) { return false; } - frame->nb_samples = frameSize; - frame->format = codecCtx->sample_fmt; + frame->nb_samples = frameSize; + frame->format = codecCtx->sample_fmt; frame->sample_rate = OPUS_SAMPLE_RATE; av_channel_layout_copy(&frame->ch_layout, &codecCtx->ch_layout); if (av_frame_get_buffer(frame.get(), 0) < 0) { @@ -264,15 +269,60 @@ static bool transcodePcmToOpus(const AByteBuffer& pcm, const APath& out, int sam return true; } -#else // !KUNI_USE_FFMPEG +#else // !KUNI_USE_FFMPEG static bool transcodePcmToOpus(const AByteBuffer&, const APath&, int) { - ALogger::err(LOG_TAG) - << "PCM voice transcoding requires building with -DKUNI_USE_FFMPEG=ON (libav is not compiled in)"; + ALogger::err(LOG_TAG) << "PCM voice transcoding requires building with -DKUNI_USE_FFMPEG=ON (libav is not compiled " + "in)"; return false; } -#endif // KUNI_USE_FFMPEG +#endif // KUNI_USE_FFMPEG + +AFuture VoiceGenerator::generate(AString text, GenerateOpts opts) { + switch (config().recordVoiceBackend) { + case Config::TTSBackend::ELEVENLABS: { + ElevenLabsClient ttsClient { + .baseUrl = "https://api.elevenlabs.io/", + .apiKey = config().recordVoiceElevenLabsKey, + .voiceId = config().recordVoiceElevenLabsVoice, + }; + ElevenLabsClient::TextToSpeechRequest request { + .text = text, + .model_id = "eleven_v3", + .language_code = std::move(opts.languageCode), + .voice_settings = { .speed = opts.speed }, + }; + auto ttsResponse = co_await ttsClient.textToSpeech(request); + if (ttsResponse.audioData.empty()) { + throw AException("ElevenLabs returned empty audio data"); + } + co_return ttsResponse.audioData; + } + case Config::TTSBackend::OPENAI: { + OpenAISpeechClient ttsClient { + .baseUrl = config().recordVoiceOpenAIUrl, + .apiKey = config().recordVoiceOpenAIKey, + .model = config().recordVoiceOpenAIModel, + .voice = config().recordVoiceOpenAIVoice, + }; + OpenAISpeechClient::TextToSpeechRequest request { + .input = text, + .model = config().recordVoiceOpenAIModel, + .voice = config().recordVoiceOpenAIVoice, + .response_format = std::move(opts.responseFormat), + .num_step = 63, + .speed = opts.speed, + }; + auto ttsResponse = co_await ttsClient.textToSpeech(request); + if (ttsResponse.audioData.empty()) { + throw AException("OpenAI Speech returned empty audio data"); + } + co_return ttsResponse.audioData; + } + } + throw AException("unsupported"); +} AFuture VoiceGenerator::generate(AString text, AString languageCode, double speed) { ALogger::info(LOG_TAG) << "Generating voice message for text: " << text; @@ -281,65 +331,25 @@ AFuture VoiceGenerator::generate(AString text, ASt APath voiceDir("data/voice_messages"); voiceDir.makeDirs(); - AByteBuffer audioData; - - switch (config().recordVoiceBackend) { - case Config::TTSBackend::ELEVENLABS: { - ElevenLabsClient ttsClient{ - .baseUrl = "https://api.elevenlabs.io/", - .apiKey = config().recordVoiceElevenLabsKey, - .voiceId = config().recordVoiceElevenLabsVoice, - }; - ElevenLabsClient::TextToSpeechRequest request{ - .text = text, - .model_id = "eleven_v3", - .language_code = languageCode, - .voice_settings = {.speed = speed}, - }; - auto ttsResponse = co_await ttsClient.textToSpeech(request); - if (ttsResponse.audioData.empty()) { - throw AException("ElevenLabs returned empty audio data"); - } - audioData = std::move(ttsResponse.audioData); - break; - } - case Config::TTSBackend::OPENAI: { - OpenAISpeechClient ttsClient{ - .baseUrl = config().recordVoiceOpenAIUrl, - .apiKey = config().recordVoiceOpenAIKey, - .model = config().recordVoiceOpenAIModel, - .voice = config().recordVoiceOpenAIVoice, - }; - OpenAISpeechClient::TextToSpeechRequest request{ - .input = text, - .model = config().recordVoiceOpenAIModel, - .voice = config().recordVoiceOpenAIVoice, - .response_format = config().recordVoiceOpenAIFormat, - .num_step = 63, - .speed = speed, - }; - auto ttsResponse = co_await ttsClient.textToSpeech(request); - if (ttsResponse.audioData.empty()) { - throw AException("OpenAI Speech returned empty audio data"); - } - audioData = std::move(ttsResponse.audioData); - break; - } - } + AByteBuffer audioData = co_await generate(std::move(text), GenerateOpts{ + .languageCode = std::move(languageCode), + .speed = speed, + }); auto timestamp = std::chrono::system_clock::now().time_since_epoch().count(); // Telegram voice notes require OGG/Opus. Providers that return raw PCM (e.g. Gemini via RouterAI) // are transcoded here; anything else is saved as-is. - const bool needsPcmTranscode = config().recordVoiceBackend == Config::TTSBackend::OPENAI - && config().recordVoiceOpenAIFormat == "pcm"; + const bool needsPcmTranscode = + config().recordVoiceBackend == Config::TTSBackend::OPENAI && config().recordVoiceOpenAIFormat == "pcm"; APath outputPath; if (needsPcmTranscode) { outputPath = voiceDir / "{}.ogg"_format(timestamp); if (!transcodePcmToOpus(audioData, outputPath, config().recordVoiceOpenAIPcmSampleRate)) { - throw AException("Failed to transcode PCM voice message to OGG/Opus " - "(build with -DKUNI_USE_FFMPEG=ON)"); + throw AException( + "Failed to transcode PCM voice message to OGG/Opus " + "(build with -DKUNI_USE_FFMPEG=ON)"); } } else { outputPath = voiceDir / "{}.mp3"_format(timestamp); @@ -350,7 +360,7 @@ AFuture VoiceGenerator::generate(AString text, ASt ALogger::info(LOG_TAG) << "Voice message saved to: " << outputPath.absolute(); - co_return VoiceMessage{ .path = outputPath.absolute() }; + co_return VoiceMessage { .path = outputPath.absolute() }; } catch (const AException& e) { ALogger::err(LOG_TAG) << "Failed to generate voice message: " << e; throw; diff --git a/src/speech/VoiceGenerator.h b/src/speech/VoiceGenerator.h index b2b2352..704a2b7 100644 --- a/src/speech/VoiceGenerator.h +++ b/src/speech/VoiceGenerator.h @@ -13,4 +13,10 @@ class VoiceGenerator { }; AFuture generate(AString text, AString languageCode = "en", double speed = 1.0); + struct GenerateOpts { + AString languageCode = "en"; + AString responseFormat = "mp3"; + double speed = 1.0; + }; + AFuture generate(AString text, GenerateOpts config); }; diff --git a/src/speech/WhisperLive.cpp b/src/speech/WhisperLive.cpp new file mode 100644 index 0000000..eeee9eb --- /dev/null +++ b/src/speech/WhisperLive.cpp @@ -0,0 +1,88 @@ +// +// Created by alex2772 on 7/20/26. +// + +#include "WhisperLive.h" + +#include "AUI/Curl/ACurlMulti.h" +#include "AUI/Json/AJson.h" + +#include +#include + +static constexpr auto LOG_TAG = "WhisperLive"; + +namespace { +struct Packet { + struct Segment { + AString start{}; + AString end{}; + AString text; + bool completed{}; + }; + + AString uid; + AVector segments; + AString message; +}; +} + +AJSON_FIELDS(Packet, +(uid, "uid", AJsonFieldFlags::OPTIONAL) +(segments, "segments", AJsonFieldFlags::OPTIONAL) +(message, "message", AJsonFieldFlags::OPTIONAL) +); + +AJSON_FIELDS(Packet::Segment, +AJSON_FIELDS_ENTRY(start) +AJSON_FIELDS_ENTRY(end) +AJSON_FIELDS_ENTRY(text) +AJSON_FIELDS_ENTRY(completed) +); + +WhisperLive::WhisperLive(Config config): mWebsocket(_new(std::move(config.endpoint))) { + connect(mWebsocket->connected, [this, config = std::move(config)] { + static size_t id = 0; + AJson json{ + {"uid", "{}"_format(id++)}, + {"language", config.language}, + {"model", config.model}, + {"use_vad", true}, + {"task", "transcribe"}, + }; + mWebsocket->writeText(AJson::toString(json)); + mConnected = true; + }); + connect(mWebsocket->received, [this](AByteBuffer buffer) { + try { + ALOG_TRACE(LOG_TAG) << AString::fromUtf8(buffer); + auto packet = aui::from_json(AJson::fromBuffer(buffer)); + if (!packet.message.empty()) { + ALogger::info(LOG_TAG) << "Message: " << packet.message; + } + emit update(packet.segments | ranges::view::transform([](Packet::Segment& segment) { + return WhisperLive::Segment { + .start = segment.start.toFloat().valueOr(0), + .end = segment.end.toFloat().valueOr(0), + .text = std::move(segment.text), + .completed = segment.completed, + }; + }) | ranges::to_vector); + } catch (const AException& e) { + ALogger::err(LOG_TAG) << "Can't process packet: " << e; + } + }); + connect(mWebsocket->websocketClosed, [this] { + emit closed; + }); + + + ACurlMulti::global() << mWebsocket; +} + +void WhisperLive::writePcm16khz(std::vector samples) { + ACurlMulti::global().getThread()->enqueue([websocket = mWebsocket, samples = std::move(samples)] { + websocket->writeBinary(AByteBufferView(reinterpret_cast(samples.data()), samples.size() * sizeof(float))); + }); +} + diff --git a/src/speech/WhisperLive.h b/src/speech/WhisperLive.h new file mode 100644 index 0000000..bd4ccab --- /dev/null +++ b/src/speech/WhisperLive.h @@ -0,0 +1,34 @@ +#pragma once +#include "AUI/Common/AObject.h" +#include "AUI/Common/AString.h" +#include "AUI/Curl/AWebsocket.h" + +class WhisperLive: public AObject { +public: + struct Config { + AString endpoint = "ws://localhost:9001"; + AString language = "en"; + AString model = "base"; + }; + WhisperLive(Config config); + + void writePcm16khz(std::vector samples); + + bool connected() const noexcept { + return mConnected; + } + + struct Segment { + float start{}; + float end{}; + AString text; + bool completed{}; + }; + + emits> update; + emits<> closed; + +private: + AArc mWebsocket; + bool mConnected = false; +}; diff --git a/src/tools/send_telegram_message.cpp b/src/tools/send_telegram_message.cpp index 8162c23..4507061 100644 --- a/src/tools/send_telegram_message.cpp +++ b/src/tools/send_telegram_message.cpp @@ -27,7 +27,8 @@ OpenAITools::Tool tools::sendTelegramMessage( _ telegram, _ openAI, _ chat, - _>> messages) { + _>> messages, + SendTelegramMessageOpts opts) { struct State { int messagesInARow = 0; @@ -77,11 +78,14 @@ OpenAITools::Tool tools::sendTelegramMessage( openAI = std::move(openAI), chat = std::move(chat), state = _new(0), - messages = std::move(messages) + messages = std::move(messages), + opts = std::move(opts) ](OpenAITools::Ctx ctx) -> AFuture { - if (state->messagesInARow > 10) { - // stupid AI can't recognize it spams messages despite the warning - throw AException("Too many messages in a row. Don't spam!"); + if (opts.maxPossibleMessagesInARow) { + if (state->messagesInARow >= *opts.maxPossibleMessagesInARow) { + // stupid AI can't recognize it spams messages despite the warning + throw AException("Too many messages in a row. Don't spam!"); + } } auto isTyping = _new(true); diff --git a/src/tools/send_telegram_message.h b/src/tools/send_telegram_message.h index 43ffcd3..074ca99 100644 --- a/src/tools/send_telegram_message.h +++ b/src/tools/send_telegram_message.h @@ -4,9 +4,13 @@ #include "telegram/ITelegramClient.h" namespace tools { +struct SendTelegramMessageOpts { + AOptional maxPossibleMessagesInARow = 10; +}; OpenAITools::Tool sendTelegramMessage( _ telegram, _ openAI, _ chat, - _>> messages); + _>> messages, + SendTelegramMessageOpts opts = {}); } diff --git a/src/util/important_things_to_remember.h b/src/util/important_things_to_remember.h index 0e6e688..a713db4 100644 --- a/src/util/important_things_to_remember.h +++ b/src/util/important_things_to_remember.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace util { AFuture importantThingsToRemember(AppBase& app, IOpenAIChat& openAI, IOpenAIChat::Session context, AStringView previousWorkingMemory) { @@ -81,6 +82,5 @@ AFuture importantThingsToRemember(AppBase& app, IOpenAIChat& openAI, IO } co_return content; } - } -} \ No newline at end of file +} From 613e23868c47f857e34cb0cb51223e286d877eb1 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Wed, 12 Aug 2026 20:15:05 +0300 Subject: [PATCH 27/32] update(reply_to): inline reply to content -> to attribute (rely on view_messages_around) change reason: the llm was misleaded/confused by nested xml tags --- src/llmui/telegram.cpp | 154 ++++++++++++++--------------- src/tools/view_messages_around.cpp | 2 +- 2 files changed, 74 insertions(+), 82 deletions(-) diff --git a/src/llmui/telegram.cpp b/src/llmui/telegram.cpp index a606be5..34ed2bc 100644 --- a/src/llmui/telegram.cpp +++ b/src/llmui/telegram.cpp @@ -466,104 +466,96 @@ AFuture llmui::formatChatHistoryMessage( } } + if (msg.reply_to_ && msg.reply_to_->get_id() == td::td_api::messageReplyToMessage::ID) { + try { + auto reply = td::td_api::move_object_as(std::move(msg.reply_to_)); + formattedXmlTag += " reply_to=\"{}\""_format(reply->message_id_); + } catch (const AException& e) { + } + } auto result = "<{}>\n"_format(formattedXmlTag); - if (xmlTag != "reply_to") { - if (msg.reply_to_ && msg.reply_to_->get_id() == td::td_api::messageReplyToMessage::ID) { - try { - auto reply = td::td_api::move_object_as(std::move(msg.reply_to_)); - auto replyToMsg = co_await telegram.getMessage(msg.chat_id_, reply->message_id_); - result += co_await llmui::formatChatHistoryMessage(telegram, *replyToMsg, chat, openAI, temporaryContext, "reply_to"); - } catch (const AException& e) { - if (e.getMessage().contains("Not Found")) { - result += "Deleted Message"; - } else { - ALogger::err("formatChatHistoryMessage") << e; - } - } + + if (msg.content_->get_id() == td::td_api::messagePhoto::ID) { + auto& photo = static_cast(*msg.content_); + if (auto targetPhotoIt = ranges::max_element( + photo.photo_->sizes_, std::less {}, [&](const auto& s) { return s->width_ * s->height_; }); + targetPhotoIt != photo.photo_->sizes_.end()) { + result += co_await llmui::image(temporaryContext, openAI, co_await fetchMedia(telegram, targetPhotoIt->get()->photo_)); } + } - if (msg.content_->get_id() == td::td_api::messagePhoto::ID) { - auto& photo = static_cast(*msg.content_); - if (auto targetPhotoIt = ranges::max_element( - photo.photo_->sizes_, std::less {}, [&](const auto& s) { return s->width_ * s->height_; }); - targetPhotoIt != photo.photo_->sizes_.end()) { - result += co_await llmui::image(temporaryContext, openAI, co_await fetchMedia(telegram, targetPhotoIt->get()->photo_)); - } + if (msg.content_->get_id() == td::td_api::messageSticker::ID) { + auto& sticker = static_cast(*msg.content_); + AString xmlTag = "sticker"; + if (!sticker.sticker_->emoji_.empty()) { + checkForMaliciousPayloads(sticker.sticker_->emoji_); + xmlTag += " emoji=\"{}\""_format(sticker.sticker_->emoji_); + } + if (config().capabilityUseStickers) { + xmlTag += " sticker_id=\"{}\""_format(sticker.sticker_->id_); } + if (sticker.sticker_->sticker_) { + result += co_await llmui::image( + temporaryContext, openAI, co_await fetchMedia(telegram, sticker.sticker_->sticker_), xmlTag); + } + const auto id = sticker.sticker_->id_; + tools::stickers::knownStickers()[id] = std::move(sticker.sticker_); + } - if (msg.content_->get_id() == td::td_api::messageSticker::ID) { - auto& sticker = static_cast(*msg.content_); - AString xmlTag = "sticker"; - if (!sticker.sticker_->emoji_.empty()) { - checkForMaliciousPayloads(sticker.sticker_->emoji_); - xmlTag += " emoji=\"{}\""_format(sticker.sticker_->emoji_); - } - if (config().capabilityUseStickers) { - xmlTag += " sticker_id=\"{}\""_format(sticker.sticker_->id_); - } - if (sticker.sticker_->sticker_) { - result += co_await llmui::image( - temporaryContext, openAI, co_await fetchMedia(telegram, sticker.sticker_->sticker_), xmlTag); - } - const auto id = sticker.sticker_->id_; - tools::stickers::knownStickers()[id] = std::move(sticker.sticker_); + if (msg.content_->get_id() == td::td_api::messageGift::ID) { + auto& gift = static_cast(*msg.content_); + auto xmlTag = "gift cost=\"{} stars\""_format(gift.gift_->star_count_); + if (gift.text_) { + checkForMaliciousPayloads(gift.text_->text_); + xmlTag += " text=\"" + gift.text_->text_ + "\""; } - if (msg.content_->get_id() == td::td_api::messageGift::ID) { - auto& gift = static_cast(*msg.content_); - auto xmlTag = "gift cost=\"{} stars\""_format(gift.gift_->star_count_); - if (gift.text_) { - checkForMaliciousPayloads(gift.text_->text_); - xmlTag += " text=\"" + gift.text_->text_ + "\""; + if (gift.gift_->sticker_) { + if (!gift.gift_->sticker_->emoji_.empty()) { + checkForMaliciousPayloads(gift.gift_->sticker_->emoji_); + xmlTag += " emoji=\"{}\""_format(gift.gift_->sticker_->emoji_); } - if (gift.gift_->sticker_) { - if (!gift.gift_->sticker_->emoji_.empty()) { - checkForMaliciousPayloads(gift.gift_->sticker_->emoji_); - xmlTag += " emoji=\"{}\""_format(gift.gift_->sticker_->emoji_); - } - - result += co_await llmui::image( - temporaryContext, openAI, co_await fetchMedia(telegram, gift.gift_->sticker_->sticker_), xmlTag); - } else { - result += "<{} />"_format(xmlTag); - } + result += co_await llmui::image( + temporaryContext, openAI, co_await fetchMedia(telegram, gift.gift_->sticker_->sticker_), xmlTag); + } else { + result += "<{} />"_format(xmlTag); } + } - if (msg.content_->get_id() == td::td_api::messageAnimation::ID) { - auto& animation = static_cast(*msg.content_); - if (animation.animation_->thumbnail_) { - result += co_await llmui::image( - temporaryContext, - openAI, - co_await fetchMedia(telegram, animation.animation_->thumbnail_->file_), - "animation"); - } + if (msg.content_->get_id() == td::td_api::messageAnimation::ID) { + auto& animation = static_cast(*msg.content_); + if (animation.animation_->thumbnail_) { + result += co_await llmui::image( + temporaryContext, + openAI, + co_await fetchMedia(telegram, animation.animation_->thumbnail_->file_), + "animation"); } + } - if (msg.content_->get_id() == td::td_api::messageVideo::ID) { - auto& videoMsg = static_cast(*msg.content_); - if (videoMsg.video_) { - result += co_await llmui::video( - temporaryContext, - openAI, - co_await fetchMedia(telegram, videoMsg.video_->video_), - "video"); - } + if (msg.content_->get_id() == td::td_api::messageVideo::ID) { + auto& videoMsg = static_cast(*msg.content_); + if (videoMsg.video_) { + result += co_await llmui::video( + temporaryContext, + openAI, + co_await fetchMedia(telegram, videoMsg.video_->video_), + "video"); } + } - if (msg.content_->get_id() == td::td_api::messageVoiceNote::ID) { - auto& voiceNote = static_cast(*msg.content_); - if (voiceNote.voice_note_) { - result += co_await llmui::voiceMessageTranscription(telegram, voiceNote, msg.chat_id_, msg.id_, openAI); - } + if (msg.content_->get_id() == td::td_api::messageVoiceNote::ID) { + auto& voiceNote = static_cast(*msg.content_); + if (voiceNote.voice_note_) { + result += co_await llmui::voiceMessageTranscription(telegram, voiceNote, msg.chat_id_, msg.id_, openAI); } + } - if (msg.content_->get_id() == td::td_api::messageVideoNote::ID) { - auto& videoNote = static_cast(*msg.content_); - if (videoNote.video_note_) { - result += co_await llmui::videoNoteTranscription(telegram, openAI, videoNote, msg.chat_id_, msg.id_); - } + if (msg.content_->get_id() == td::td_api::messageVideoNote::ID) { + auto& videoNote = static_cast(*msg.content_); + if (videoNote.video_note_) { + result += co_await llmui::videoNoteTranscription(telegram, openAI, videoNote, msg.chat_id_, msg.id_); } } diff --git a/src/tools/view_messages_around.cpp b/src/tools/view_messages_around.cpp index 6018b41..fa6b235 100644 --- a/src/tools/view_messages_around.cpp +++ b/src/tools/view_messages_around.cpp @@ -25,7 +25,7 @@ OpenAITools::Tool tools::viewMessagesAround(_ telegram, _ tag you've previously seen.\n" + "- `message_id` is taken from the `message_id` or `reply_to` attribute of a tag you've previously seen.\n" "- The target message itself is included in the result and marked with a `target` attribute.", .parameters = { From 065ab7be79a79df987ca018ee0ff0bf483a4d46f Mon Sep 17 00:00:00 2001 From: alex2772 Date: Fri, 21 Aug 2026 04:08:35 +0300 Subject: [PATCH 28/32] fix(JsonAsLongInt): on stupid models --- src/util/json_utils.h | 14 ++++++++++++-- tests/JsonUtilsUnitTest.cpp | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/util/json_utils.h b/src/util/json_utils.h index e566f8a..3df8262 100644 --- a/src/util/json_utils.h +++ b/src/util/json_utils.h @@ -14,8 +14,18 @@ inline AOptional jsonAsLongInt(const AJson& v) { } if (auto s = v.asStringOpt()) { try { - auto result = static_cast(std::stod(s->toStdString())); - ALogger::warn("jsonAsLongInt") << "coerced string " << AJson::toString(v) << " to long long: " << result; + const bool isScientific = s->contains("e") || s->contains("E"); + if (isScientific) { + // conversion with losses. + auto result = static_cast(std::stod(s + ->replacedAll(",", "") + .toStdString())); + ALogger::warn("jsonAsLongInt") << "coerced string " << AJson::toString(v) << " to long long: " << result; + return result; + } + auto result = std::stoll(s + ->replacedAll(",", "") + .toStdString()); return result; } catch (...) {} } diff --git a/tests/JsonUtilsUnitTest.cpp b/tests/JsonUtilsUnitTest.cpp index 6ad169a..37e6c73 100644 --- a/tests/JsonUtilsUnitTest.cpp +++ b/tests/JsonUtilsUnitTest.cpp @@ -125,6 +125,13 @@ TEST(JsonAsLongIntUnit, StringFloat) { EXPECT_EQ(*result, 3); } +TEST(JsonAsLongIntUnit, StringLarge) { + AJson v("5,233,462,979,260,874,478"); + auto result = util::jsonAsLongInt(v); + ASSERT_TRUE(result.hasValue()); + EXPECT_EQ(*result, 5'233'462'979'260'874'478); +} + // --- Edge cases: null, bool, array, object --- TEST(JsonAsLongIntUnit, NullReturnsNullopt) { From 78bbeb7ba2ebccea25793f3d22e427928d664dd0 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 23 Aug 2026 18:50:24 +0300 Subject: [PATCH 29/32] update(image,video): disable reasoning effort for describing images --- src/llmui/image.cpp | 1 + src/llmui/video.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/llmui/image.cpp b/src/llmui/image.cpp index 572ffaf..b7de7fd 100644 --- a/src/llmui/image.cpp +++ b/src/llmui/image.cpp @@ -59,6 +59,7 @@ AFuture llmui::image(std::span temporaryCon auto response = co_await openAI.chat({ .systemPrompt = prompt, .config = config().llmImageToText, + .reasoningEffort = "none", }, { { .role = IOpenAIChat::Message::Role::USER, .content = context }}); auto content = std::move(response.choices.at(0).message.content); if (content.trim().empty()) { diff --git a/src/llmui/video.cpp b/src/llmui/video.cpp index 9d7c064..8dea3a4 100644 --- a/src/llmui/video.cpp +++ b/src/llmui/video.cpp @@ -299,6 +299,7 @@ AFuture> llmui::videoFrames(std::span Date: Sun, 23 Aug 2026 18:55:15 +0300 Subject: [PATCH 30/32] attempt to fix large sticker_id for small models --- src/OpenAIChatImpl.cpp | 7 ++++++- src/config.cpp | 10 ++++++++++ src/llmui/telegram.cpp | 2 +- src/tools/stickers.cpp | 6 +++--- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/OpenAIChatImpl.cpp b/src/OpenAIChatImpl.cpp index e5569a7..0f77d89 100644 --- a/src/OpenAIChatImpl.cpp +++ b/src/OpenAIChatImpl.cpp @@ -69,7 +69,7 @@ AJson OpenAIChatImpl::makeQueryString(Params params, const IOpenAIChat::Session& json["seed"] = *params.seed; } if (params.reasoningEffort) { - json["reasoning_effort"] = *params.reasoningEffort; + json["reasoning_effort"] = params.reasoningEffort->replacedAll("off", "none"); } return json; } @@ -108,6 +108,11 @@ AFuture OpenAIChatImpl::makeHttpRequest(Endpoint endpoint, std::string qu _ OpenAIChatImpl::chatStreaming(Params params, IOpenAIChat::Session messages) { messages.insert(messages.begin(), {Message::Role::SYSTEM_PROMPT, params.systemPrompt}); + + if (messages.isTooLarge()) { + throw AException("context is too large"); + } + AString query = [&] { auto json = makeQueryString(params, messages); json["stream"] = true; diff --git a/src/config.cpp b/src/config.cpp index ebb6dac..21201a2 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -440,6 +440,16 @@ static const std::unordered_map CONFIG_COMMENTS = { "Amount of Kuni's subpersons that process messages. 1 is totally fine.\n" "This was implemented as a countermeasure to spamming Kuni's DM.", }, + { + "misc.reasoning_effort", + "`reasoning_effort` passed to the OpenAI api.\n" + "- \"none\" - unset (aka by default)\n" + "- \"off\" - reasoning completely disabled (replaced by \"none\")\n" + "- \"low\" - low reasoning effort\n" + "- \"medium\" - medium reasoning effort\n" + "- \"high\" - high reasoning effort\n" + , + }, }; static constexpr auto CONFIG_TOML = "config.toml"; diff --git a/src/llmui/telegram.cpp b/src/llmui/telegram.cpp index 34ed2bc..d71ece6 100644 --- a/src/llmui/telegram.cpp +++ b/src/llmui/telegram.cpp @@ -492,7 +492,7 @@ AFuture llmui::formatChatHistoryMessage( xmlTag += " emoji=\"{}\""_format(sticker.sticker_->emoji_); } if (config().capabilityUseStickers) { - xmlTag += " sticker_id=\"{}\""_format(sticker.sticker_->id_); + xmlTag += " sticker_id=\"{}\""_format(fmt::group_digits(sticker.sticker_->id_)); } if (sticker.sticker_->sticker_) { result += co_await llmui::image( diff --git a/src/tools/stickers.cpp b/src/tools/stickers.cpp index fc4a38c..b295b9c 100644 --- a/src/tools/stickers.cpp +++ b/src/tools/stickers.cpp @@ -38,7 +38,7 @@ AFuture llmui::listFavoriteStickers(ITelegramClient& telegram, IOpenAIC for (auto& sticker : co_await getSavedStickers(telegram)) { llmui::checkForMaliciousPayloads(sticker->emoji_); const auto xmlTag = - "sticker sticker_id=\"{}\" emoji=\"{}\""_format(sticker->id_, sticker->emoji_); + "sticker sticker_id=\"{}\" emoji=\"{}\""_format(fmt::group_digits(sticker->id_), sticker->emoji_); // we rely on cache in llmui::image. out += co_await llmui::image({}, openAI, co_await llmui::fetchMedia(telegram, sticker->sticker_), xmlTag); out += "\n"; @@ -63,7 +63,7 @@ OpenAITools::Tool tools::stickers::save(_ telegram) { .description = "Saves sticker so you can use them later. Use this if you liked a sticker", .parameters = { .properties = { - {"sticker_id", {.type = "integer", .description = "sticker_id of the sticker you would like to save"}}, + {"sticker_id", {.type = "string", .description = "sticker_id of the sticker you would like to save"}}, }, .required = {"sticker_id"}, }, @@ -92,7 +92,7 @@ OpenAITools::Tool tools::stickers::send(_ telegram, _title_), .parameters = { .properties = { - {"sticker_id", {.type = "integer", .description = "sticker_id of the sticker you would like to send"}}, + {"sticker_id", {.type = "string", .description = "sticker_id of the sticker you would like to send"}}, {"reply_to_message_id", { .type = "integer", .description = "If specified, the message will be rendered as a reply to the " From 56b07056e03ba24bdae6912c698f63104d3fa982 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 23 Aug 2026 18:55:43 +0300 Subject: [PATCH 31/32] update(send_telegram_message): encourage reply_to in group chats --- src/tools/send_telegram_message.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tools/send_telegram_message.cpp b/src/tools/send_telegram_message.cpp index 4507061..e760ceb 100644 --- a/src/tools/send_telegram_message.cpp +++ b/src/tools/send_telegram_message.cpp @@ -116,6 +116,14 @@ OpenAITools::Tool tools::sendTelegramMessage( const auto allowTypos = ctx.args["allow_typos"].asBoolOpt().valueOr(true); const auto replyTo = [&]() -> int64_t { const auto value = util::jsonAsLongInt(ctx.args["reply_to_message_id"]).valueOr(0); + if (value == 0) { + if (chat->type_->get_id() == td::td_api::chatTypeBasicGroup::ID + || chat->type_->get_id() == td::td_api::chatTypeSupergroup::ID) { + if (std::uniform_real_distribution(0.f, 1.f)(gRandomEngine) > 0.5f) { + throw AException("please specify `reply_to_message_id` to address specific message in group chat"); + } + } + } if (state->lastReplyToMessageId == value) { // we don't need to reply to the same message multiple times in a row. return 0; From a97fa5ff8e899ca3980b83472ed3ebfe232b6199 Mon Sep 17 00:00:00 2001 From: alex2772 Date: Sun, 23 Aug 2026 18:57:07 +0300 Subject: [PATCH 32/32] fix(App): check if Kuni has the chat in chat list before passing event to LLM Kuni started subscribing to random chats because she received random updateNewMessage events from then, despite not being subscribed. --- CMakeLists.txt | 3 ++- src/App.h | 5 +++++ src/IOpenAIChat.cpp | 12 ++++++++++++ src/IOpenAIChat.h | 2 ++ src/Worker.cpp | 7 +++++++ src/telegram/TelegramClientImpl.cpp | 7 +++++++ 6 files changed, 35 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 552588f..4752761 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ auib_mark_var_forwardable(AUI_COROUTINES) # import AUI auib_import(aui https://github.com/aui-framework/aui - COMPONENTS core json curl crypt image views + COMPONENTS core json curl crypt image views audio VERSION ${AUI_VERSION}) @@ -75,6 +75,7 @@ aui_link(${PROJECT_NAME} PUBLIC aui::crypt aui::image aui::views + aui::audio Td::TdStatic toml11::toml11 prometheus-cpp::pull diff --git a/src/App.h b/src/App.h index 4227ae1..70c8eb7 100644 --- a/src/App.h +++ b/src/App.h @@ -351,6 +351,11 @@ class App : public AppBase { } auto chat = co_await mTelegram->getChat(u->message_->chat_id_); + if (chat->chat_lists_.empty()) { + // telegram may report events for random chats we are not subscribed to. + co_return; + } + // ALogger::info(LOG_TAG) << "Queued chat: " << chat->title_; if (chat->notification_settings_) { if (chat->notification_settings_->mute_for_ > 0) { diff --git a/src/IOpenAIChat.cpp b/src/IOpenAIChat.cpp index 9a13bce..b752024 100644 --- a/src/IOpenAIChat.cpp +++ b/src/IOpenAIChat.cpp @@ -8,6 +8,10 @@ #include "AUI/Util/ARandom.h" #include +#include +#include + +static constexpr auto MAX_REQUEST_LENGTH = 200'000; // ~65k tokens static constexpr auto LOG_TAG = "IOpenAIChat"; @@ -87,6 +91,14 @@ AString IOpenAIChat::embedBinary(AStringView mimeType, AByteBufferView data) { return "<{}>data:{};base64,{}"_format(EMBEDDING_TAG, mimeType, data.toBase64String(), EMBEDDING_TAG); } +bool IOpenAIChat::Session::isTooLarge() const { + static constexpr auto TRANSFORM_LENGTH = ranges::view::transform([](const IOpenAIChat::Message& i) { + return i.reasoning.utf8().length() + i.reasoning_content.utf8().length() + i.content.utf8().length(); + }); + const auto length = ranges::accumulate(*this | TRANSFORM_LENGTH, size_t(0)); + return length >= MAX_REQUEST_LENGTH; +} + AString IOpenAIChat::Session::nextSessionId() { static ARandom r; return r.nextUuid().toString(); diff --git a/src/IOpenAIChat.h b/src/IOpenAIChat.h index 52ea6f4..b7fe1c6 100644 --- a/src/IOpenAIChat.h +++ b/src/IOpenAIChat.h @@ -122,6 +122,8 @@ struct IOpenAIChat { using AVector::AVector; AString sessionId = nextSessionId(); + bool isTooLarge() const; + private: static AString nextSessionId(); }; diff --git a/src/Worker.cpp b/src/Worker.cpp index 6746516..13ad09a 100644 --- a/src/Worker.cpp +++ b/src/Worker.cpp @@ -153,6 +153,13 @@ AFuture<> Worker::handleNotification(std::shared_ptr alive, NotificationMa .content = std::move(notification.message), }; + if (mTemporaryContext.isTooLarge()) { + // we are stuck; ignore the event + mLogger.warn("AppBase") << "Fatal overflow of context; can't recover"; + mTemporaryContext.clear(); + co_return; + } + // naxyi was here. // the reasons why I have moved it below diary lookup: // 1. Each lookup adds ~1s delay. So each time LLM uses send_telegram_message, there is a diary diff --git a/src/telegram/TelegramClientImpl.cpp b/src/telegram/TelegramClientImpl.cpp index bcc8a9b..4874edf 100644 --- a/src/telegram/TelegramClientImpl.cpp +++ b/src/telegram/TelegramClientImpl.cpp @@ -311,6 +311,13 @@ void TelegramClientImpl::commonHandler(td::tl::unique_ptr ob dst->populated.supplyValue(); } }, + [this](td::td_api::updateChatAddedToList& u) { + auto chat = getChat(u.chat_id_); + if (!chat.hasValue()) { + return; + } + (*chat)->chat_lists_.push_back(std::move(u.chat_list_)); + }, [this](td::td_api::updateChatPosition& u) { auto chat = getChat(u.chat_id_); if (!chat.hasValue()) {