From 14fbf479e039ea7d1601c8034924e6231bb0caef Mon Sep 17 00:00:00 2001 From: Danswar <48102227+Danswar@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:07:55 -0300 Subject: [PATCH] Collapse Telegram polling-error noise into onset and recovery lines (#127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Collapse Telegram polling-error noise into onset and recovery lines Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted console error for every failed poll, at error level and outside the application logger. The polling loop also retries on a fixed 300ms interval with no backoff, so a brief Telegram gateway outage produces several lines per second — a recent nine-second 502 window logged 27 lines — and sustained retries earn a 429 on top of the original 502 because the loop ignores Telegram's retry-after. Attach a listener that reports one line per distinct failure plus one on recovery with duration and attempt count. The same outage becomes two lines, an escalation from 502 to 429 is still surfaced, and an outage never appears to stay open. Both lines are logged at warn so they remain visible together under a warn-level log configuration. No unit tests added: the repo's jest config has rootDir="src" but the sources live at the repo root, so `yarn test` finds zero tests today. * Keep unrecovered polling outages visible in the logs Review follow-up on the first commit. The signature used the raw error message compared against the previous one only, which fails in the two cases that actually occur: "Too Many Requests: retry after N" counts down and a connect failure carries a rotating gateway address, so each retry looked like a new failure, and alternating errors re-reported on every poll. Mask digits out of the signature and track the signatures already reported within the current outage. A permanently failing poll — a revoked token answers 401 forever — reported once and then stayed silent, because the recovery timer only fires once errors stop. Repeat the report every five minutes while the outage is open. Also move the outage shape into telegram.types.ts next to the other state types, and truncate the error text, which wraps the upstream response body. * Bound the outage signature set and stop over-claiming recovery Second review follow-up. The signature prefixed the error code onto a message that the library already prefixes with that same code, so the prefix discriminated nothing. Use the message alone. The signature set had no upper bound: a message carrying a per-attempt token that survives digit masking - a request id in an upstream error body reaches the parse-error branch verbatim - produced a fresh signature per poll, which defeated the collapsing and grew the set without limit. Cap it, past which only the periodic report remains. A replay of 2000 such polls now yields 22 lines and 20 retained signatures instead of 2000 of each. An error arriving more than the grace period after the previous one opened and closed its own outage, so an isolated blip cost two lines where it used to cost one. Skip the closing line for a single attempt. The closing line said "recovered", but nothing probes the poll - the grace period only establishes that no further error arrived, and a request that never settles would look the same. Say what is actually known instead. --- socialmedia/telegram/telegram.service.ts | 66 +++++++++++++++++++++++- socialmedia/telegram/telegram.types.ts | 10 ++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index ca6cc86..e7b8d77 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -30,11 +30,20 @@ import { PositionProposalMessage } from './messages/PositionProposal.message'; import { SavingUpdateMessage } from './messages/SavingUpdate.message'; import { StablecoinBridgeMessage } from './messages/StablecoinBridgeUpdate.message'; import { TradeMessage } from './messages/Trade.message'; -import { TelegramGroupState, TelegramState } from './telegram.types'; +import { TelegramGroupState, TelegramPollingOutage, TelegramState } from './telegram.types'; // Stay under telegram per-chat rate limit (~30 msg/s) when bursting position-lifecycle alerts. const TELEGRAM_THROTTLE_MS = 100; +// Consider polling healthy again once no further error arrived for this long. +const POLLING_RECOVERY_GRACE_MS = 30_000; +// Repeat an ongoing outage at this interval so a permanent failure never goes silent. +const POLLING_REPORT_INTERVAL_MS = 300_000; +// Telegram errors wrap the upstream response body, which is not always short. +const MAX_POLLING_ERROR_LENGTH = 200; +// Beyond this many distinct failures one outage reports on its interval only. +const MAX_POLLING_SIGNATURES = 20; + @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly logger = new Logger(this.constructor.name); @@ -42,6 +51,8 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; + private pollingOutage: TelegramPollingOutage | null = null; + private pollingRecoveryTimer: NodeJS.Timeout | null = null; constructor( private readonly socialMediaService: SocialMediaService, @@ -51,6 +62,10 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly position: PositionsService, private readonly challenge: ChallengesService ) { + // Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted + // console error for every failed poll, and it retries on a fixed interval with no backoff. + this.bot.on('polling_error', (error) => this.onPollingError(error)); + const time: number = Date.now() + 365 * 24 * 60 * 60 * 1000; this.telegramState = { @@ -363,6 +378,55 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { } } + // Telegram's gateway answers 502/504 during its own restarts, so a short outage would + // otherwise produce several identical lines per second. Report one line per distinct + // failure, repeat an outage that never clears every POLLING_REPORT_INTERVAL_MS, and close + // it with one line — all at warn, so an outage never appears to stay open. + private onPollingError(error: Error): void { + const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); + // The library prefixes its own error code onto the message, so the message alone + // identifies the failure. Digits carry the volatile parts — the countdown in + // "retry after 5", the rotating gateway address in a connect failure — and are masked + // out to keep a repeating failure on one line. + const signature = message.replace(/\d+/g, '#'); + const now = Date.now(); + + if (!this.pollingOutage) { + this.pollingOutage = { since: now, lastAt: now, lastReportAt: 0, attempts: 0, signatures: new Set() }; + } + + this.pollingOutage.attempts++; + this.pollingOutage.lastAt = now; + + // Past the cap only the periodic report remains, which bounds the log volume and the + // set itself if a message carries a token that survives digit masking. + const isNewFailure = !this.pollingOutage.signatures.has(signature) && this.pollingOutage.signatures.size < MAX_POLLING_SIGNATURES; + if (isNewFailure) this.pollingOutage.signatures.add(signature); + + if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { + this.pollingOutage.lastReportAt = now; + this.logger.warn(`Telegram polling failing (attempt ${this.pollingOutage.attempts}): ${message}`); + } + + if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); + this.pollingRecoveryTimer = setTimeout(() => this.onPollingRecovered(), POLLING_RECOVERY_GRACE_MS); + this.pollingRecoveryTimer.unref(); + } + + private onPollingRecovered(): void { + if (!this.pollingOutage) return; + + const { since, lastAt, attempts } = this.pollingOutage; + this.pollingOutage = null; + this.pollingRecoveryTimer = null; + + // A single failure needs no closing line. Silence only proves that no further error + // arrived — the poll itself is not probed here, so the line does not claim more. + if (attempts <= 1) return; + + this.logger.warn(`Telegram polling errors stopped after ${Math.round((lastAt - since) / 1000)}s and ${attempts} attempts`); + } + private async doSendMessage(group: string | number, message: string): Promise { await this.bot.sendMessage(group.toString(), message, { parse_mode: 'Markdown', disable_web_page_preview: true }); } diff --git a/socialmedia/telegram/telegram.types.ts b/socialmedia/telegram/telegram.types.ts index 4158b4c..6cb09bf 100644 --- a/socialmedia/telegram/telegram.types.ts +++ b/socialmedia/telegram/telegram.types.ts @@ -19,6 +19,16 @@ export type TelegramSubscriptionState = { tradeUpdates: number; }; +// @dev: in-memory state of an ongoing telegram polling outage, used to report it once +// per distinct failure instead of once per retry +export type TelegramPollingOutage = { + since: number; + lastAt: number; + lastReportAt: number; + attempts: number; + signatures: Set; +}; + export type TelegramGroupState = { apiVersion: string; createdAt: number;