diff --git a/README.md b/README.md index 7b0f755..5235abe 100644 --- a/README.md +++ b/README.md @@ -14,415 +14,198 @@ npm i --save @kne/fastify-statistics ### 项目概述 -`@kne/fastify-statistics` 是一个基于 Fastify 的数据采集与周期统计插件，提供数据采集、多周期聚合统计和灵活查询功能。 +`@kne/fastify-statistics` 是一个基于 Fastify 的数据采集与多周期聚合统计插件。它提供从原始数据上报到多级周期聚合、灵活查询与实时推送的完整数据管道，适用于 IoT 传感器数据、业务指标监控、多通道多属性聚合等场景。 -### 主要特性 +### 核心架构与数据流 -- **数据采集**：支持单条和批量数据上报，自动展开多属性对象和多级通道 -- **缓冲写入**：支持缓存缓冲模式，定时批量写入数据库，减少写入压力 -- **多周期聚合**：支持时(h)、日(d)、周(w)、月(m)、季(q)、年(y) 六种统计周期，自动通过 Cron 定时聚合 -- **聚合方法**：支持 sum、avg、count、min、max 五种聚合计算 -- **灵活查询**：按通道、时间范围、属性名、聚合方法查询统计结果，自动合并当前小时未聚合的原始数据 -- **SSE 实时推送**：基于 Server-Sent Events 的实时数据推送，支持自定义间隔、心跳保活、超时断开和缓存复用 -- **时区支持**：查询时支持传入客户端时区（IANA 格式），解决客户端与服务器时区不一致问题 -- **事务安全**：所有数据库写操作使用事务保证原子性，聚合操作支持幂等（upsert） -- **dayjs 时间处理**：所有时间操作统一使用 dayjs 处理，支持时区扩展 - -### 使用场景 - -- IoT 设备传感器数据采集与统计分析 -- 业务指标实时监控与多周期报表 -- 多通道多属性的数据聚合与趋势查询 - -### 使用方法 - -#### 快速开始 - -```js -const fastify = require('fastify')(); - -// 注册依赖插件 -fastify.register(require('@kne/fastify-sequelize'), { /* sequelize 配置 */ }); -fastify.register(require('@kne/fastify-cron'), { /* cron 配置 */ }); - -// 注册统计插件 -fastify.register(require('@kne/fastify-statistics'), { - prefix: '/api/statistics', - cache: redisCacheInstance, // 传入缓存实例启用缓冲模式和查询缓存 - compensationEnabled: true, // 启动时自动补偿聚合 - compensationBatchSize: 24, // 每次补偿最多24个时间窗口 - dataRetentionDays: 7, // 原始数据保留7天 - queryCacheEnabled: true, // 启用查询缓存 - queryCacheTTL: 30, // 实时查询缓存30秒 - queryCacheHistoryTTL: 3600, // 历史查询缓存1小时 - queryCacheMaxEntries: 100, // 内存缓存最大100条（无外部缓存时生效） - getAuthenticate: type => { - // type 为 'read' 或 'write'，返回认证信息 - } -}); - -fastify.listen({ port: 3000 }); +插件的核心理念是**逐级聚合**——原始数据采集后，按 h→d→w/m→q→y 的依赖链自动滚动聚合，每一级只从其直接上游读取数据，形成清晰的数据流管道： + +``` +数据采集(collect) → data-record 表 + ↓ h 周期聚合 (Cron: 每小时第1分钟) + period-stat (period=h) + ↓ d 周期聚合 (Cron: 每日0:01) + period-stat (period=d) + ↙ ↘ + w 聚合(周一0:01) m 聚合(每月1日0:01) + ↓ ↓ + period-stat(w) period-stat(m) + ↓ q 聚合(每季首月1日0:01) + period-stat(q) + ↓ y 聚合(每年1月1日0:01) + period-stat(y) ``` -#### Channel 与 AttributeName 的设计理念 +**聚合依赖关系**（`PERIOD_DEPENDENCY`）： -**Channel（数据通道）** 是数据的第一级分类维度，采用冒号分隔的多级结构（`a:b:c`）。它的核心思想是：**从宏观到微观的层级划分**。 +| 周期 | 数据来源 | 上游周期 | +|------|----------|----------| +| h | data-record | - | +| d | period-stat | h | +| w | period-stat | d | +| m | period-stat | d | +| q | period-stat | m | +| y | period-stat | q | -- **一级 channel**（如 `sales`）是根通道，对应唯一的 `channel-meta` 记录（标题、描述） -- **多级 channel**（如 `sales:beijing`、`sales:beijing:team-a`）是更细粒度的子通道 -- 查询时传入一级 channel 即可匹配所有子通道的数据 -- 同一根通道下的所有子通道共享同一个 `channel-meta` +### 六种统计周期 -**AttributeName（属性名）** 是数据的第二级分类维度，用于在同一 channel 下区分不同的数据指标。 +| 周期 | key | Cron 表达式 | 时间截断规则 | 数据来源 | +|------|-----|-------------|-------------|----------| +| 时 | h | `1 * * * *` | `startOf('hour')` | data-record | +| 日 | d | `1 0 * * *` | `startOf('day')` | period-stat(h) | +| 周 | w | `1 0 * * 1` | `startOf('week')+1天` (周一) | period-stat(d) | +| 月 | m | `1 0 1 * *` | `startOf('month')` | period-stat(d) | +| 季 | q | `1 0 1 1,4,7,10 *` | `Math.floor(month/3)*3` 月首日 | period-stat(m) | +| 年 | y | `1 0 1 1 *` | `startOf('year')` | period-stat(q) | -- 默认值为 `default`，适用于单一指标的场景 -- 当 `data` 传入对象时自动展开为多属性（如 `{revenue: 10000, orders: 50}` 拆分为两条记录） +### 聚合方法与级联计算 -#### 实际场景：企业部门数据统计 +五种聚合方法在聚合过程中协同计算，确保高级别周期可以正确推导： -假设一家公司要统计各部门的经营数据，我们可以这样设计 channel： +| 方法 | key | 从 data-record 聚合 | 从 period-stat 聚合 | +|------|-----|---------------------|---------------------| +| 合计 | sum | `SUM(data)` | 各子窗口 sum 求和 | +| 计数 | count | `COUNT(data)` | 各子窗口 count 求和 | +| 平均 | avg | `AVG(data)` | `sum总 / count总`（非 AVG(AVG)） | +| 最小 | min | `MIN(data)` | 各子窗口 min 取最小值 | +| 最大 | max | `MAX(data)` | 各子窗口 max 取最大值 | -``` -company ← 根通道：公司整体 -company:sales ← 子通道：销售部 -company:sales:beijing ← 子通道：销售部北京分部 -company:sales:shanghai ← 子通道：销售部上海分部 -company:rd ← 子通道：研发部 -company:rd:frontend ← 子通道：研发部前端组 -company:rd:backend ← 子通道：研发部后端组 -company:hr ← 子通道：人力资源部 -``` +**关键设计**：avg 不直接对上游 avg 取平均，而是用 sum/count 重新计算，避免二次平均偏差。 -对应的 `channel-meta` 只需为根通道 `company` 创建一条记录： - -| channel | title | description | -|---------|-------|-------------| -| company | 公司经营数据 | 各部门经营数据统计 | - -**采集数据**： - -```js -// 1. 销售部北京分部上报单指标（默认 attributeName=default） -await fastify.statistics.services.collect({ - channel: 'company:sales:beijing', - data: 58000, - unit: '元', - title: '公司', - description: '各部门经营数据统计' -}); - -// 2. 销售部上海分部上报多指标，unit 为字符串时所有属性共用同一单位 -await fastify.statistics.services.collect({ - channel: 'company:sales:shanghai', - data: { revenue: 72000, orders: 150 }, - unit: '元', - title: '公司', - description: '各部门经营数据统计' -}); - -// 3. 研发部前端组上报，unit 为对象时按 attributeName 映射不同单位 -await fastify.statistics.services.collect({ - channel: 'company:rd:frontend', - data: { tasks: 12, bugs: 3 }, - unit: { tasks: '个', bugs: '个' }, - title: '公司', - description: '各部门经营数据统计' -}); -``` +### 聚合区间语义 -采集后数据会自动展开并入库： - -| channel | attributeName | data | unit | -|---------|--------------|------|------| -| company | default | 58000 | 元 | -| company:sales | default | 58000 | 元 | -| company:sales:beijing | default | 58000 | 元 | -| company | revenue | 72000 | 元 | -| company | orders | 150 | 元 | -| company:sales | revenue | 72000 | 元 | -| company:sales | orders | 150 | 元 | -| company:sales:shanghai | revenue | 72000 | 元 | -| company:sales:shanghai | orders | 150 | 元 | -| ... | ... | ... | ... | - -> 通道展开规则：`company:sales:beijing` 自动展开为 `company`、`company:sales`、`company:sales:beijing` 三条记录，确保每一级都能查到汇总数据。 - -**查询数据**： - -```js -// 1. 查询销售部本月合计（仅自身） -const salesResult = await fastify.statistics.services.query({ - channels: ['company:sales'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - aggregates: ['sum'] -}); - -// 2. 查询公司所有部门的本月合计（传入一级 channel + includeChildren） -const companyResult = await fastify.statistics.services.query({ - channels: ['company'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - aggregates: ['sum'], - includeChildren: true -}); - -// 3. 同时查询多个通道（仅自身） -const multiResult = await fastify.statistics.services.query({ - channels: ['company:sales', 'company:rd'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - aggregates: ['sum'] -}); - -// 4. 查询 revenue 和 orders 两个属性的合计与平均 -const revenueResult = await fastify.statistics.services.query({ - channels: ['company'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - attributeNames: ['revenue', 'orders'], - aggregates: ['sum', 'avg'] -}); -``` +所有聚合使用**左闭右开区间** `[startTime, endTime)`： -**查询返回格式**： +- `startTime`：当前时间窗口的截断起始 +- `endTime`：下一个时间窗口的起始（`getNextStart(startTime)`） +- 查询条件使用 `Op.gte(startTime)` + `Op.lt(endTime)`，确保 `endTime` 所在时刻的数据**不被包含** -> **注意**：默认（`includeChildren=false`）只返回精确匹配通道的扁平列表。`includeChildren=true` 时按通道构建树形结构，子通道数据嵌套在 `children` 数组中。`data` 字段始终为对象（按属性名映射），例如单聚合时 `data` 为 `{"default": 58000}`，多聚合时 `data` 为 `{"sum": {"default": 58000}, "avg": {"default": 29000}}`。 +> 这一设计修复了此前使用 `Op.between`（闭区间）导致边界数据被重复聚合到两个窗口的 bug。 -查询销售部（`channels=['company:sales']`，默认不包含子通道）返回： +### Channel 通道层级设计 -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company:sales", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150 }, - "unit": { "default": "元", "revenue": "元", "orders": "元" } - } - ] -} -``` +**Channel（数据通道）** 采用冒号分隔的多级结构（`a:b:c`），核心思想是**从宏观到微观的层级划分**： -查询整个公司（`channels=['company'], includeChildren=true`）返回： +- 一级 channel（如 `sales`）是根通道，对应唯一的 `channel-meta` 记录 +- 多级 channel（如 `sales:beijing`、`sales:beijing:team-a`）是更细粒度的子通道 +- **采集时自动展开**：`company:sales:beijing` 展开为 `company`、`company:sales`、`company:sales:beijing` 三条记录 +- **查询时**：默认精确匹配；`includeChildren=true` 时匹配通道及所有子通道，返回树形结构 -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150, "tasks": 12, "bugs": 3 }, - "unit": { "default": "元", "revenue": "元", "orders": "元", "tasks": "个", "bugs": "个" } - } - ], - "children": [ - { - "channel": "company:sales", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150 }, - "unit": { "default": "元", "revenue": "元", "orders": "元" } - } - ], - "children": [ - { - "channel": "company:sales:beijing", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 58000 }, - "unit": { "default": "元" } - } - ] - }, - { - "channel": "company:sales:shanghai", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "revenue": 72000, "orders": 150 }, - "unit": { "revenue": "元", "orders": "元" } - } - ] - } - ] - }, - { - "channel": "company:rd", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "tasks": 12, "bugs": 3 }, - "unit": { "tasks": "个", "bugs": "个" } - } - ], - "children": [ - { - "channel": "company:rd:frontend", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "tasks": 12, "bugs": 3 }, - "unit": { "tasks": "个", "bugs": "个" } - } - ] - } - ] - } - ] - } - ] -} -``` +**AttributeName（属性名）** 是同一 channel 下的第二级分类维度： -查询多个通道（`channels=['company:sales','company:rd']`，默认不包含子通道）返回： +- 默认值 `default`，适用于单指标场景 +- `data` 传入对象时自动展开（如 `{revenue: 10000, orders: 50}` → 两条记录） +- `unit` 支持字符串（所有属性共用）或对象（按 attributeName 映射不同单位） -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company:sales", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150 }, - "unit": { "default": "元", "revenue": "元", "orders": "元" } - }, - { - "channel": "company:rd", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "tasks": 12, "bugs": 3 }, - "unit": { "tasks": "个", "bugs": "个" } - } - ] -} -``` +### 水位线机制与补偿聚合 -查询 revenue 和 orders 两个属性的合计与平均（`channels=['company']`, `attributeNames=['revenue','orders']`, `aggregates=['sum','avg']`）返回： +**水位线（aggregation-watermark）** 记录每个周期下一次应聚合的起始时间，是补偿聚合的核心： -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "sum": { "revenue": 72000, "orders": 150 }, "avg": { "revenue": 72000, "orders": 150 } }, - "unit": { "revenue": "元", "orders": "元" } - } - ] -} -``` +- 每个周期一条记录：`(period, nextTime)` +- 聚合完成后，`nextTime` 推进到下一窗口起始 +- 补偿时从 `nextTime` 开始逐步向前，追上当前时间 -> `channelMetas` 按 root channel 去重，所有子通道共享同一份元数据，避免数据冗余。 +**启动初始化流程**（`period-stat.init()`）： -#### Channel Meta 管理 +1. 按 h→d→w→m→q→y 顺序依次处理 +2. 对每个周期： + - 若水位线存在且过期 → 从水位线开始补偿 + - 若水位线不存在 → 调用 `determineStartFromSource` 从源数据推断起始点 + - 若无任何源数据 → 水位线设为当前截断时间（全新系统） +3. 逐窗口执行聚合，每完成一个窗口就推进水位线 +4. 上游未完成时自动先补偿上游（如聚合 d 前确保 h 已完成） -通道元数据在首次采集时自动创建，也可通过服务接口管理： +**`determineStartFromSource` 推断逻辑**： -```js -// 查询通道元数据 -const meta = await fastify.statistics.services.channelMeta.detail({ - channel: 'company' -}); +- h 周期：从 `data-record` 的 `MIN(time)` 截断到小时 +- 其他周期：从上游 `period-stat` 的 `MIN(time)` 截断到当前周期起始 -// 列出所有元数据 -const list = await fastify.statistics.services.channelMeta.list(); +> 早期版本使用 `MAX(time) + nextStart` 推断起始点，导致遗漏上游最早数据。现已改为 `MIN(time)` 截断，确保所有历史数据都被覆盖。 -// 按通道筛选 -const list = await fastify.statistics.services.channelMeta.list({ - filter: { channel: 'company' } -}); +**运行时补偿**（Cron 触发）： -// 修改元数据 -await fastify.statistics.services.channelMeta.save({ - channel: 'company', - title: '企业经营数据总览', - description: '全公司各部门经营指标汇总' -}); -``` +- 每个 Cron 周期调用 `compensate(period)` +- 每次最多处理 `compensationBatchSize` 个窗口（默认 24） +- 连续失败 `maxCompensationFailCount` 次（默认 3）后停止，下次 Cron 继续 +- 每个周期有独立锁（`compensatingLocks`），防止并发补偿 -#### SSE 实时推送 - -通过 HTTP 接口或程序化 API 获取实时统计数据推送： - -```js -// HTTP 接口：GET /api/statistics/sse?channel=company&aggregates=sum&interval=5 -// 浏览器端使用 EventSource 接收 -const eventSource = new EventSource('/api/statistics/sse?channel=company&aggregates=sum&interval=5'); -eventSource.onmessage = (event) => { - const data = JSON.parse(event.data); - console.log(data); // { channelMetas, list: [{ channel, items, children }] } -}; - -// 程序化调用（在 Fastify 路由中） -fastify.get('/my-sse', async (request, reply) => { - const sseContext = await fastify.statistics.services.sseStream.send(reply, { - name: 'my-sse-channel', - params: { - channel: ['company'], - startTime: new Date(Date.now() - 3600000).toISOString(), - endTime: new Date().toISOString(), - aggregates: ['sum'] - }, - fetchData: async (params) => { - return fastify.statistics.services.query(params); - }, - interval: 5, - heartbeatInterval: 30000, - maxDuration: 1800000 - }); - - // 可手动关闭 - // sseContext.close(); - - // 监听关闭事件 - sseContext.onClose(() => { - console.log('SSE 连接已关闭'); - }); -}); -``` +### 数据保留策略 -**SSE 事件类型**： +通过 Cron 定时清理过期数据，避免数据无限增长： -| 事件 | 说明 | -|------|------| -| `data`（默认） | 正常数据推送，内容为查询结果 JSON | -| `timeout` | 连接超过 maxDuration 后自动断开通知 | -| `error` | fetchData 出错时的错误事件 | -| 心跳（`: heartbeat`） | 保活注释行 | +| 数据类型 | 保留策略 | 安全检查 | +|----------|----------|----------| +| data-record | `dataRetentionDays` 天（默认 7 天） | 不超过 h 周期水位线 | +| period-stat(h) | 当月 | 不超过 d 周期水位线 | +| period-stat(d) | 当年 | 不超过 w、m 周期水位线 | +| period-stat(w) | 当年 | 无下游依赖 | +| period-stat(m/q/y) | 永久保留 | - | -**SSE 上下文方法**： +**安全检查**：删除前检查下游水位线，确保尚未聚合的数据不会被提前删除。 -| 方法 | 说明 | +### 缓冲写入模式 + +当配置 `cache` 实例时，采集数据先写入内存缓冲区，再定时批量持久化： + +- 缓冲区达到 `collectMaxBufferSize`（默认 1000）时触发 flush +- 定时 `collectFlushInterval`（默认 5000ms）自动 flush +- 缓冲区溢出上限 `collectMaxBufferOverflow`（默认 `maxBufferSize * 2`），超出时丢弃最旧数据 +- 进程关闭时（`onClose` hook）持久化缓冲区到 cache 并执行最终 flush +- 启动时从 cache 恢复缓冲区数据（`restoreBuffer`） + +无 cache 时，每次采集直接写入数据库（`collectImmediate`）。 + +### 查询缓存 + +查询结果自动缓存，减少重复查询的数据库压力： + +| 特性 | 说明 | |------|------| -| `isConnected()` | 返回当前连接状态 | -| `close()` | 手动关闭 SSE 连接 | -| `onClose(callback)` | 注册连接关闭回调，若已断开则立即执行 | +| 外部缓存 | 配置 `cache` 时使用，支持 TTL | +| 内存 LRU 缓存 | 无外部缓存时使用，最大 `queryCacheMaxEntries` 条 | +| 版本校验 | 缓存条目记录写入时的通道版本号，读取时校验版本是否变化 | +| TTL 策略 | 实时查询（endTime 在当前小时内）用 `queryCacheTTL`（30s），历史查询用 `queryCacheHistoryTTL`（3600s） | +| 补偿期间 | 正在补偿聚合时查询不走缓存，确保数据实时性 | +**缓存失效**：采集数据时自动调用 `invalidateQueryCache(affectedChannels)`，递增对应通道及其所有前缀的版本号。 + +### SSE 实时推送 + +基于 Server-Sent Events 的实时统计推送： + +- 按 `interval`（默认 5 秒）定时调用 `fetchData` 获取最新数据推送 +- 心跳保活（默认 30 秒），防止连接被代理/负载均衡器断开 +- 最大连接时长（默认 30 分钟），超时自动断开并推送 `timeout` 事件 +- 防止推送重叠：当前推送未完成时跳过下一次 +- 缓存复用：相同 `name`+`params`+`interval` 在同一时间窗口内命中缓存 + +### 重置与修复 + +提供 `resetPeriodStats` 方法用于修复错误的聚合数据： + +- 删除指定周期和时间范围的 period-stat 记录 +- 重置水位线到指定起始点 +- 支持 `cascade=true` 级联重置下游周期（如重置 h 时同时重置依赖 h 的 d、w、m、q、y） + +### 技术栈 + +- **Fastify** + fastify-plugin + fastify-namespace +- **Sequelize**（数据持久化，支持多种数据库） +- **fastify-cron**（定时聚合与清理任务） +- **dayjs**（时间处理，支持 UTC 和时区扩展） +- **lodash**（工具函数） + +### 主要特性 + +- **数据采集**：支持单条和批量数据上报，自动展开多属性对象和多级通道 +- **缓冲写入**：支持缓存缓冲模式，定时批量写入数据库，减少写入压力 +- **多周期聚合**：h/d/w/m/q/y 六种统计周期，自动通过 Cron 定时聚合 +- **聚合方法**：sum、avg、count、min、max 五种聚合计算 +- **灵活查询**：按通道、时间范围、属性名、聚合方法查询统计结果，自动合并当前小时未聚合的原始数据 +- **SSE 实时推送**：基于 Server-Sent Events 的实时数据推送 +- **时区支持**：查询时支持传入客户端时区（IANA 格式） +- **事务安全**：所有数据库写操作使用事务保证原子性，聚合操作支持幂等（upsert） +- **查询缓存**：支持内存 LRU 或外部缓存，带版本校验和 TTL 策略 ### 示例 @@ -464,18 +247,18 @@ fastify.get('/my-sse', async (request, reply) => { | title | string | 否 | channel | 标题 | | description | string | 否 | - | 描述 | | attributeName | string | 否 | `default` | 属性名 | -| unit | string / object | 否 | - | 数据单位。字符串时所有属性共用同一单位；对象时按 attributeName 映射单位，如 `{temp: '℃', humidity: '%'}` | +| unit | string / object | 否 | - | 数据单位。字符串时所有属性共用同一单位；对象时按 attributeName 映射单位 | | time | string | 否 | 当前时间 | 采集时间(ISO格式) | **请求体（批量）**：以上对象的数组。 -**通道展开规则**：`device:sensor1` 会展开为 `device:sensor1` 和 `device` 两个通道记录。 +**通道展开规则**：`device:sensor1` 展开为 `device` 和 `device:sensor1` 两个通道记录。 -**数据展开规则**：`data` 为对象时，按属性名拆分为多条记录。例如 `data: {temp: 25, humidity: 60}` 拆分为两条：`{attributeName: 'temp', data: 25}` 和 `{attributeName: 'humidity', data: 60}`。 +**数据展开规则**：`data` 为对象时按属性名拆分。如 `data: {temp: 25, humidity: 60}` 拆分为 `{attributeName: 'temp', data: 25}` 和 `{attributeName: 'humidity', data: 60}`。 -**单位展开规则**：`unit` 为字符串时，所有属性共用同一单位；`unit` 为对象时，以 attributeName 为 key 获取对应单位，未匹配到的属性不设置单位。例如 `data: {temp: 25, humidity: 60}`，`unit: {temp: '℃', humidity: '%'}` 展开后 temp 的单位为 `℃`，humidity 的单位为 `%`；若 `unit: '℃'`，则两者单位均为 `℃`。 +**单位展开规则**：`unit` 为字符串时所有属性共用同一单位；为对象时按 attributeName 映射，未匹配到的不设置单位。 -**缓冲模式**：当配置 `cache` 时，采集数据先写入内存缓冲区，定时或缓冲区满时批量写入数据库；否则直接写入。 +**缓冲模式**：配置 `cache` 时，采集数据先写入内存缓冲区，定时或缓冲区满时批量写入数据库；否则直接写入。 ### 统计查询 @@ -495,7 +278,7 @@ fastify.get('/my-sse', async (request, reply) => { | timezone | string | 否 | 服务器时区 | 客户端时区(IANA格式，如 Asia/Shanghai) | | includeChildren | boolean | 否 | false | 是否包含子通道数据 | -**通道匹配规则**：默认只精确匹配传入的 channels。当 `includeChildren=true` 时，传入 `sensor` 会匹配 `sensor` 和 `sensor:*` 的所有通道，返回树形结构。 +**通道匹配规则**：默认只精确匹配传入的 channels。`includeChildren=true` 时，传入 `sensor` 匹配 `sensor` 和 `sensor:*` 的所有通道，返回树形结构。 **返回格式**： @@ -518,7 +301,7 @@ fastify.get('/my-sse', async (request, reply) => { } ``` -`includeChildren=true` 时返回树形结构，子通道数据嵌套在 `children` 数组中： +`includeChildren=true` 时返回树形结构： ```json { @@ -528,25 +311,11 @@ fastify.get('/my-sse', async (request, reply) => { "list": [ { "channel": "sensor", - "items": [ - { - "period": "h", - "time": "2026-05-22T08:00:00.000Z", - "data": { "default": 100 }, - "unit": { "default": "℃" } - } - ], + "items": [{ "period": "h", "time": "...", "data": {"default": 100}, "unit": {"default": "℃"} }], "children": [ { "channel": "sensor:temp", - "items": [ - { - "period": "h", - "time": "2026-05-22T08:00:00.000Z", - "data": { "default": 25 }, - "unit": { "default": "℃" } - } - ] + "items": [{ "period": "h", "time": "...", "data": {"default": 25}, "unit": {"default": "℃"} }] } ] } @@ -562,118 +331,105 @@ fastify.get('/my-sse', async (request, reply) => { | items | array | 该通道的统计结果数组（按时间排序），每项包含 `period`、`time`、`data`、`unit` | | children | array | 子通道数组（递归结构，仅存在子通道时返回） | -`data` 字段格式始终为对象（按属性名映射），根据聚合方法数量决定层级： +`data` 字段格式： | 条件 | data 格式 | 示例 | |------|-----------|------| | 单聚合 | object | `{"default": 100}` 或 `{"temperature": 25, "humidity": 60}` | -| 多聚合 | 嵌套object | `{"sum": {"default": 100}, "avg": {"default": 50}}` 或 `{"sum": {"temperature": 25}, "avg": {"temperature": 12.5}}` | - -`unit` 字段为对象，按属性名映射单位：`{"default": "℃"}` 或 `{"temperature": "℃", "humidity": "%"}` - -> **注意**：查询结果中 `aggregate` 不作为独立字段返回。聚合方法（如 sum、avg）被用作 `data` 对象的键名。例如多聚合时 `data` 为 `{"sum": {"default": 100}, "avg": {"default": 50}}`，而非 `[{aggregate: "sum", data: 100}, {aggregate: "avg", data: 50}]`。 - -### 统计周期 - -| 周期 | key | Cron 表达式 | 数据来源 | -|------|-----|-------------|----------| -| 时 | h | `1 * * * *` | 原始数据(data-record) | -| 日 | d | `1 0 * * *` | 小时统计(period-stat) | -| 周 | w | `1 0 * * 1` | 日统计(period-stat) | -| 月 | m | `1 0 1 * *` | 日统计(period-stat) | -| 季 | q | `1 0 1 1,4,7,10 *` | 月统计(period-stat) | -| 年 | y | `1 0 1 1 *` | 季统计(period-stat) | +| 多聚合 | 嵌套object | `{"sum": {"default": 100}, "avg": {"default": 50}}` | -### 聚合方法 - -| 方法 | key | 说明 | -|------|-----|------| -| 合计 | sum | 数值求和 | -| 平均 | avg | 数值平均(由sum/count计算) | -| 计数 | count | 记录计数 | -| 最小 | min | 最小值 | -| 最大 | max | 最大值 | +### SSE 实时推送 -### 补偿聚合机制 +#### GET `{prefix}/sse` -插件启动时自动执行补偿聚合（可通过 `compensationEnabled: false` 关闭）。补偿逻辑基于 `aggregation-watermark` 表记录的各周期水位线（下一个待聚合时间），从水位线开始逐步向前聚合，直到追上当前时间。 +基于 Server-Sent Events 的实时统计推送，自动查询最近一小时的统计数据并按指定间隔推送。 -- 每次补偿最多处理 `compensationBatchSize` 个时间窗口 -- 上游周期未完成时，自动先补偿上游（如聚合 `d` 前先确保 `h` 已完成） -- 每个周期有独立的补偿锁，防止并发补偿 -- 补偿聚合通过 Cron 定时触发，同时启动时也会执行一次 +**查询参数**： -### 查询缓存 +| 属性名 | 类型 | 必填 | 默认值 | 说明 | +|--------|------|------|--------|------| +| channels | string | 是 | - | 数据通道(逗号分隔多个通道) | +| attributeNames | string | 否 | 全部 | 属性名列表(逗号分隔) | +| aggregates | string | 否 | 全部 | 聚合方法列表(逗号分隔): sum,avg,count,min,max | +| timezone | string | 否 | 服务器时区 | 客户端时区(IANA格式) | +| includeChildren | boolean | 否 | false | 是否包含子通道数据 | +| interval | number | 否 | `5` | 推送间隔秒数 | -查询结果自动缓存，减少重复查询的数据库压力： +**响应格式**：`Content-Type: text/event-stream` -- **无外部缓存**：使用内存 LRU 缓存，最大条数由 `queryCacheMaxEntries` 控制 -- **有外部缓存**（配置 `cache`）：查询结果存入外部缓存，支持 TTL 和版本校验 -- **版本校验**：缓存条目记录写入时的通道版本号，读取时校验版本是否变化，版本不匹配则缓存失效 -- **TTL 策略**：实时查询（endTime 在当前小时内）使用 `queryCacheTTL`（默认30秒），历史查询使用 `queryCacheHistoryTTL`（默认3600秒） -- **补偿期间**：正在执行补偿聚合时，查询不走缓存，确保数据实时性 +| 事件类型 | 说明 | +|----------|------| +| `data`（无 event 字段） | 正常数据推送，内容为查询结果的 JSON | +| `timeout` | 连接超过 maxDuration 后自动断开通知 | +| `error` | fetchData 出错时的错误事件 | +| 注释行（`: heartbeat`） | 心跳保活 | ### 程序化 API 通过 `fastify.statistics.services` 访问： +#### 通用方法 + | 方法 | 说明 | |------|------| | `services.collect(data)` | 采集数据，同 `/collect` 接口逻辑 | | `services.query(params)` | 查询统计，同 `/query` 接口逻辑 | -| `services.periodStat.aggregate(period, opts)` | 手动触发指定周期的聚合 | -| `services.periodStat.query(params)` | 同 `services.query` | + +#### dataRecord 服务 + +| 方法 | 说明 | +|------|------| | `services.dataRecord.collect(data)` | 同 `services.collect` | | `services.dataRecord.flush()` | 手动刷新缓冲区 | -| `services.sseStream.send(reply, config)` | 发送 SSE 实时数据流（详见下方） | - -### SSE 实时推送 +| `services.dataRecord.cleanup()` | 清理过期的原始数据 | -#### GET `{prefix}/sse` +#### periodStat 服务 -基于 Server-Sent Events 的实时统计推送，自动查询最近一小时的统计数据并按指定间隔推送。 +| 方法 | 说明 | +|------|------| +| `services.periodStat.init()` | 初始化水位线并执行启动补偿（插件 onReady 自动调用） | +| `services.periodStat.aggregate(period, opts)` | 手动触发指定周期的聚合。`opts.startTime`/`opts.endTime` 可选，默认聚合上一个时间窗口 | +| `services.periodStat.query(params)` | 同 `services.query` | +| `services.periodStat.isCompensating()` | 当前是否正在执行补偿聚合 | +| `services.periodStat.invalidateQueryCache(channels?)` | 使查询缓存失效。传入 channels 时只失效相关通道，不传则失效全部 | +| `services.periodStat.cleanupOldPeriodStats()` | 清理过期的周期统计数据 | +| `services.periodStat.resetPeriodStats(period, opts)` | 重置指定周期的数据和水位线，详见下方 | -**查询参数**： +#### resetPeriodStats 参数 | 属性名 | 类型 | 必填 | 默认值 | 说明 | |--------|------|------|--------|------| -| channels | string | 是 | - | 数据通道(逗号分隔多个通道) | -| attributeNames | string | 否 | 全部 | 属性名列表(逗号分隔) | -| aggregates | string | 否 | 全部 | 聚合方法列表(逗号分隔): sum,avg,count,min,max | -| timezone | string | 否 | 服务器时区 | 客户端时区(IANA格式，如 Asia/Shanghai) | -| interval | number | 否 | `5` | 推送间隔秒数 | +| period | string | 是 | - | 周期类型: h/d/w/m/q/y | +| opts.startTime | Date | 否 | 当前截断时间 | 重置起始时间（水位线将设为此值） | +| opts.endTime | Date | 否 | 全部 | 仅删除此时间范围内的 period-stat 数据 | +| opts.cascade | boolean | 否 | false | 是否级联重置下游周期 | -**响应格式**：`Content-Type: text/event-stream` +**返回值**：`{ period, deletedCount, nextTime, cascade_h?, cascade_d?, ... }` -``` -data: {"channelMetas":{},"list":[{"channel":"sensor","items":[{"period":"h","time":"...","data":{"default":100}}],"children":[...]}]} +#### channelMeta 服务 -event: timeout -data: {"message":"连接已超过30分钟，自动断开"} - -: heartbeat +| 方法 | 说明 | +|------|------| +| `services.channelMeta.detail({ channel })` | 查询通道元数据 | +| `services.channelMeta.list({ filter? })` | 列出元数据，`filter.channel` 可按通道筛选 | +| `services.channelMeta.save({ channel, title?, description? })` | 修改元数据 | -event: error -data: {"message":"错误信息"} -``` +#### sseStream 服务 -| 事件类型 | 说明 | -|----------|------| -| `data`（无 event 字段） | 正常数据推送，内容为查询结果的 JSON | -| `timeout` | 连接超过 maxDuration 后自动断开通知 | -| `error` | fetchData 出错时的错误事件 | -| 注释行（`: heartbeat`） | 心跳保活 | +| 方法 | 说明 | +|------|------| +| `services.sseStream.send(reply, config)` | 发送 SSE 实时数据流 | -**程序化调用**：`services.sseStream.send(reply, config)` +**send 配置**： | config 属性 | 类型 | 必填 | 默认值 | 说明 | |-------------|------|------|--------|------| | name | string | 是 | - | 缓存键名称标识 | | params | object | 是 | - | 传递给 fetchData 的参数 | -| fetchData | function | 是 | - | 异步函数 `(params) => data`，获取推送数据 | +| fetchData | function | 是 | - | 异步函数 `(params) => data` | | interval | number | 否 | `5` | 推送间隔秒数 | | heartbeatInterval | number | 否 | `30000` | 心跳间隔(ms) | -| maxDuration | number | 否 | `1800000` | 最大连接时长(ms)，超时自动断开 | +| maxDuration | number | 否 | `1800000` | 最大连接时长(ms) | **返回值**：SSE 上下文对象 @@ -683,8 +439,6 @@ data: {"message":"错误信息"} | `close()` | 手动关闭 SSE 连接 | | `onClose(callback)` | 注册连接关闭回调，若已断开则立即执行 | -**缓存机制**：当插件配置了 `cache` 时，SSE 推送的数据会按时间窗口缓存，相同 `name`+`params`+`interval` 的请求在同一时间窗口内会命中缓存，避免重复调用 `fetchData`。 - ### 数据模型 #### data-record（数据采集记录） @@ -697,13 +451,13 @@ data: {"message":"错误信息"} | time | DATE | 采集时间(必填) | | unit | STRING | 数据单位 | -> `title`、`description` 已移至 `channel-meta` 表，按 root channel 关联。 +索引：`channel`、`time`、`[channel, time]`、`[channel, attributeName, time]`、`attributeName` #### period-stat（周期统计） | 属性名 | 类型 | 说明 | |--------|------|------| -| period | STRING | 统计周期(必填) | +| period | STRING | 统计周期(必填): h/d/w/m/q/y | | time | DATE | 统计时间(必填) | | channel | STRING | 数据通道(必填) | | attributeName | STRING | 属性名(默认 default) | @@ -711,9 +465,9 @@ data: {"message":"错误信息"} | data | DECIMAL(16,2) | 统计数据值(必填) | | unit | STRING | 数据单位 | -> `title`、`description` 已移至 `channel-meta` 表，按 root channel 关联。 +唯一约束：`(period, channel, attributeName, aggregate, time)` -**唯一约束**：`(period, channel, attributeName, aggregate, time)` +索引：`[channel, attributeName, time]`、`[period, time]`、`attributeName` #### channel-meta（通道元数据） @@ -723,9 +477,9 @@ data: {"message":"错误信息"} | title | STRING | 标题(必填) | | description | TEXT | 描述 | -**唯一约束**：`channel` +唯一约束：`channel` -**说明**：`channel-meta` 按 root channel（一级通道）唯一存储，一条元数据被所有以该 root channel 为前缀的子通道共享。首次采集某通道数据时，自动以其 root channel 创建元数据记录。`title` 和 `description` 从采集参数中提取，后续采集忽略（不更新）。`unit` 字段保留在 `data-record` 和 `period-stat` 表中。 +说明：按 root channel（一级通道）唯一存储，所有子通道共享同一份元数据。 #### aggregation-watermark（聚合水位线） @@ -734,6 +488,59 @@ data: {"message":"错误信息"} | period | STRING | 统计周期(唯一键): h/d/w/m/q/y | | nextTime | DATE | 下一个待聚合时间 | -**唯一约束**：`period` +唯一约束：`period` + +说明：水位线记录各周期下一次应聚合的时间起点，用于补偿聚合逻辑。首次聚合时，根据原始数据或上游周期统计的最早时间自动初始化。 + +### 统计周期 + +| 周期 | key | Cron 表达式 | 数据来源 | +|------|-----|-------------|----------| +| 时 | h | `1 * * * *` | 原始数据(data-record) | +| 日 | d | `1 0 * * *` | 小时统计(period-stat) | +| 周 | w | `1 0 * * 1` | 日统计(period-stat) | +| 月 | m | `1 0 1 * *` | 日统计(period-stat) | +| 季 | q | `1 0 1 1,4,7,10 *` | 月统计(period-stat) | +| 年 | y | `1 0 1 1 *` | 季统计(period-stat) | + +### 聚合方法 -**说明**：水位线记录各周期下一次应聚合的时间起点，用于补偿聚合逻辑。首次聚合时，根据原始数据或上游周期统计的最早时间自动初始化。 +| 方法 | key | 说明 | +|------|-----|------| +| 合计 | sum | 数值求和 | +| 平均 | avg | 数值平均(由sum/count计算) | +| 计数 | count | 记录计数 | +| 最小 | min | 最小值 | +| 最大 | max | 最大值 | + +### 补偿聚合机制 + +插件启动时自动执行补偿聚合（可通过 `compensationEnabled: false` 关闭）： + +- 水位线记录各周期下一次应聚合的时间起点 +- 补偿时从水位线开始逐步向前聚合，直到追上当前时间 +- 每次最多处理 `compensationBatchSize` 个时间窗口 +- 上游周期未完成时自动先补偿上游 +- 每个周期有独立的补偿锁，防止并发补偿 +- 连续失败 `maxCompensationFailCount` 次（默认 3）后停止，下次 Cron 继续 +- 补偿聚合通过 Cron 定时触发，启动时也会执行一次 + +### 查询缓存 + +| 特性 | 说明 | +|------|------| +| 无外部缓存 | 使用内存 LRU 缓存，最大 `queryCacheMaxEntries` 条 | +| 有外部缓存 | 查询结果存入外部缓存，支持 TTL 和版本校验 | +| 版本校验 | 缓存条目记录写入时的通道版本号，读取时校验版本是否变化 | +| TTL 策略 | 实时查询用 `queryCacheTTL`（30s），历史查询用 `queryCacheHistoryTTL`（3600s） | +| 补偿期间 | 正在执行补偿聚合时查询不走缓存 | + +### 数据保留策略 + +| 数据类型 | 保留策略 | 安全检查 | +|----------|----------|----------| +| data-record | `dataRetentionDays` 天（默认 7 天） | 不超过 h 周期水位线 | +| period-stat(h) | 当月 | 不超过 d 水位线 | +| period-stat(d) | 当年 | 不超过 w、m 水位线 | +| period-stat(w) | 当年 | 无下游依赖 | +| period-stat(m/q/y) | 永久保留 | - | diff --git a/benchmark/aggregate-bench.js b/benchmark/aggregate-bench.js index a6cad86..d5b6493 100644 --- a/benchmark/aggregate-bench.js +++ b/benchmark/aggregate-bench.js @@ -9,7 +9,8 @@ */ const dayjs = require('dayjs'); const { - createRealFastify, seedDataRecords, seedPeriodStats, ensureChannelMetas, + createRealFastify, createCronFastify, setWatermark, + seedDataRecords, seedPeriodStats, ensureChannelMetas, measure, formatStatsHeader, formatStats, memorySnapshot, formatMemoryDelta, createResultCollector, getResultFilePath } = require('./helpers'); @@ -128,7 +129,7 @@ async function run() { console.log('\n--- 4a. 小窗口补偿 (h 周期, 从 dataRecord) ---'); for (const windowCount of windowCounts) { - const { fastify, cleanup } = await createRealFastify(); + const { fastify, cleanup, createdJobs } = await createCronFastify({ pluginOptions: { compensationBatchSize: 100 } }); const now = dayjs().startOf('hour').toDate(); const pastStart = dayjs(now).subtract(windowCount, 'hour').toDate(); const channels = ['sensor']; @@ -143,29 +144,13 @@ async function run() { }); // 设置 watermark 到 pastStart（需要补偿 windowCount 个窗口） - await fastify.statistics.models.aggregationWatermark.upsert({ - period: 'h', nextTime: pastStart - }); - - // 通过 cron onTick 触发 compensate - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - // 重新注册 periodStat 服务（带 cron） - const fp = require('fastify-plugin'); - await fp(require('../libs/services/period-stat'))(fastify, { - name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 - }); + await setWatermark(fastify, 'h', pastStart); const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); const stats = await measure(async () => { // 重置 watermark - await fastify.statistics.models.aggregationWatermark.upsert({ - period: 'h', nextTime: pastStart - }); + await setWatermark(fastify, 'h', pastStart); await hJob.onTick(); }, { iterations: 10, warmup: 2 }); @@ -181,7 +166,7 @@ async function run() { console.log('\n--- 4b. 大批量补偿 (h 周期, compensationBatchSize 变化) ---'); for (const batchSize of batchSizes) { - const { fastify, cleanup } = await createRealFastify(); + const { fastify, cleanup, createdJobs } = await createCronFastify({ pluginOptions: { compensationBatchSize: batchSize } }); const now = dayjs().startOf('hour').toDate(); const dayAgo = dayjs(now).subtract(24, 'hour').toDate(); const channels = ['sensor']; @@ -193,19 +178,7 @@ async function run() { startTime: dayAgo, endTime: now }); - await fastify.statistics.models.aggregationWatermark.upsert({ - period: 'h', nextTime: dayAgo - }); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - const fp = require('fastify-plugin'); - await fp(require('../libs/services/period-stat'))(fastify, { - name: 'statistics', compensationEnabled: false, compensationBatchSize: batchSize - }); + await setWatermark(fastify, 'h', dayAgo); const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); @@ -224,7 +197,7 @@ async function run() { { console.log('\n--- 4c. 级联补偿 (d → h) ---'); - const { fastify, cleanup } = await createRealFastify(); + const { fastify, cleanup, createdJobs } = await createCronFastify({ pluginOptions: { compensationBatchSize: 100 } }); const now = dayjs().startOf('day').toDate(); const dayAgo = dayjs(now).subtract(1, 'day').toDate(); const channels = ['sensor']; @@ -236,24 +209,9 @@ async function run() { startTime: dayAgo, endTime: now }); - // d 的 watermark 在过去 - await fastify.statistics.models.aggregationWatermark.upsert({ - period: 'd', nextTime: dayAgo - }); - // h 的 watermark 也在过去，需要先补偿 h - await fastify.statistics.models.aggregationWatermark.upsert({ - period: 'h', nextTime: dayAgo - }); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - const fp = require('fastify-plugin'); - await fp(require('../libs/services/period-stat'))(fastify, { - name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 - }); + // d 和 h 的 watermark 都在过去 + await setWatermark(fastify, 'd', dayAgo); + await setWatermark(fastify, 'h', dayAgo); const dJob = createdJobs.find(j => j.name === 'statistics-period-stat-d'); diff --git a/benchmark/helpers.js b/benchmark/helpers.js index ecc693f..a32d51e 100644 --- a/benchmark/helpers.js +++ b/benchmark/helpers.js @@ -63,6 +63,10 @@ async function createRealFastify({ dbPath, pluginOptions = {} } = {}) { ] }); + // 初始化 periodStat 服务（必须在 ready 之后调用 init） + await fastify.ready(); + await fastify.statistics.services.periodStat.init(); + const cleanup = async () => { await fastify.close(); try { fs.unlinkSync(dbPath); } catch (e) { /* ignore */ } @@ -421,8 +425,105 @@ function printSummaryReport(scenarioKeys) { } } +/** + * 创建带 cron 支持的 Fastify 实例（用于 compensate 基准测试） + * cron 任务被收集到 createdJobs 数组，便于手动触发 onTick + * @param {object} [options] + * @param {string} [options.dbPath] - SQLite 文件路径 + * @param {object} [options.pluginOptions] - 传给 services 的选项 + * @returns {Promise<{fastify, dbPath, cleanup, createdJobs}>} + */ +async function createCronFastify({ dbPath, pluginOptions = {} } = {}) { + if (!dbPath) { + dbPath = path.join(os.tmpdir(), `benchmark-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + } + + const fastify = require('fastify')({ logger: false }); + + // 注册 sequelize + await fastify.register(require('@kne/fastify-sequelize'), { + db: { + dialect: 'sqlite', + storage: dbPath, + logging: false, + dialectOptions: { + mode: require('sqlite3').OPEN_READWRITE | require('sqlite3').OPEN_CREATE + } + }, + syncOptions: { alter: true } + }); + + // 加载模型 + const models = await fastify.sequelize.addModels( + path.resolve(__dirname, '../libs/models'), + { prefix: 't_', modelPrefix: 'statistics' } + ); + + // 同步数据库 + await fastify.sequelize.sync(); + + // 启用 WAL 模式 + const seqInstance = fastify.sequelize.instance; + await seqInstance.query('PRAGMA journal_mode=WAL'); + await seqInstance.query('PRAGMA busy_timeout=5000'); + + // 收集 cron jobs + const createdJobs = []; + + // 注册 namespace + services（带 cron 收集器） + await fastify.register(require('@kne/fastify-namespace'), { + options: Object.assign({ + prefix: '/api/statistics', + dbTableNamePrefix: 't_', + name: 'statistics', + compensationEnabled: false, + queryCacheEnabled: true, + dataRetentionDays: 365 + }, pluginOptions), + name: 'statistics', + modules: [ + ['models', models], + ['services', path.resolve(__dirname, '../libs/services')] + ] + }); + + // 注册 cron 收集器（必须在 namespace 之后，因为 periodStat init() 会检查 fastify.cron） + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + // 初始化 periodStat 服务 + await fastify.ready(); + await fastify.statistics.services.periodStat.init(); + + const cleanup = async () => { + await fastify.close(); + try { fs.unlinkSync(dbPath); } catch (e) { /* ignore */ } + }; + + return { fastify, dbPath, cleanup, createdJobs }; +} + +/** + * 设置指定周期的 watermark（兼容 findOne+update/create 模式） + * @param {object} fastify + * @param {string} period + * @param {Date} nextTime + */ +async function setWatermark(fastify, period, nextTime) { + const models = fastify.statistics.models; + const existing = await models.aggregationWatermark.findOne({ where: { period } }); + if (existing) { + await existing.update({ nextTime }); + } else { + await models.aggregationWatermark.create({ period, nextTime }); + } +} + module.exports = { createRealFastify, + createCronFastify, + setWatermark, seedDataRecords, seedPeriodStats, ensureChannelMetas, diff --git a/doc/api.md b/doc/api.md index e0cb050..4c4144d 100644 --- a/doc/api.md +++ b/doc/api.md @@ -33,18 +33,18 @@ | title | string | 否 | channel | 标题 | | description | string | 否 | - | 描述 | | attributeName | string | 否 | `default` | 属性名 | -| unit | string / object | 否 | - | 数据单位。字符串时所有属性共用同一单位；对象时按 attributeName 映射单位，如 `{temp: '℃', humidity: '%'}` | +| unit | string / object | 否 | - | 数据单位。字符串时所有属性共用同一单位；对象时按 attributeName 映射单位 | | time | string | 否 | 当前时间 | 采集时间(ISO格式) | **请求体（批量）**：以上对象的数组。 -**通道展开规则**：`device:sensor1` 会展开为 `device:sensor1` 和 `device` 两个通道记录。 +**通道展开规则**：`device:sensor1` 展开为 `device` 和 `device:sensor1` 两个通道记录。 -**数据展开规则**：`data` 为对象时，按属性名拆分为多条记录。例如 `data: {temp: 25, humidity: 60}` 拆分为两条：`{attributeName: 'temp', data: 25}` 和 `{attributeName: 'humidity', data: 60}`。 +**数据展开规则**：`data` 为对象时按属性名拆分。如 `data: {temp: 25, humidity: 60}` 拆分为 `{attributeName: 'temp', data: 25}` 和 `{attributeName: 'humidity', data: 60}`。 -**单位展开规则**：`unit` 为字符串时，所有属性共用同一单位；`unit` 为对象时，以 attributeName 为 key 获取对应单位，未匹配到的属性不设置单位。例如 `data: {temp: 25, humidity: 60}`，`unit: {temp: '℃', humidity: '%'}` 展开后 temp 的单位为 `℃`，humidity 的单位为 `%`；若 `unit: '℃'`，则两者单位均为 `℃`。 +**单位展开规则**：`unit` 为字符串时所有属性共用同一单位；为对象时按 attributeName 映射，未匹配到的不设置单位。 -**缓冲模式**：当配置 `cache` 时，采集数据先写入内存缓冲区，定时或缓冲区满时批量写入数据库；否则直接写入。 +**缓冲模式**：配置 `cache` 时，采集数据先写入内存缓冲区，定时或缓冲区满时批量写入数据库；否则直接写入。 ### 统计查询 @@ -64,7 +64,7 @@ | timezone | string | 否 | 服务器时区 | 客户端时区(IANA格式，如 Asia/Shanghai) | | includeChildren | boolean | 否 | false | 是否包含子通道数据 | -**通道匹配规则**：默认只精确匹配传入的 channels。当 `includeChildren=true` 时，传入 `sensor` 会匹配 `sensor` 和 `sensor:*` 的所有通道，返回树形结构。 +**通道匹配规则**：默认只精确匹配传入的 channels。`includeChildren=true` 时，传入 `sensor` 匹配 `sensor` 和 `sensor:*` 的所有通道，返回树形结构。 **返回格式**： @@ -87,7 +87,7 @@ } ``` -`includeChildren=true` 时返回树形结构，子通道数据嵌套在 `children` 数组中： +`includeChildren=true` 时返回树形结构： ```json { @@ -97,25 +97,11 @@ "list": [ { "channel": "sensor", - "items": [ - { - "period": "h", - "time": "2026-05-22T08:00:00.000Z", - "data": { "default": 100 }, - "unit": { "default": "℃" } - } - ], + "items": [{ "period": "h", "time": "...", "data": {"default": 100}, "unit": {"default": "℃"} }], "children": [ { "channel": "sensor:temp", - "items": [ - { - "period": "h", - "time": "2026-05-22T08:00:00.000Z", - "data": { "default": 25 }, - "unit": { "default": "℃" } - } - ] + "items": [{ "period": "h", "time": "...", "data": {"default": 25}, "unit": {"default": "℃"} }] } ] } @@ -131,118 +117,105 @@ | items | array | 该通道的统计结果数组（按时间排序），每项包含 `period`、`time`、`data`、`unit` | | children | array | 子通道数组（递归结构，仅存在子通道时返回） | -`data` 字段格式始终为对象（按属性名映射），根据聚合方法数量决定层级： +`data` 字段格式： | 条件 | data 格式 | 示例 | |------|-----------|------| | 单聚合 | object | `{"default": 100}` 或 `{"temperature": 25, "humidity": 60}` | -| 多聚合 | 嵌套object | `{"sum": {"default": 100}, "avg": {"default": 50}}` 或 `{"sum": {"temperature": 25}, "avg": {"temperature": 12.5}}` | - -`unit` 字段为对象，按属性名映射单位：`{"default": "℃"}` 或 `{"temperature": "℃", "humidity": "%"}` - -> **注意**：查询结果中 `aggregate` 不作为独立字段返回。聚合方法（如 sum、avg）被用作 `data` 对象的键名。例如多聚合时 `data` 为 `{"sum": {"default": 100}, "avg": {"default": 50}}`，而非 `[{aggregate: "sum", data: 100}, {aggregate: "avg", data: 50}]`。 - -### 统计周期 - -| 周期 | key | Cron 表达式 | 数据来源 | -|------|-----|-------------|----------| -| 时 | h | `1 * * * *` | 原始数据(data-record) | -| 日 | d | `1 0 * * *` | 小时统计(period-stat) | -| 周 | w | `1 0 * * 1` | 日统计(period-stat) | -| 月 | m | `1 0 1 * *` | 日统计(period-stat) | -| 季 | q | `1 0 1 1,4,7,10 *` | 月统计(period-stat) | -| 年 | y | `1 0 1 1 *` | 季统计(period-stat) | - -### 聚合方法 +| 多聚合 | 嵌套object | `{"sum": {"default": 100}, "avg": {"default": 50}}` | -| 方法 | key | 说明 | -|------|-----|------| -| 合计 | sum | 数值求和 | -| 平均 | avg | 数值平均(由sum/count计算) | -| 计数 | count | 记录计数 | -| 最小 | min | 最小值 | -| 最大 | max | 最大值 | +### SSE 实时推送 -### 补偿聚合机制 +#### GET `{prefix}/sse` -插件启动时自动执行补偿聚合（可通过 `compensationEnabled: false` 关闭）。补偿逻辑基于 `aggregation-watermark` 表记录的各周期水位线（下一个待聚合时间），从水位线开始逐步向前聚合，直到追上当前时间。 +基于 Server-Sent Events 的实时统计推送，自动查询最近一小时的统计数据并按指定间隔推送。 -- 每次补偿最多处理 `compensationBatchSize` 个时间窗口 -- 上游周期未完成时，自动先补偿上游（如聚合 `d` 前先确保 `h` 已完成） -- 每个周期有独立的补偿锁，防止并发补偿 -- 补偿聚合通过 Cron 定时触发，同时启动时也会执行一次 +**查询参数**： -### 查询缓存 +| 属性名 | 类型 | 必填 | 默认值 | 说明 | +|--------|------|------|--------|------| +| channels | string | 是 | - | 数据通道(逗号分隔多个通道) | +| attributeNames | string | 否 | 全部 | 属性名列表(逗号分隔) | +| aggregates | string | 否 | 全部 | 聚合方法列表(逗号分隔): sum,avg,count,min,max | +| timezone | string | 否 | 服务器时区 | 客户端时区(IANA格式) | +| includeChildren | boolean | 否 | false | 是否包含子通道数据 | +| interval | number | 否 | `5` | 推送间隔秒数 | -查询结果自动缓存，减少重复查询的数据库压力： +**响应格式**：`Content-Type: text/event-stream` -- **无外部缓存**：使用内存 LRU 缓存，最大条数由 `queryCacheMaxEntries` 控制 -- **有外部缓存**（配置 `cache`）：查询结果存入外部缓存，支持 TTL 和版本校验 -- **版本校验**：缓存条目记录写入时的通道版本号，读取时校验版本是否变化，版本不匹配则缓存失效 -- **TTL 策略**：实时查询（endTime 在当前小时内）使用 `queryCacheTTL`（默认30秒），历史查询使用 `queryCacheHistoryTTL`（默认3600秒） -- **补偿期间**：正在执行补偿聚合时，查询不走缓存，确保数据实时性 +| 事件类型 | 说明 | +|----------|------| +| `data`（无 event 字段） | 正常数据推送，内容为查询结果的 JSON | +| `timeout` | 连接超过 maxDuration 后自动断开通知 | +| `error` | fetchData 出错时的错误事件 | +| 注释行（`: heartbeat`） | 心跳保活 | ### 程序化 API 通过 `fastify.statistics.services` 访问： +#### 通用方法 + | 方法 | 说明 | |------|------| | `services.collect(data)` | 采集数据，同 `/collect` 接口逻辑 | | `services.query(params)` | 查询统计，同 `/query` 接口逻辑 | -| `services.periodStat.aggregate(period, opts)` | 手动触发指定周期的聚合 | -| `services.periodStat.query(params)` | 同 `services.query` | + +#### dataRecord 服务 + +| 方法 | 说明 | +|------|------| | `services.dataRecord.collect(data)` | 同 `services.collect` | | `services.dataRecord.flush()` | 手动刷新缓冲区 | -| `services.sseStream.send(reply, config)` | 发送 SSE 实时数据流（详见下方） | - -### SSE 实时推送 +| `services.dataRecord.cleanup()` | 清理过期的原始数据 | -#### GET `{prefix}/sse` +#### periodStat 服务 -基于 Server-Sent Events 的实时统计推送，自动查询最近一小时的统计数据并按指定间隔推送。 +| 方法 | 说明 | +|------|------| +| `services.periodStat.init()` | 初始化水位线并执行启动补偿（插件 onReady 自动调用） | +| `services.periodStat.aggregate(period, opts)` | 手动触发指定周期的聚合。`opts.startTime`/`opts.endTime` 可选，默认聚合上一个时间窗口 | +| `services.periodStat.query(params)` | 同 `services.query` | +| `services.periodStat.isCompensating()` | 当前是否正在执行补偿聚合 | +| `services.periodStat.invalidateQueryCache(channels?)` | 使查询缓存失效。传入 channels 时只失效相关通道，不传则失效全部 | +| `services.periodStat.cleanupOldPeriodStats()` | 清理过期的周期统计数据 | +| `services.periodStat.resetPeriodStats(period, opts)` | 重置指定周期的数据和水位线，详见下方 | -**查询参数**： +#### resetPeriodStats 参数 | 属性名 | 类型 | 必填 | 默认值 | 说明 | |--------|------|------|--------|------| -| channels | string | 是 | - | 数据通道(逗号分隔多个通道) | -| attributeNames | string | 否 | 全部 | 属性名列表(逗号分隔) | -| aggregates | string | 否 | 全部 | 聚合方法列表(逗号分隔): sum,avg,count,min,max | -| timezone | string | 否 | 服务器时区 | 客户端时区(IANA格式，如 Asia/Shanghai) | -| interval | number | 否 | `5` | 推送间隔秒数 | +| period | string | 是 | - | 周期类型: h/d/w/m/q/y | +| opts.startTime | Date | 否 | 当前截断时间 | 重置起始时间（水位线将设为此值） | +| opts.endTime | Date | 否 | 全部 | 仅删除此时间范围内的 period-stat 数据 | +| opts.cascade | boolean | 否 | false | 是否级联重置下游周期 | -**响应格式**：`Content-Type: text/event-stream` +**返回值**：`{ period, deletedCount, nextTime, cascade_h?, cascade_d?, ... }` -``` -data: {"channelMetas":{},"list":[{"channel":"sensor","items":[{"period":"h","time":"...","data":{"default":100}}],"children":[...]}]} +#### channelMeta 服务 -event: timeout -data: {"message":"连接已超过30分钟，自动断开"} +| 方法 | 说明 | +|------|------| +| `services.channelMeta.detail({ channel })` | 查询通道元数据 | +| `services.channelMeta.list({ filter? })` | 列出元数据，`filter.channel` 可按通道筛选 | +| `services.channelMeta.save({ channel, title?, description? })` | 修改元数据 | -: heartbeat +#### sseStream 服务 -event: error -data: {"message":"错误信息"} -``` - -| 事件类型 | 说明 | -|----------|------| -| `data`（无 event 字段） | 正常数据推送，内容为查询结果的 JSON | -| `timeout` | 连接超过 maxDuration 后自动断开通知 | -| `error` | fetchData 出错时的错误事件 | -| 注释行（`: heartbeat`） | 心跳保活 | +| 方法 | 说明 | +|------|------| +| `services.sseStream.send(reply, config)` | 发送 SSE 实时数据流 | -**程序化调用**：`services.sseStream.send(reply, config)` +**send 配置**： | config 属性 | 类型 | 必填 | 默认值 | 说明 | |-------------|------|------|--------|------| | name | string | 是 | - | 缓存键名称标识 | | params | object | 是 | - | 传递给 fetchData 的参数 | -| fetchData | function | 是 | - | 异步函数 `(params) => data`，获取推送数据 | +| fetchData | function | 是 | - | 异步函数 `(params) => data` | | interval | number | 否 | `5` | 推送间隔秒数 | | heartbeatInterval | number | 否 | `30000` | 心跳间隔(ms) | -| maxDuration | number | 否 | `1800000` | 最大连接时长(ms)，超时自动断开 | +| maxDuration | number | 否 | `1800000` | 最大连接时长(ms) | **返回值**：SSE 上下文对象 @@ -252,8 +225,6 @@ data: {"message":"错误信息"} | `close()` | 手动关闭 SSE 连接 | | `onClose(callback)` | 注册连接关闭回调，若已断开则立即执行 | -**缓存机制**：当插件配置了 `cache` 时，SSE 推送的数据会按时间窗口缓存，相同 `name`+`params`+`interval` 的请求在同一时间窗口内会命中缓存，避免重复调用 `fetchData`。 - ### 数据模型 #### data-record（数据采集记录） @@ -266,13 +237,13 @@ data: {"message":"错误信息"} | time | DATE | 采集时间(必填) | | unit | STRING | 数据单位 | -> `title`、`description` 已移至 `channel-meta` 表，按 root channel 关联。 +索引：`channel`、`time`、`[channel, time]`、`[channel, attributeName, time]`、`attributeName` #### period-stat（周期统计） | 属性名 | 类型 | 说明 | |--------|------|------| -| period | STRING | 统计周期(必填) | +| period | STRING | 统计周期(必填): h/d/w/m/q/y | | time | DATE | 统计时间(必填) | | channel | STRING | 数据通道(必填) | | attributeName | STRING | 属性名(默认 default) | @@ -280,9 +251,9 @@ data: {"message":"错误信息"} | data | DECIMAL(16,2) | 统计数据值(必填) | | unit | STRING | 数据单位 | -> `title`、`description` 已移至 `channel-meta` 表，按 root channel 关联。 +唯一约束：`(period, channel, attributeName, aggregate, time)` -**唯一约束**：`(period, channel, attributeName, aggregate, time)` +索引：`[channel, attributeName, time]`、`[period, time]`、`attributeName` #### channel-meta（通道元数据） @@ -292,9 +263,9 @@ data: {"message":"错误信息"} | title | STRING | 标题(必填) | | description | TEXT | 描述 | -**唯一约束**：`channel` +唯一约束：`channel` -**说明**：`channel-meta` 按 root channel（一级通道）唯一存储，一条元数据被所有以该 root channel 为前缀的子通道共享。首次采集某通道数据时，自动以其 root channel 创建元数据记录。`title` 和 `description` 从采集参数中提取，后续采集忽略（不更新）。`unit` 字段保留在 `data-record` 和 `period-stat` 表中。 +说明：按 root channel（一级通道）唯一存储，所有子通道共享同一份元数据。 #### aggregation-watermark（聚合水位线） @@ -303,6 +274,59 @@ data: {"message":"错误信息"} | period | STRING | 统计周期(唯一键): h/d/w/m/q/y | | nextTime | DATE | 下一个待聚合时间 | -**唯一约束**：`period` +唯一约束：`period` + +说明：水位线记录各周期下一次应聚合的时间起点，用于补偿聚合逻辑。首次聚合时，根据原始数据或上游周期统计的最早时间自动初始化。 + +### 统计周期 + +| 周期 | key | Cron 表达式 | 数据来源 | +|------|-----|-------------|----------| +| 时 | h | `1 * * * *` | 原始数据(data-record) | +| 日 | d | `1 0 * * *` | 小时统计(period-stat) | +| 周 | w | `1 0 * * 1` | 日统计(period-stat) | +| 月 | m | `1 0 1 * *` | 日统计(period-stat) | +| 季 | q | `1 0 1 1,4,7,10 *` | 月统计(period-stat) | +| 年 | y | `1 0 1 1 *` | 季统计(period-stat) | + +### 聚合方法 -**说明**：水位线记录各周期下一次应聚合的时间起点，用于补偿聚合逻辑。首次聚合时，根据原始数据或上游周期统计的最早时间自动初始化。 +| 方法 | key | 说明 | +|------|-----|------| +| 合计 | sum | 数值求和 | +| 平均 | avg | 数值平均(由sum/count计算) | +| 计数 | count | 记录计数 | +| 最小 | min | 最小值 | +| 最大 | max | 最大值 | + +### 补偿聚合机制 + +插件启动时自动执行补偿聚合（可通过 `compensationEnabled: false` 关闭）： + +- 水位线记录各周期下一次应聚合的时间起点 +- 补偿时从水位线开始逐步向前聚合，直到追上当前时间 +- 每次最多处理 `compensationBatchSize` 个时间窗口 +- 上游周期未完成时自动先补偿上游 +- 每个周期有独立的补偿锁，防止并发补偿 +- 连续失败 `maxCompensationFailCount` 次（默认 3）后停止，下次 Cron 继续 +- 补偿聚合通过 Cron 定时触发，启动时也会执行一次 + +### 查询缓存 + +| 特性 | 说明 | +|------|------| +| 无外部缓存 | 使用内存 LRU 缓存，最大 `queryCacheMaxEntries` 条 | +| 有外部缓存 | 查询结果存入外部缓存，支持 TTL 和版本校验 | +| 版本校验 | 缓存条目记录写入时的通道版本号，读取时校验版本是否变化 | +| TTL 策略 | 实时查询用 `queryCacheTTL`（30s），历史查询用 `queryCacheHistoryTTL`（3600s） | +| 补偿期间 | 正在执行补偿聚合时查询不走缓存 | + +### 数据保留策略 + +| 数据类型 | 保留策略 | 安全检查 | +|----------|----------|----------| +| data-record | `dataRetentionDays` 天（默认 7 天） | 不超过 h 周期水位线 | +| period-stat(h) | 当月 | 不超过 d 水位线 | +| period-stat(d) | 当年 | 不超过 w、m 水位线 | +| period-stat(w) | 当年 | 无下游依赖 | +| period-stat(m/q/y) | 永久保留 | - | diff --git a/doc/summary.md b/doc/summary.md index e57024e..a1de324 100644 --- a/doc/summary.md +++ b/doc/summary.md @@ -1,411 +1,194 @@ ### 项目概述 -`@kne/fastify-statistics` 是一个基于 Fastify 的数据采集与周期统计插件，提供数据采集、多周期聚合统计和灵活查询功能。 +`@kne/fastify-statistics` 是一个基于 Fastify 的数据采集与多周期聚合统计插件。它提供从原始数据上报到多级周期聚合、灵活查询与实时推送的完整数据管道，适用于 IoT 传感器数据、业务指标监控、多通道多属性聚合等场景。 -### 主要特性 +### 核心架构与数据流 + +插件的核心理念是**逐级聚合**——原始数据采集后，按 h→d→w/m→q→y 的依赖链自动滚动聚合，每一级只从其直接上游读取数据，形成清晰的数据流管道： -- **数据采集**：支持单条和批量数据上报，自动展开多属性对象和多级通道 -- **缓冲写入**：支持缓存缓冲模式，定时批量写入数据库，减少写入压力 -- **多周期聚合**：支持时(h)、日(d)、周(w)、月(m)、季(q)、年(y) 六种统计周期，自动通过 Cron 定时聚合 -- **聚合方法**：支持 sum、avg、count、min、max 五种聚合计算 -- **灵活查询**：按通道、时间范围、属性名、聚合方法查询统计结果，自动合并当前小时未聚合的原始数据 -- **SSE 实时推送**：基于 Server-Sent Events 的实时数据推送，支持自定义间隔、心跳保活、超时断开和缓存复用 -- **时区支持**：查询时支持传入客户端时区（IANA 格式），解决客户端与服务器时区不一致问题 -- **事务安全**：所有数据库写操作使用事务保证原子性，聚合操作支持幂等（upsert） -- **dayjs 时间处理**：所有时间操作统一使用 dayjs 处理，支持时区扩展 - -### 使用场景 - -- IoT 设备传感器数据采集与统计分析 -- 业务指标实时监控与多周期报表 -- 多通道多属性的数据聚合与趋势查询 - -### 使用方法 - -#### 快速开始 - -```js -const fastify = require('fastify')(); - -// 注册依赖插件 -fastify.register(require('@kne/fastify-sequelize'), { /* sequelize 配置 */ }); -fastify.register(require('@kne/fastify-cron'), { /* cron 配置 */ }); - -// 注册统计插件 -fastify.register(require('@kne/fastify-statistics'), { - prefix: '/api/statistics', - cache: redisCacheInstance, // 传入缓存实例启用缓冲模式和查询缓存 - compensationEnabled: true, // 启动时自动补偿聚合 - compensationBatchSize: 24, // 每次补偿最多24个时间窗口 - dataRetentionDays: 7, // 原始数据保留7天 - queryCacheEnabled: true, // 启用查询缓存 - queryCacheTTL: 30, // 实时查询缓存30秒 - queryCacheHistoryTTL: 3600, // 历史查询缓存1小时 - queryCacheMaxEntries: 100, // 内存缓存最大100条（无外部缓存时生效） - getAuthenticate: type => { - // type 为 'read' 或 'write'，返回认证信息 - } -}); - -fastify.listen({ port: 3000 }); +``` +数据采集(collect) → data-record 表 + ↓ h 周期聚合 (Cron: 每小时第1分钟) + period-stat (period=h) + ↓ d 周期聚合 (Cron: 每日0:01) + period-stat (period=d) + ↙ ↘ + w 聚合(周一0:01) m 聚合(每月1日0:01) + ↓ ↓ + period-stat(w) period-stat(m) + ↓ q 聚合(每季首月1日0:01) + period-stat(q) + ↓ y 聚合(每年1月1日0:01) + period-stat(y) ``` -#### Channel 与 AttributeName 的设计理念 +**聚合依赖关系**（`PERIOD_DEPENDENCY`）： -**Channel（数据通道）** 是数据的第一级分类维度，采用冒号分隔的多级结构（`a:b:c`）。它的核心思想是：**从宏观到微观的层级划分**。 +| 周期 | 数据来源 | 上游周期 | +|------|----------|----------| +| h | data-record | - | +| d | period-stat | h | +| w | period-stat | d | +| m | period-stat | d | +| q | period-stat | m | +| y | period-stat | q | -- **一级 channel**（如 `sales`）是根通道，对应唯一的 `channel-meta` 记录（标题、描述） -- **多级 channel**（如 `sales:beijing`、`sales:beijing:team-a`）是更细粒度的子通道 -- 查询时传入一级 channel 即可匹配所有子通道的数据 -- 同一根通道下的所有子通道共享同一个 `channel-meta` +### 六种统计周期 -**AttributeName（属性名）** 是数据的第二级分类维度，用于在同一 channel 下区分不同的数据指标。 +| 周期 | key | Cron 表达式 | 时间截断规则 | 数据来源 | +|------|-----|-------------|-------------|----------| +| 时 | h | `1 * * * *` | `startOf('hour')` | data-record | +| 日 | d | `1 0 * * *` | `startOf('day')` | period-stat(h) | +| 周 | w | `1 0 * * 1` | `startOf('week')+1天` (周一) | period-stat(d) | +| 月 | m | `1 0 1 * *` | `startOf('month')` | period-stat(d) | +| 季 | q | `1 0 1 1,4,7,10 *` | `Math.floor(month/3)*3` 月首日 | period-stat(m) | +| 年 | y | `1 0 1 1 *` | `startOf('year')` | period-stat(q) | -- 默认值为 `default`，适用于单一指标的场景 -- 当 `data` 传入对象时自动展开为多属性（如 `{revenue: 10000, orders: 50}` 拆分为两条记录） +### 聚合方法与级联计算 -#### 实际场景：企业部门数据统计 +五种聚合方法在聚合过程中协同计算，确保高级别周期可以正确推导： -假设一家公司要统计各部门的经营数据，我们可以这样设计 channel： +| 方法 | key | 从 data-record 聚合 | 从 period-stat 聚合 | +|------|-----|---------------------|---------------------| +| 合计 | sum | `SUM(data)` | 各子窗口 sum 求和 | +| 计数 | count | `COUNT(data)` | 各子窗口 count 求和 | +| 平均 | avg | `AVG(data)` | `sum总 / count总`（非 AVG(AVG)） | +| 最小 | min | `MIN(data)` | 各子窗口 min 取最小值 | +| 最大 | max | `MAX(data)` | 各子窗口 max 取最大值 | -``` -company ← 根通道：公司整体 -company:sales ← 子通道：销售部 -company:sales:beijing ← 子通道：销售部北京分部 -company:sales:shanghai ← 子通道：销售部上海分部 -company:rd ← 子通道：研发部 -company:rd:frontend ← 子通道：研发部前端组 -company:rd:backend ← 子通道：研发部后端组 -company:hr ← 子通道：人力资源部 -``` +**关键设计**：avg 不直接对上游 avg 取平均，而是用 sum/count 重新计算，避免二次平均偏差。 -对应的 `channel-meta` 只需为根通道 `company` 创建一条记录： - -| channel | title | description | -|---------|-------|-------------| -| company | 公司经营数据 | 各部门经营数据统计 | - -**采集数据**： - -```js -// 1. 销售部北京分部上报单指标（默认 attributeName=default） -await fastify.statistics.services.collect({ - channel: 'company:sales:beijing', - data: 58000, - unit: '元', - title: '公司', - description: '各部门经营数据统计' -}); - -// 2. 销售部上海分部上报多指标，unit 为字符串时所有属性共用同一单位 -await fastify.statistics.services.collect({ - channel: 'company:sales:shanghai', - data: { revenue: 72000, orders: 150 }, - unit: '元', - title: '公司', - description: '各部门经营数据统计' -}); - -// 3. 研发部前端组上报，unit 为对象时按 attributeName 映射不同单位 -await fastify.statistics.services.collect({ - channel: 'company:rd:frontend', - data: { tasks: 12, bugs: 3 }, - unit: { tasks: '个', bugs: '个' }, - title: '公司', - description: '各部门经营数据统计' -}); -``` +### 聚合区间语义 -采集后数据会自动展开并入库： - -| channel | attributeName | data | unit | -|---------|--------------|------|------| -| company | default | 58000 | 元 | -| company:sales | default | 58000 | 元 | -| company:sales:beijing | default | 58000 | 元 | -| company | revenue | 72000 | 元 | -| company | orders | 150 | 元 | -| company:sales | revenue | 72000 | 元 | -| company:sales | orders | 150 | 元 | -| company:sales:shanghai | revenue | 72000 | 元 | -| company:sales:shanghai | orders | 150 | 元 | -| ... | ... | ... | ... | - -> 通道展开规则：`company:sales:beijing` 自动展开为 `company`、`company:sales`、`company:sales:beijing` 三条记录，确保每一级都能查到汇总数据。 - -**查询数据**： - -```js -// 1. 查询销售部本月合计（仅自身） -const salesResult = await fastify.statistics.services.query({ - channels: ['company:sales'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - aggregates: ['sum'] -}); - -// 2. 查询公司所有部门的本月合计（传入一级 channel + includeChildren） -const companyResult = await fastify.statistics.services.query({ - channels: ['company'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - aggregates: ['sum'], - includeChildren: true -}); - -// 3. 同时查询多个通道（仅自身） -const multiResult = await fastify.statistics.services.query({ - channels: ['company:sales', 'company:rd'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - aggregates: ['sum'] -}); - -// 4. 查询 revenue 和 orders 两个属性的合计与平均 -const revenueResult = await fastify.statistics.services.query({ - channels: ['company'], - startTime: '2026-05-01T00:00:00.000Z', - endTime: '2026-06-01T00:00:00.000Z', - attributeNames: ['revenue', 'orders'], - aggregates: ['sum', 'avg'] -}); -``` +所有聚合使用**左闭右开区间** `[startTime, endTime)`： -**查询返回格式**： - -> **注意**：默认（`includeChildren=false`）只返回精确匹配通道的扁平列表。`includeChildren=true` 时按通道构建树形结构，子通道数据嵌套在 `children` 数组中。`data` 字段始终为对象（按属性名映射），例如单聚合时 `data` 为 `{"default": 58000}`，多聚合时 `data` 为 `{"sum": {"default": 58000}, "avg": {"default": 29000}}`。 - -查询销售部（`channels=['company:sales']`，默认不包含子通道）返回： - -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company:sales", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150 }, - "unit": { "default": "元", "revenue": "元", "orders": "元" } - } - ] -} -``` +- `startTime`：当前时间窗口的截断起始 +- `endTime`：下一个时间窗口的起始（`getNextStart(startTime)`） +- 查询条件使用 `Op.gte(startTime)` + `Op.lt(endTime)`，确保 `endTime` 所在时刻的数据**不被包含** -查询整个公司（`channels=['company'], includeChildren=true`）返回： - -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150, "tasks": 12, "bugs": 3 }, - "unit": { "default": "元", "revenue": "元", "orders": "元", "tasks": "个", "bugs": "个" } - } - ], - "children": [ - { - "channel": "company:sales", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150 }, - "unit": { "default": "元", "revenue": "元", "orders": "元" } - } - ], - "children": [ - { - "channel": "company:sales:beijing", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 58000 }, - "unit": { "default": "元" } - } - ] - }, - { - "channel": "company:sales:shanghai", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "revenue": 72000, "orders": 150 }, - "unit": { "revenue": "元", "orders": "元" } - } - ] - } - ] - }, - { - "channel": "company:rd", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "tasks": 12, "bugs": 3 }, - "unit": { "tasks": "个", "bugs": "个" } - } - ], - "children": [ - { - "channel": "company:rd:frontend", - "items": [ - { - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "tasks": 12, "bugs": 3 }, - "unit": { "tasks": "个", "bugs": "个" } - } - ] - } - ] - } - ] - } - ] -} -``` +> 这一设计修复了此前使用 `Op.between`（闭区间）导致边界数据被重复聚合到两个窗口的 bug。 -查询多个通道（`channels=['company:sales','company:rd']`，默认不包含子通道）返回： - -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company:sales", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "default": 130000, "revenue": 72000, "orders": 150 }, - "unit": { "default": "元", "revenue": "元", "orders": "元" } - }, - { - "channel": "company:rd", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "tasks": 12, "bugs": 3 }, - "unit": { "tasks": "个", "bugs": "个" } - } - ] -} -``` +### Channel 通道层级设计 -查询 revenue 和 orders 两个属性的合计与平均（`channels=['company']`, `attributeNames=['revenue','orders']`, `aggregates=['sum','avg']`）返回： - -```json -{ - "channelMetas": { - "company": { "channel": "company", "title": "公司", "description": "各部门经营数据统计" } - }, - "list": [ - { - "channel": "company", - "period": "m", - "time": "2026-05-01T00:00:00.000Z", - "data": { "sum": { "revenue": 72000, "orders": 150 }, "avg": { "revenue": 72000, "orders": 150 } }, - "unit": { "revenue": "元", "orders": "元" } - } - ] -} -``` +**Channel（数据通道）** 采用冒号分隔的多级结构（`a:b:c`），核心思想是**从宏观到微观的层级划分**： -> `channelMetas` 按 root channel 去重，所有子通道共享同一份元数据，避免数据冗余。 +- 一级 channel（如 `sales`）是根通道，对应唯一的 `channel-meta` 记录 +- 多级 channel（如 `sales:beijing`、`sales:beijing:team-a`）是更细粒度的子通道 +- **采集时自动展开**：`company:sales:beijing` 展开为 `company`、`company:sales`、`company:sales:beijing` 三条记录 +- **查询时**：默认精确匹配；`includeChildren=true` 时匹配通道及所有子通道，返回树形结构 -#### Channel Meta 管理 +**AttributeName（属性名）** 是同一 channel 下的第二级分类维度： -通道元数据在首次采集时自动创建，也可通过服务接口管理： +- 默认值 `default`，适用于单指标场景 +- `data` 传入对象时自动展开（如 `{revenue: 10000, orders: 50}` → 两条记录） +- `unit` 支持字符串（所有属性共用）或对象（按 attributeName 映射不同单位） -```js -// 查询通道元数据 -const meta = await fastify.statistics.services.channelMeta.detail({ - channel: 'company' -}); +### 水位线机制与补偿聚合 -// 列出所有元数据 -const list = await fastify.statistics.services.channelMeta.list(); +**水位线（aggregation-watermark）** 记录每个周期下一次应聚合的起始时间，是补偿聚合的核心： -// 按通道筛选 -const list = await fastify.statistics.services.channelMeta.list({ - filter: { channel: 'company' } -}); +- 每个周期一条记录：`(period, nextTime)` +- 聚合完成后，`nextTime` 推进到下一窗口起始 +- 补偿时从 `nextTime` 开始逐步向前，追上当前时间 -// 修改元数据 -await fastify.statistics.services.channelMeta.save({ - channel: 'company', - title: '企业经营数据总览', - description: '全公司各部门经营指标汇总' -}); -``` +**启动初始化流程**（`period-stat.init()`）： -#### SSE 实时推送 - -通过 HTTP 接口或程序化 API 获取实时统计数据推送： - -```js -// HTTP 接口：GET /api/statistics/sse?channel=company&aggregates=sum&interval=5 -// 浏览器端使用 EventSource 接收 -const eventSource = new EventSource('/api/statistics/sse?channel=company&aggregates=sum&interval=5'); -eventSource.onmessage = (event) => { - const data = JSON.parse(event.data); - console.log(data); // { channelMetas, list: [{ channel, items, children }] } -}; - -// 程序化调用（在 Fastify 路由中） -fastify.get('/my-sse', async (request, reply) => { - const sseContext = await fastify.statistics.services.sseStream.send(reply, { - name: 'my-sse-channel', - params: { - channel: ['company'], - startTime: new Date(Date.now() - 3600000).toISOString(), - endTime: new Date().toISOString(), - aggregates: ['sum'] - }, - fetchData: async (params) => { - return fastify.statistics.services.query(params); - }, - interval: 5, - heartbeatInterval: 30000, - maxDuration: 1800000 - }); - - // 可手动关闭 - // sseContext.close(); - - // 监听关闭事件 - sseContext.onClose(() => { - console.log('SSE 连接已关闭'); - }); -}); -``` +1. 按 h→d→w→m→q→y 顺序依次处理 +2. 对每个周期： + - 若水位线存在且过期 → 从水位线开始补偿 + - 若水位线不存在 → 调用 `determineStartFromSource` 从源数据推断起始点 + - 若无任何源数据 → 水位线设为当前截断时间（全新系统） +3. 逐窗口执行聚合，每完成一个窗口就推进水位线 +4. 上游未完成时自动先补偿上游（如聚合 d 前确保 h 已完成） -**SSE 事件类型**： +**`determineStartFromSource` 推断逻辑**： -| 事件 | 说明 | -|------|------| -| `data`（默认） | 正常数据推送，内容为查询结果 JSON | -| `timeout` | 连接超过 maxDuration 后自动断开通知 | -| `error` | fetchData 出错时的错误事件 | -| 心跳（`: heartbeat`） | 保活注释行 | +- h 周期：从 `data-record` 的 `MIN(time)` 截断到小时 +- 其他周期：从上游 `period-stat` 的 `MIN(time)` 截断到当前周期起始 + +> 早期版本使用 `MAX(time) + nextStart` 推断起始点，导致遗漏上游最早数据。现已改为 `MIN(time)` 截断，确保所有历史数据都被覆盖。 + +**运行时补偿**（Cron 触发）： + +- 每个 Cron 周期调用 `compensate(period)` +- 每次最多处理 `compensationBatchSize` 个窗口（默认 24） +- 连续失败 `maxCompensationFailCount` 次（默认 3）后停止，下次 Cron 继续 +- 每个周期有独立锁（`compensatingLocks`），防止并发补偿 + +### 数据保留策略 + +通过 Cron 定时清理过期数据，避免数据无限增长： -**SSE 上下文方法**： +| 数据类型 | 保留策略 | 安全检查 | +|----------|----------|----------| +| data-record | `dataRetentionDays` 天（默认 7 天） | 不超过 h 周期水位线 | +| period-stat(h) | 当月 | 不超过 d 周期水位线 | +| period-stat(d) | 当年 | 不超过 w、m 周期水位线 | +| period-stat(w) | 当年 | 无下游依赖 | +| period-stat(m/q/y) | 永久保留 | - | -| 方法 | 说明 | +**安全检查**：删除前检查下游水位线，确保尚未聚合的数据不会被提前删除。 + +### 缓冲写入模式 + +当配置 `cache` 实例时，采集数据先写入内存缓冲区，再定时批量持久化： + +- 缓冲区达到 `collectMaxBufferSize`（默认 1000）时触发 flush +- 定时 `collectFlushInterval`（默认 5000ms）自动 flush +- 缓冲区溢出上限 `collectMaxBufferOverflow`（默认 `maxBufferSize * 2`），超出时丢弃最旧数据 +- 进程关闭时（`onClose` hook）持久化缓冲区到 cache 并执行最终 flush +- 启动时从 cache 恢复缓冲区数据（`restoreBuffer`） + +无 cache 时，每次采集直接写入数据库（`collectImmediate`）。 + +### 查询缓存 + +查询结果自动缓存，减少重复查询的数据库压力： + +| 特性 | 说明 | |------|------| -| `isConnected()` | 返回当前连接状态 | -| `close()` | 手动关闭 SSE 连接 | -| `onClose(callback)` | 注册连接关闭回调，若已断开则立即执行 | +| 外部缓存 | 配置 `cache` 时使用，支持 TTL | +| 内存 LRU 缓存 | 无外部缓存时使用，最大 `queryCacheMaxEntries` 条 | +| 版本校验 | 缓存条目记录写入时的通道版本号，读取时校验版本是否变化 | +| TTL 策略 | 实时查询（endTime 在当前小时内）用 `queryCacheTTL`（30s），历史查询用 `queryCacheHistoryTTL`（3600s） | +| 补偿期间 | 正在补偿聚合时查询不走缓存，确保数据实时性 | + +**缓存失效**：采集数据时自动调用 `invalidateQueryCache(affectedChannels)`，递增对应通道及其所有前缀的版本号。 + +### SSE 实时推送 + +基于 Server-Sent Events 的实时统计推送： + +- 按 `interval`（默认 5 秒）定时调用 `fetchData` 获取最新数据推送 +- 心跳保活（默认 30 秒），防止连接被代理/负载均衡器断开 +- 最大连接时长（默认 30 分钟），超时自动断开并推送 `timeout` 事件 +- 防止推送重叠：当前推送未完成时跳过下一次 +- 缓存复用：相同 `name`+`params`+`interval` 在同一时间窗口内命中缓存 +### 重置与修复 + +提供 `resetPeriodStats` 方法用于修复错误的聚合数据： + +- 删除指定周期和时间范围的 period-stat 记录 +- 重置水位线到指定起始点 +- 支持 `cascade=true` 级联重置下游周期（如重置 h 时同时重置依赖 h 的 d、w、m、q、y） + +### 技术栈 + +- **Fastify** + fastify-plugin + fastify-namespace +- **Sequelize**（数据持久化，支持多种数据库） +- **fastify-cron**（定时聚合与清理任务） +- **dayjs**（时间处理，支持 UTC 和时区扩展） +- **lodash**（工具函数） + +### 主要特性 + +- **数据采集**：支持单条和批量数据上报，自动展开多属性对象和多级通道 +- **缓冲写入**：支持缓存缓冲模式，定时批量写入数据库，减少写入压力 +- **多周期聚合**：h/d/w/m/q/y 六种统计周期，自动通过 Cron 定时聚合 +- **聚合方法**：sum、avg、count、min、max 五种聚合计算 +- **灵活查询**：按通道、时间范围、属性名、聚合方法查询统计结果，自动合并当前小时未聚合的原始数据 +- **SSE 实时推送**：基于 Server-Sent Events 的实时数据推送 +- **时区支持**：查询时支持传入客户端时区（IANA 格式） +- **事务安全**：所有数据库写操作使用事务保证原子性，聚合操作支持幂等（upsert） +- **查询缓存**：支持内存 LRU 或外部缓存，带版本校验和 TTL 策略 diff --git a/index.js b/index.js index 773d13e..71d4656 100644 --- a/index.js +++ b/index.js @@ -45,6 +45,10 @@ module.exports = fp( ['services', path.resolve(__dirname, './libs/services')] ] }); + + fastify.addHook('onReady', async () => { + await fastify[options.name].services.periodStat.init(); + }); }, { name: 'fastify-statistics', diff --git a/libs/services/period-stat.js b/libs/services/period-stat.js index 283b36b..7ea2c18 100644 --- a/libs/services/period-stat.js +++ b/libs/services/period-stat.js @@ -184,51 +184,64 @@ module.exports = fp(async (fastify, options) => { }; const aggregateFromDataRecord = async (period, startTime, endTime) => { - const results = await models.dataRecord.findAll({ - attributes: ['channel', 'attributeName', [fn('MAX', col('unit')), 'unit'], ...AGGREGATE_TYPES.map(({ key, fn: aggFn }) => [fn(aggFn, col('data')), key])], - where: { - time: { [Op.between]: [startTime, endTime] } - }, - group: ['channel', 'attributeName'], - raw: true - }); + const transaction = await sequelize.transaction(); + try { + // 使用 [startTime, endTime) 左闭右开区间，endTime 是下一窗口起始，不应包含 + const timeRange = { [Op.gte]: startTime, [Op.lt]: endTime }; - const records = []; - for (const row of results) { - for (const { key } of AGGREGATE_TYPES) { - const value = row[key]; - if (value === null || value === undefined) continue; - records.push({ - period, - time: startTime, - channel: row.channel, - attributeName: row.attributeName, - aggregate: key, - data: parseFloat(value), - unit: row.unit - }); + const results = await models.dataRecord.findAll({ + attributes: ['channel', 'attributeName', [fn('MAX', col('unit')), 'unit'], ...AGGREGATE_TYPES.map(({ key, fn: aggFn }) => [fn(aggFn, col('data')), key])], + where: { + time: timeRange + }, + group: ['channel', 'attributeName'], + raw: true, + transaction + }); + + const records = []; + for (const row of results) { + for (const { key } of AGGREGATE_TYPES) { + const value = row[key]; + if (value === null || value === undefined) continue; + records.push({ + period, + time: startTime, + channel: row.channel, + attributeName: row.attributeName, + aggregate: key, + data: parseFloat(value), + unit: row.unit + }); + } } - } - if (records.length > 0) { - const transaction = await sequelize.transaction(); - try { + if (records.length > 0) { await models.periodStat.bulkCreate(records, { transaction, updateOnDuplicate: UPSERT_FIELDS }); - await transaction.commit(); - } catch (e) { - await transaction.rollback(); - throw e; } - } - return records; + // 聚合完成后删除已聚合的源数据 + await models.dataRecord.destroy({ + where: { + time: timeRange + }, + transaction + }); + + await transaction.commit(); + return records; + } catch (e) { + await transaction.rollback(); + throw e; + } }; const aggregateFromPeriodStat = async (period, fromPeriod, startTime, endTime) => { + // 使用 [startTime, endTime) 左闭右开区间，endTime 是下一窗口起始，不应包含 const rows = await models.periodStat.findAll({ where: { period: fromPeriod, - time: { [Op.between]: [startTime, endTime] } + time: { [Op.gte]: startTime, [Op.lt]: endTime } }, raw: true }); @@ -249,10 +262,10 @@ module.exports = fp(async (fastify, options) => { const records = []; for (const group of Object.values(grouped)) { const items = group.items; - const sums = items.filter(i => i.aggregate === 'sum').map(i => i.data); - const counts = items.filter(i => i.aggregate === 'count').map(i => i.data); - const mins = items.filter(i => i.aggregate === 'min').map(i => i.data); - const maxs = items.filter(i => i.aggregate === 'max').map(i => i.data); + const sums = items.filter(i => i.aggregate === 'sum').map(i => parseFloat(i.data)); + const counts = items.filter(i => i.aggregate === 'count').map(i => parseFloat(i.data)); + const mins = items.filter(i => i.aggregate === 'min').map(i => parseFloat(i.data)); + const maxs = items.filter(i => i.aggregate === 'max').map(i => parseFloat(i.data)); const base = { period, @@ -329,53 +342,59 @@ module.exports = fp(async (fastify, options) => { }; const setWatermark = async (period, nextTime) => { - await models.aggregationWatermark.upsert({ period, nextTime }); + const existing = await models.aggregationWatermark.findOne({ where: { period } }); + if (existing) { + await existing.update({ nextTime }); + } else { + await models.aggregationWatermark.create({ period, nextTime }); + } }; - const initWatermark = async period => { + const determineStartFromSource = async period => { const config = PERIOD_CONFIG[period]; const dependency = PERIOD_DEPENDENCY[period]; - const existing = await getWatermark(period); - if (existing) return existing; - - let minTime = null; if (dependency.source === 'data-record') { + // h 周期：从 data-record 的最早记录开始 const row = await models.dataRecord.findOne({ attributes: [[fn('MIN', col('time')), 'minTime']], raw: true }); - minTime = row?.minTime; - } else { - const row = await models.periodStat.findOne({ - attributes: [[fn('MIN', col('time')), 'minTime']], - where: { period: dependency.fromPeriod }, - raw: true - }); - minTime = row?.minTime; + return row?.minTime ? config.truncateTime(new Date(row.minTime)) : null; } - const nextTime = minTime ? config.truncateTime(new Date(minTime)) : config.truncateTime(new Date()); - await setWatermark(period, nextTime); - return nextTime; + // 其他周期：从上游 period-stat 已有数据的最小时间截断到当前周期开始 + // 使用 MIN 而非 MAX+nextStart，确保所有上游数据都被聚合到当前周期 + const row = await models.periodStat.findOne({ + attributes: [[fn('MIN', col('time')), 'minTime']], + where: { period: dependency.fromPeriod }, + raw: true + }); + return row?.minTime ? config.truncateTime(new Date(row.minTime)) : null; }; const compensatingLocks = {}; let startupCompensating = false; + let initialized = false; const isCompensating = () => startupCompensating || Object.values(compensatingLocks).some(v => v); - const compensate = async period => { + const compensate = async (period, { maxWindows } = {}) => { if (compensatingLocks[period]) return; compensatingLocks[period] = true; try { const config = PERIOD_CONFIG[period]; const dependency = PERIOD_DEPENDENCY[period]; - let nextTime = await initWatermark(period); + const watermark = await getWatermark(period); + if (!watermark) return; + let nextTime = new Date(watermark); const nowTruncated = config.truncateTime(new Date()); + const windowLimit = maxWindows ?? compensationBatchSize; let count = 0; - while (nextTime < nowTruncated && count < compensationBatchSize) { + let failCount = 0; + const maxFailCount = options.maxCompensationFailCount ?? 3; + while (nextTime < nowTruncated && count < windowLimit && failCount < maxFailCount) { const endTime = config.getNextStart(nextTime); if (dependency.source === 'period-stat') { @@ -387,17 +406,23 @@ module.exports = fp(async (fastify, options) => { try { await aggregate(period, { startTime: nextTime, endTime }); + nextTime = endTime; + await setWatermark(period, nextTime); + count++; + failCount = 0; } catch (e) { - fastify.log.error(`Failed to compensate period ${period} [${nextTime.toISOString()} - ${endTime.toISOString()}]: ${e.message}`); - break; + failCount++; + fastify.log.error(`Failed to compensate period ${period} [${nextTime.toISOString()} - ${endTime.toISOString()}] (${failCount}/${maxFailCount}): ${e.message}`); + // 跳过失败窗口，推进水位线，避免无限重试同一窗口 + nextTime = endTime; + await setWatermark(period, nextTime); + count++; } - - nextTime = endTime; - await setWatermark(period, nextTime); - count++; } - if (nextTime < nowTruncated) { + if (failCount >= maxFailCount && nextTime < nowTruncated) { + fastify.log.error(`period=${period} 补偿连续失败 ${maxFailCount} 次，停止补偿，下次 cron 继续`); + } else if (nextTime < nowTruncated) { fastify.log.warn(`period=${period} 补偿未完成，已补 ${count} 个窗口，下次继续`); } else if (count > 1) { fastify.log.info(`period=${period} 补偿完成，共补 ${count} 个窗口`); @@ -407,37 +432,159 @@ module.exports = fp(async (fastify, options) => { } }; - if (options.compensationEnabled !== false) { + // period-stat 数据保留策略：h 保留当月，d/w 保留当年，m/q/y 永久保留 + const cleanupOldPeriodStats = async () => { + const now = dayjs(); + + // h: 保留当月，需确保 d 已聚合（检查 d 水位线） + const dWatermark = await getWatermark('d'); + const hCutoff = now.startOf('month').toDate(); + const hSafeCutoff = dWatermark && new Date(dWatermark) < hCutoff ? new Date(dWatermark) : hCutoff; + const hCount = await models.periodStat.destroy({ + where: { period: 'h', time: { [Op.lt]: hSafeCutoff } } + }); + if (hCount > 0) { + fastify.log.info(`Cleaned up ${hCount} old period-stat records for period=h before ${hSafeCutoff.toISOString()}`); + } + + // d: 保留当年，需确保 w/m 已聚合（检查 w、m 水位线） + const wWatermark = await getWatermark('w'); + const mWatermark = await getWatermark('m'); + let dSafeCutoff = now.startOf('year').toDate(); + if (wWatermark && new Date(wWatermark) < dSafeCutoff) dSafeCutoff = new Date(wWatermark); + if (mWatermark && new Date(mWatermark) < dSafeCutoff) dSafeCutoff = new Date(mWatermark); + const dCount = await models.periodStat.destroy({ + where: { period: 'd', time: { [Op.lt]: dSafeCutoff } } + }); + if (dCount > 0) { + fastify.log.info(`Cleaned up ${dCount} old period-stat records for period=d before ${dSafeCutoff.toISOString()}`); + } + + // w: 保留当年，无下游依赖 + const wCutoff = now.startOf('year').toDate(); + const wCount = await models.periodStat.destroy({ + where: { period: 'w', time: { [Op.lt]: wCutoff } } + }); + if (wCount > 0) { + fastify.log.info(`Cleaned up ${wCount} old period-stat records for period=w before ${wCutoff.toISOString()}`); + } + }; + + const init = async () => { + const maxCompensationWindows = options.maxCompensationWindows ?? Infinity; + const maxFailCount = options.maxCompensationFailCount ?? 3; + const compensationEnabled = options.compensationEnabled !== false; + startupCompensating = true; - (async () => { - try { - for (const period of PERIOD_ORDER) { - await compensate(period); + try { + for (const period of PERIOD_ORDER) { + const config = PERIOD_CONFIG[period]; + const nowTruncated = config.truncateTime(new Date()); + + // Step 1: 确定补偿起始点 + let startTime; + const existing = await getWatermark(period); + + if (existing && new Date(existing) < nowTruncated) { + // 场景一：水位线存在但过期 + startTime = new Date(existing); + fastify.log.info(`period=${period} 水位线过期 (${existing}), 从 ${startTime.toISOString()} 开始补偿`); + } else if (existing) { + // 水位线已是最新的，跳过 + fastify.log.info(`period=${period} 水位线正常: ${existing}`); + continue; + } else { + // 场景二：水位线不存在，从源数据推断 + startTime = await determineStartFromSource(period); + if (!startTime) { + // 场景三：全新系统，无任何数据 + await setWatermark(period, nowTruncated); + fastify.log.info(`period=${period} 全新系统，水位线设为 ${nowTruncated.toISOString()}`); + continue; + } + fastify.log.info(`period=${period} 水位线不存在，从源数据推断起始点 ${startTime.toISOString()}`); + } + + if (!compensationEnabled) { + // 补偿未启用，仅设置水位线到起始点，不执行补偿 + await setWatermark(period, startTime); + fastify.log.info(`period=${period} 补偿未启用，水位线设为 ${startTime.toISOString()}`); + continue; + } + + // Step 2: 执行补偿聚合，逐窗口推进水位线 + let nextTime = startTime; + let count = 0; + let failCount = 0; + + while (nextTime < nowTruncated && count < maxCompensationWindows) { + const endTime = config.getNextStart(nextTime); + + try { + await aggregate(period, { startTime: nextTime, endTime }); + count++; + failCount = 0; + } catch (e) { + failCount++; + fastify.log.error(`period=${period} 补偿失败 [${nextTime.toISOString()}] (${failCount}/${maxFailCount}): ${e.message}`); + if (failCount >= maxFailCount) { + fastify.log.error(`period=${period} 连续失败 ${maxFailCount} 次，停止补偿`); + break; + } + } + + nextTime = endTime; + // 逐窗口推进水位线，确保水位线与实际聚合进度一致 + await setWatermark(period, nextTime); + } + + if (nextTime < nowTruncated) { + fastify.log.warn(`period=${period} 补偿未完成（到 ${nextTime.toISOString()}），下次 cron 继续`); + } else { + fastify.log.info(`period=${period} 补偿完成，共 ${count} 个窗口`); } - } catch (e) { - fastify.log.error(`Startup compensation failed: ${e.message}`); - } finally { - startupCompensating = false; } - })(); - } - if (fastify.cron) { - for (const [period, config] of Object.entries(PERIOD_CONFIG)) { + fastify.log.info('Startup compensation finished'); + } catch (e) { + fastify.log.error(`Startup compensation failed: ${e.message}`); + } finally { + startupCompensating = false; + } + + if (fastify.cron) { + for (const [period, config] of Object.entries(PERIOD_CONFIG)) { + fastify.cron.createJob({ + name: `statistics-period-stat-${period}`, + cronTime: config.cronTime, + onTick: async () => { + try { + await compensate(period); + } catch (e) { + fastify.log.error(`Failed to compensate period ${period}: ${e.message}`); + } + }, + startWhenReady: true + }); + } + fastify.cron.createJob({ - name: `statistics-period-stat-${period}`, - cronTime: config.cronTime, + name: 'statistics-period-stat-cleanup', + cronTime: '0 3 * * *', onTick: async () => { try { - await compensate(period); + await cleanupOldPeriodStats(); } catch (e) { - fastify.log.error(`Failed to compensate period ${period}: ${e.message}`); + fastify.log.error(`Failed to cleanup old period-stat records: ${e.message}`); } }, startWhenReady: true }); } - } + + initialized = true; + fastify.log.info('Period statistics service initialized'); + }; const formatGroupData = items => { const aggSet = new Set(); @@ -532,6 +679,10 @@ module.exports = fp(async (fastify, options) => { } } + if (!initialized) { + throw new Error('Period statistics service is not initialized yet'); + } + const channelList = Array.isArray(channels) ? channels : channels ? [channels] : []; const now = tz ? dayjs().tz(tz) : dayjs(); const currentHourStart = now.startOf('hour').toDate(); @@ -556,7 +707,7 @@ module.exports = fp(async (fastify, options) => { const result = await doQuery({ channelList, startTime, endTime, attributeNames, aggregateList, tz, includeChildren, currentHourStart, isRealtime }); const ttl = isRealtime ? queryCacheTTL : queryCacheHistoryTTL; - await queryCacheSet(cacheKey, result, ttl, isRealtime ? channelList : null); + await queryCacheSet(cacheKey, result, ttl, channelList); return result; } @@ -676,13 +827,71 @@ module.exports = fp(async (fastify, options) => { return { channelMetas, list }; }; + /** + * 重置指定周期的 period-stat 数据和水位线 + * 用于修复错误聚合数据：删除旧数据 → 重置水位线 → 重新补偿聚合 + * @param {string} period - 周期类型 + * @param {object} [options] + * @param {Date} [options.startTime] - 重置起始时间（水位线将设为此值），默认为当前截断时间 + * @param {Date} [options.endTime] - 重置结束时间（仅删除此范围内的 period-stat 数据），默认删除全部 + * @param {boolean} [options.cascade=false] - 是否级联重置下游周期（如重置 h 时同时重置依赖 h 的 d） + * @returns {{ period, deletedCount, nextTime }} + */ + const resetPeriodStats = async (period, { startTime, endTime, cascade = false } = {}) => { + const config = PERIOD_CONFIG[period]; + if (!config) { + throw new Error(`Unsupported period: ${period}, supported: ${Object.keys(PERIOD_CONFIG).join(',')}`); + } + + const where = { period }; + if (startTime && endTime) { + where.time = { [Op.between]: [startTime, endTime] }; + } else if (startTime) { + where.time = { [Op.gte]: startTime }; + } else if (endTime) { + where.time = { [Op.lte]: endTime }; + } + + const deletedCount = await models.periodStat.destroy({ where }); + + // 重置水位线 + const nextTime = startTime || config.truncateTime(new Date()); + await setWatermark(period, nextTime); + + // 使查询缓存失效 + invalidateQueryCache(); + + const result = { period, deletedCount, nextTime }; + + // 级联重置下游周期 + if (cascade) { + const downstreamPeriods = Object.entries(PERIOD_DEPENDENCY) + .filter(([, dep]) => dep.source === 'period-stat' && dep.fromPeriod === period) + .map(([p]) => p); + + for (const dp of downstreamPeriods) { + const subResult = await resetPeriodStats(dp, { + startTime: startTime ? PERIOD_CONFIG[dp].truncateTime(startTime) : undefined, + endTime: endTime ? PERIOD_CONFIG[dp].truncateTime(endTime) : undefined, + cascade: true + }); + result[`cascade_${dp}`] = subResult; + } + } + + return result; + }; + Object.assign(fastify[options.name].services, { query, periodStat: { + init, aggregate, query, isCompensating, - invalidateQueryCache + invalidateQueryCache, + cleanupOldPeriodStats, + resetPeriodStats } }); }); diff --git a/package.json b/package.json index bcdbc60..a7cb1af 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@kne/fastify-statistics", - "version": "0.1.0-alpha.8", + "version": "0.1.0-alpha.9", "description": "基于 Fastify 的数据采集与多周期聚合统计插件，支持缓冲写入、时区查询和自动 Cron 聚合", "main": "index.js", "scripts": { diff --git a/test/period-stat-aggregate.test.js b/test/period-stat-aggregate.test.js new file mode 100644 index 0000000..9c229b3 --- /dev/null +++ b/test/period-stat-aggregate.test.js @@ -0,0 +1,422 @@ +const { expect } = require('chai'); +const { mockPeriodStatService, createMockFastify, createFullMockFastify } = require('./period-stat-helpers'); + +describe('@kne/fastify-statistics', function () { + describe('aggregate 聚合测试', () => { + describe('aggregate 方法 - 从 data-record 聚合 (period=h)', () => { + it('should throw error for unsupported period', async () => { + const { fastify } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + try { + await fastify.statistics.services.periodStat.aggregate('x'); + expect.fail('should have thrown'); + } catch (e) { + expect(e.message).to.include('Unsupported period: x'); + } + + await fastify.close(); + }); + + it('should generate records for all aggregate types from data-record when period=h', async () => { + const { fastify, findAllResults, bulkCreateCalls } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + findAllResults.push({ + channel: 'temperature', + attributeName: 'value', + sum: 100, + avg: 25, + count: 4, + min: 10, + max: 40 + }); + + const records = await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); + + expect(records.length).to.equal(5); + expect(bulkCreateCalls.length).to.equal(1); + expect(bulkCreateCalls[0].records.length).to.equal(5); + expect(bulkCreateCalls[0].opts.updateOnDuplicate).to.deep.equal(['data', 'unit']); + + const aggregates = records.map(r => r.aggregate); + expect(aggregates).to.have.members(['sum', 'avg', 'count', 'min', 'max']); + + const sumRecord = records.find(r => r.aggregate === 'sum'); + expect(sumRecord.period).to.equal('h'); + expect(sumRecord.channel).to.equal('temperature'); + expect(sumRecord.attributeName).to.equal('value'); + expect(sumRecord.data).to.equal(100); + expect(sumRecord.title).to.be.undefined; + expect(sumRecord.description).to.be.undefined; + expect(sumRecord.unit).to.be.undefined; + + await fastify.close(); + }); + + it('should delete data-record after successful h aggregation', async () => { + const { fastify, findAllResults, destroyCalls } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); + + expect(destroyCalls.length).to.equal(1); + expect(destroyCalls[0].where.time.gte).to.deep.equal(startTime); + expect(destroyCalls[0].where.time.lt).to.deep.equal(endTime); + + await fastify.close(); + }); + + it('should delete data-record even when no records to aggregate', async () => { + const { fastify, destroyCalls } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); + + expect(destroyCalls.length).to.equal(1); + + await fastify.close(); + }); + + it('should skip null aggregate values from data-record', async () => { + const { fastify, findAllResults } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 100, avg: null, count: 5, min: null, max: 50 + }); + + const records = await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); + + expect(records.length).to.equal(3); + const aggregates = records.map(r => r.aggregate); + expect(aggregates).to.have.members(['sum', 'count', 'max']); + + await fastify.close(); + }); + }); + + describe('aggregate 方法 - 从 period-stat 聚合 (period>d/w/m/q/y)', () => { + it('should aggregate from period-stat(h) when period=d', async () => { + const { fastify, periodStatRows, bulkCreateCalls, destroyCalls } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'sum', data: 30, time: new Date('2026-05-01T00:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'sum', data: 20, time: new Date('2026-05-01T01:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'count', data: 3, time: new Date('2026-05-01T00:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'count', data: 2, time: new Date('2026-05-01T01:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'min', data: 8, time: new Date('2026-05-01T00:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'min', data: 12, time: new Date('2026-05-01T01:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'max', data: 22, time: new Date('2026-05-01T00:00:00.000Z') }, + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'max', data: 28, time: new Date('2026-05-01T01:00:00.000Z') } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + + expect(records.length).to.equal(5); + + const sumRecord = records.find(r => r.aggregate === 'sum'); + expect(sumRecord.data).to.equal(50); + + const countRecord = records.find(r => r.aggregate === 'count'); + expect(countRecord.data).to.equal(5); + + const avgRecord = records.find(r => r.aggregate === 'avg'); + expect(avgRecord.data).to.equal(50 / 5); + + const minRecord = records.find(r => r.aggregate === 'min'); + expect(minRecord.data).to.equal(8); + + const maxRecord = records.find(r => r.aggregate === 'max'); + expect(maxRecord.data).to.equal(28); + + expect(destroyCalls.length).to.equal(0); + + // Verify bulkCreate used updateOnDuplicate for idempotency + expect(bulkCreateCalls.length).to.equal(1); + expect(bulkCreateCalls[0].opts.updateOnDuplicate).to.deep.equal(['data', 'unit']); + + await fastify.close(); + }); + + it('should aggregate multiple channels separately from period-stat', async () => { + const { fastify, periodStatRows } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'sum', data: 50, time: startTime }, + { period: 'h', channel: 'humidity', attributeName: 'value', aggregate: 'sum', data: 200, time: startTime } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + + expect(records.filter(r => r.channel === 'temperature').length).to.equal(1); + expect(records.filter(r => r.channel === 'humidity').length).to.equal(1); + + await fastify.close(); + }); + + it('should compute avg from sum and count of lower period', async () => { + const { fastify, periodStatRows } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 120, time: startTime }, + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'count', data: 3, time: startTime } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + + const avgRecord = records.find(r => r.aggregate === 'avg'); + expect(avgRecord.data).to.equal(40); + + await fastify.close(); + }); + + it('should not call bulkCreate when no period-stat rows to aggregate', async () => { + const { fastify, bulkCreateCalls } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + + expect(records.length).to.equal(0); + expect(bulkCreateCalls.length).to.equal(0); + + await fastify.close(); + }); + + it('should group by channel and attributeName', async () => { + const { fastify, periodStatRows } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'd', channel: 'temperature', attributeName: 'high', aggregate: 'sum', data: 50, time: startTime }, + { period: 'd', channel: 'temperature', attributeName: 'low', aggregate: 'sum', data: 30, time: startTime } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('m', { startTime, endTime }); + + expect(records.length).to.equal(2); + const highRecord = records.find(r => r.attributeName === 'high'); + const lowRecord = records.find(r => r.attributeName === 'low'); + expect(highRecord.data).to.equal(50); + expect(lowRecord.data).to.equal(30); + + await fastify.close(); + }); + + it('should handle null attributeName in aggregateFromPeriodStat', async () => { + const { fastify, periodStatRows } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: null, aggregate: 'sum', data: 50, time: startTime } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + expect(records.length).to.equal(1); + expect(records[0].attributeName).to.equal(null); + + await fastify.close(); + }); + + it('should handle only count aggregate without sum in aggregateFromPeriodStat', async () => { + const { fastify, periodStatRows } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'count', data: 5, time: startTime }, + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'min', data: 1, time: startTime }, + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'max', data: 10, time: startTime } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + const aggregates = records.map(r => r.aggregate); + expect(aggregates).to.include('count'); + expect(aggregates).to.include('min'); + expect(aggregates).to.include('max'); + expect(aggregates).to.not.include('sum'); + expect(aggregates).to.not.include('avg'); + + await fastify.close(); + }); + }); + + describe('事务回滚测试', () => { + it('should rollback transaction when bulkCreate fails in aggregateFromDataRecord', async () => { + const { fastify, findAllResults, mockModel } = createMockFastify(); + let rollbackCalled = false; + const mockTransaction = { + commit: async () => {}, + rollback: async () => { rollbackCalled = true; } + }; + fastify.sequelize.instance.transaction = async () => mockTransaction; + + mockModel.periodStat.bulkCreate = async () => { + throw new Error('bulkCreate error'); + }; + + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + try { + await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); + expect.fail('should have thrown'); + } catch (e) { + expect(e.message).to.equal('bulkCreate error'); + expect(rollbackCalled).to.be.true; + } + + await fastify.close(); + }); + + it('should rollback transaction when bulkCreate fails in aggregateFromPeriodStat', async () => { + const { fastify, periodStatRows, mockModel } = createMockFastify(); + let rollbackCalled = false; + const mockTransaction = { + commit: async () => {}, + rollback: async () => { rollbackCalled = true; } + }; + fastify.sequelize.instance.transaction = async () => mockTransaction; + + mockModel.periodStat.bulkCreate = async () => { + throw new Error('bulkCreate error'); + }; + + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: startTime } + ); + + try { + await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + expect.fail('should have thrown'); + } catch (e) { + expect(e.message).to.equal('bulkCreate error'); + expect(rollbackCalled).to.be.true; + } + + await fastify.close(); + }); + }); + + describe('aggregate 自动时间计算', () => { + it('should auto-calculate startTime and endTime when not provided', async () => { + const { fastify, findAllResults } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + const records = await fastify.statistics.services.periodStat.aggregate('h'); + + expect(records.length).to.equal(1); + expect(records[0].aggregate).to.equal('sum'); + expect(records[0].data).to.equal(10); + + await fastify.close(); + }); + + it('should auto-calculate time for all period types', async () => { + const periods = ['h', 'd', 'w', 'm', 'q', 'y']; + for (const period of periods) { + const { fastify, findAllResults, periodStatRows } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + if (period === 'h') { + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + } else { + periodStatRows.push( + { period: period === 'd' ? 'h' : period === 'w' || period === 'm' ? 'd' : period === 'q' ? 'm' : 'q', + channel: 'ch1', attributeName: 'val', + aggregate: 'sum', data: 10, time: new Date() } + ); + } + + const records = await fastify.statistics.services.periodStat.aggregate(period); + // Should not throw, auto time calculation works for all periods + expect(records).to.be.an('array'); + + await fastify.close(); + } + }); + }); + + describe('aggregateFromPeriodStat unit 继承测试', () => { + it('should set unit to null when items have no unit field', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-02T00:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 50, time: startTime } + ); + + const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); + + expect(records.length).to.equal(1); + expect(records[0].unit).to.equal(null); + + await fastify.close(); + }); + }); + }); +}); diff --git a/test/period-stat-compensate.test.js b/test/period-stat-compensate.test.js new file mode 100644 index 0000000..a30f9db --- /dev/null +++ b/test/period-stat-compensate.test.js @@ -0,0 +1,571 @@ +const { expect } = require('chai'); +const { + mockPeriodStatService, createMockFastify, createCompensateMockFastify, createFullMockFastify +} = require('./period-stat-helpers'); + +describe('@kne/fastify-statistics', function () { + describe('compensate 补偿与调度测试', () => { + describe('cron 任务注册测试', () => { + it('should register cron jobs when fastify.cron is available', async () => { + const { fastify } = createMockFastify(); + const createdJobs = []; + + fastify.decorate('cron', { + createJob: (jobConfig) => { + createdJobs.push(jobConfig); + } + }); + + await mockPeriodStatService(fastify, { name: 'statistics' }); + + expect(createdJobs.length).to.equal(7); + + const jobNames = createdJobs.map(j => j.name); + expect(jobNames).to.include('statistics-period-stat-h'); + expect(jobNames).to.include('statistics-period-stat-d'); + expect(jobNames).to.include('statistics-period-stat-w'); + expect(jobNames).to.include('statistics-period-stat-m'); + expect(jobNames).to.include('statistics-period-stat-q'); + expect(jobNames).to.include('statistics-period-stat-y'); + + expect(createdJobs[0].cronTime).to.exist; + expect(createdJobs[0].onTick).to.be.a('function'); + expect(createdJobs[0].startWhenReady).to.be.true; + + await fastify.close(); + }); + + it('should not register cron jobs when fastify.cron is not available', async () => { + const { fastify } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + expect(fastify.statistics.services.periodStat.aggregate).to.be.a('function'); + + await fastify.close(); + }); + }); + + describe('cron onTick 错误处理', () => { + it('should catch and log error when aggregate fails in cron onTick', async () => { + const { fastify } = createMockFastify(); + const createdJobs = []; + let logErrorCalled = false; + + const originalLogError = fastify.log.error; + fastify.log.error = (msg) => { + logErrorCalled = true; + return originalLogError ? originalLogError.call(fastify.log, msg) : undefined; + }; + + const watermarkStore = {}; + fastify.statistics.models.aggregationWatermark = { + findOne: async ({ where }) => watermarkStore[where.period] || null, + upsert: async (data) => { watermarkStore[data.period] = data; }, + create: async (data) => { watermarkStore[data.period] = data; return data; } + }; + + fastify.decorate('cron', { + createJob: (jobConfig) => { + createdJobs.push(jobConfig); + } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const pastTime = new Date('2020-01-01T00:00:00.000Z'); + watermarkStore['h'] = { period: 'h', nextTime: pastTime }; + + fastify.statistics.models.dataRecord.findAll = async () => { + throw new Error('Cron aggregate error'); + }; + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + expect(hJob).to.exist; + + await hJob.onTick(); + + expect(logErrorCalled).to.be.true; + + await fastify.close(); + }); + }); + + describe('compensate 补偿逻辑测试', () => { + it('should initialize watermark from data-record min time when no existing watermark', async () => { + const { fastify, findAllResults, watermarkStore, mockModel } = createCompensateMockFastify(); + + const minTime = new Date('2026-05-01T00:00:00.000Z'); + mockModel.dataRecord.findOne = async () => ({ minTime }); + mockModel.periodStat.findOne = async () => ({ minTime }); + + expect(watermarkStore['h']).to.be.undefined; + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + expect(watermarkStore['h']).to.exist; + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { + createdJobs.push(jobConfig); + } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + if (hJob) { + await hJob.onTick(); + expect(watermarkStore['h']).to.exist; + } + + await fastify.close(); + }); + + it('should return existing watermark without reinitializing', async () => { + const { fastify, watermarkStore } = createCompensateMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const existingTime = new Date('2026-05-01T00:00:00.000Z'); + watermarkStore['h'] = { period: 'h', nextTime: existingTime }; + + let findOneCalled = false; + fastify.statistics.models.dataRecord.findOne = async () => { findOneCalled = true; return null; }; + fastify.statistics.models.periodStat.findOne = async () => { findOneCalled = true; return null; }; + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { + createdJobs.push(jobConfig); + } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + if (hJob) { + await hJob.onTick(); + } + + await fastify.close(); + }); + + it('should initialize watermark from period-stat min time for non-h periods', async () => { + const { fastify, findAllResults, watermarkStore, mockModel } = createCompensateMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const minTime = new Date('2026-05-01T00:00:00.000Z'); + mockModel.periodStat.findOne = async (opts) => { + if (opts.where && opts.where.period === 'h') return { minTime }; + return null; + }; + mockModel.dataRecord.findOne = async () => ({ minTime }); + + watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-27T00:00:00.000Z') }; + + fastify.statistics.models.periodStat.findAll = async (opts) => { + const period = opts.where && opts.where.period; + if (period === 'h') { + return [{ period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: minTime }]; + } + return []; + }; + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { + createdJobs.push(jobConfig); + } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const dJob = createdJobs.find(j => j.name === 'statistics-period-stat-d'); + if (dJob) { + await dJob.onTick(); + } + + await fastify.close(); + }); + + it('should return existing watermark without reinitializing (via aggregate)', async () => { + const { fastify, watermarkStore } = createCompensateMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const existingTime = new Date('2026-05-01T00:00:00.000Z'); + watermarkStore['h'] = { period: 'h', nextTime: existingTime }; + + const records = await fastify.statistics.services.periodStat.aggregate('h'); + expect(watermarkStore['h'].nextTime).to.deep.equal(existingTime); + + await fastify.close(); + }); + + it('should skip compensate when lock is already held', async () => { + const { fastify, watermarkStore } = createCompensateMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + expect(fastify.statistics.services.periodStat.isCompensating()).to.be.false; + + await fastify.close(); + }); + }); + + describe('invalidateQueryCache 版本递增测试', () => { + it('should increment channel versions for multi-level channels', async () => { + const { fastify } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + fastify.statistics.services.periodStat.invalidateQueryCache(['device:sensor:temp']); + + expect(fastify.statistics.services.periodStat.isCompensating).to.be.a('function'); + + await fastify.close(); + }); + + it('should increment globalVersion on every invalidateQueryCache call', async () => { + const { fastify } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + fastify.statistics.services.periodStat.invalidateQueryCache(['ch1']); + fastify.statistics.services.periodStat.invalidateQueryCache(['ch2']); + + expect(fastify.statistics.services.periodStat.invalidateQueryCache).to.be.a('function'); + + await fastify.close(); + }); + + it('should handle empty channels array in invalidateQueryCache', async () => { + const { fastify } = createMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + fastify.statistics.services.periodStat.invalidateQueryCache([]); + + await fastify.close(); + }); + }); + + describe('compensate 详细逻辑测试', () => { + it('should log error when aggregate fails during compensation', async () => { + const { fastify, watermarkStore, logCalls, mockModel } = createFullMockFastify(); + + watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-27T00:00:00.000Z') }; + mockModel.dataRecord.findAll = async () => { throw new Error('DB read error'); }; + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + expect(hJob).to.exist; + await hJob.onTick(); + + expect(logCalls.error.some(msg => msg.includes('Failed to compensate period h'))).to.be.true; + + await fastify.close(); + }); + + it('should log warning when compensation is incomplete', async () => { + const { fastify, watermarkStore, logCalls, findAllResults } = createFullMockFastify(); + + watermarkStore['h'] = { period: 'h', nextTime: new Date('2020-01-01T00:00:00.000Z') }; + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 1 }); + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + expect(hJob).to.exist; + await hJob.onTick(); + + expect(logCalls.warn.some(msg => msg.includes('补偿未完成'))).to.be.true; + + await fastify.close(); + }); + + it('should log info when compensation completes with multiple windows', async () => { + const { fastify, watermarkStore, logCalls, findAllResults } = createFullMockFastify(); + + const now = new Date(); + const threeHoursAgo = new Date(now.getTime() - 3 * 3600000); + threeHoursAgo.setMinutes(0, 0, 0); + + watermarkStore['h'] = { period: 'h', nextTime: threeHoursAgo }; + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + expect(hJob).to.exist; + await hJob.onTick(); + + expect(logCalls.info.some(msg => msg.includes('补偿完成'))).to.be.true; + + await fastify.close(); + }); + + it('should log error when startup compensation fails', async () => { + const { fastify, logCalls, mockModel, watermarkStore } = createFullMockFastify(); + + const pastTime = new Date('2020-01-01T00:00:00.000Z'); + watermarkStore['h'] = { period: 'h', nextTime: pastTime }; + watermarkStore['d'] = { period: 'd', nextTime: pastTime }; + watermarkStore['w'] = { period: 'w', nextTime: pastTime }; + watermarkStore['m'] = { period: 'm', nextTime: pastTime }; + watermarkStore['q'] = { period: 'q', nextTime: pastTime }; + watermarkStore['y'] = { period: 'y', nextTime: pastTime }; + + // Override findOne to return object with failing update, and create to fail + mockModel.aggregationWatermark.findOne = async ({ where }) => { + const entry = watermarkStore[where.period]; + if (!entry) return null; + return { + nextTime: entry.nextTime, + update: async () => { throw new Error('Watermark write error'); } + }; + }; + mockModel.aggregationWatermark.create = async () => { throw new Error('Watermark write error'); }; + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: true }); + + for (let i = 0; i < 20; i++) { + if (logCalls.error.some(msg => msg.includes('Startup compensation failed'))) break; + await new Promise(resolve => setTimeout(resolve, 100)); + } + + expect(logCalls.error.some(msg => msg.includes('Startup compensation failed'))).to.be.true; + + await fastify.close(); + }); + + it('should trigger upstream compensation when upstream watermark is behind', async () => { + const { fastify, watermarkStore, findAllResults, bulkCreateCalls, periodStatRows, mockModel } = createFullMockFastify(); + + watermarkStore['d'] = { period: 'd', nextTime: new Date('2026-05-26T00:00:00.000Z') }; + watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-26T00:00:00.000Z') }; + + findAllResults.push({ + channel: 'ch1', attributeName: 'val', + sum: 10, avg: null, count: null, min: null, max: null + }); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: new Date('2026-05-26T00:00:00.000Z') } + ); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const dJob = createdJobs.find(j => j.name === 'statistics-period-stat-d'); + expect(dJob).to.exist; + await dJob.onTick(); + + expect(bulkCreateCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should log error via cron onTick when compensate throws outside aggregate', async () => { + const { fastify, watermarkStore, logCalls, mockModel } = createFullMockFastify(); + + const pastTime = new Date('2026-05-27T00:00:00.000Z'); + for (const p of ['h', 'd', 'w', 'm', 'q', 'y']) { + watermarkStore[p] = { period: p, nextTime: pastTime }; + } + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + // After init succeeds, make setWatermark fail + mockModel.aggregationWatermark.findOne = async ({ where }) => { + const entry = watermarkStore[where.period]; + if (!entry) return null; + return { + nextTime: entry.nextTime, + update: async () => { throw new Error('Watermark write error'); } + }; + }; + mockModel.aggregationWatermark.create = async () => { throw new Error('Watermark write error'); }; + + const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); + expect(hJob).to.exist; + await hJob.onTick(); + + expect(logCalls.error.some(msg => msg.includes('Failed to compensate period h'))).to.be.true; + + await fastify.close(); + }); + }); + + describe('w/m/q/y 周期 compensate getNextStart 覆盖测试', () => { + it('should compensate w period using getNextStart (week)', async () => { + const { fastify, watermarkStore, findAllResults, bulkCreateCalls, periodStatRows } = createFullMockFastify(); + + const twoWeeksAgo = new Date(); + twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14); + const dayjs = require('dayjs'); + const wStart = dayjs(twoWeeksAgo).startOf('week').add(1, 'day').startOf('day').toDate(); + + watermarkStore['w'] = { period: 'w', nextTime: wStart }; + watermarkStore['d'] = { period: 'd', nextTime: new Date() }; + + periodStatRows.push( + { period: 'd', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: wStart } + ); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); + + const wJob = createdJobs.find(j => j.name === 'statistics-period-stat-w'); + expect(wJob).to.exist; + await wJob.onTick(); + + expect(bulkCreateCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should compensate m period using getNextStart (month)', async () => { + const { fastify, watermarkStore, bulkCreateCalls, periodStatRows } = createFullMockFastify(); + + const twoMonthsAgo = new Date(); + twoMonthsAgo.setMonth(twoMonthsAgo.getMonth() - 2); + twoMonthsAgo.setDate(1); + twoMonthsAgo.setHours(0, 0, 0, 0); + + watermarkStore['m'] = { period: 'm', nextTime: twoMonthsAgo }; + watermarkStore['d'] = { period: 'd', nextTime: new Date() }; + + periodStatRows.push( + { period: 'd', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: twoMonthsAgo } + ); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); + + const mJob = createdJobs.find(j => j.name === 'statistics-period-stat-m'); + expect(mJob).to.exist; + await mJob.onTick(); + + expect(bulkCreateCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should compensate q period using getNextStart (quarter)', async () => { + const { fastify, watermarkStore, bulkCreateCalls, periodStatRows } = createFullMockFastify(); + + const sixMonthsAgo = new Date(); + sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); + const qMonth = Math.floor(sixMonthsAgo.getMonth() / 3) * 3; + sixMonthsAgo.setMonth(qMonth, 1); + sixMonthsAgo.setHours(0, 0, 0, 0); + + watermarkStore['q'] = { period: 'q', nextTime: sixMonthsAgo }; + watermarkStore['m'] = { period: 'm', nextTime: new Date() }; + + periodStatRows.push( + { period: 'm', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: sixMonthsAgo } + ); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); + + const qJob = createdJobs.find(j => j.name === 'statistics-period-stat-q'); + expect(qJob).to.exist; + await qJob.onTick(); + + expect(bulkCreateCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should compensate y period using getNextStart (year)', async () => { + const { fastify, watermarkStore, bulkCreateCalls, periodStatRows } = createFullMockFastify(); + + const twoYearsAgo = new Date(); + twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2); + twoYearsAgo.setMonth(0, 1); + twoYearsAgo.setHours(0, 0, 0, 0); + + watermarkStore['y'] = { period: 'y', nextTime: twoYearsAgo }; + watermarkStore['q'] = { period: 'q', nextTime: new Date() }; + + periodStatRows.push( + { period: 'q', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: twoYearsAgo } + ); + + const createdJobs = []; + fastify.decorate('cron', { + createJob: (jobConfig) => { createdJobs.push(jobConfig); } + }); + + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); + + const yJob = createdJobs.find(j => j.name === 'statistics-period-stat-y'); + expect(yJob).to.exist; + await yJob.onTick(); + + expect(bulkCreateCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + }); + + describe('invalidateQueryCache 边界测试', () => { + it('should handle invalidateQueryCache with no arguments (default empty array)', async () => { + const { fastify } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + fastify.statistics.services.periodStat.invalidateQueryCache(); + + await fastify.close(); + }); + }); + }); +}); diff --git a/test/period-stat-helpers.js b/test/period-stat-helpers.js new file mode 100644 index 0000000..d0fdbf7 --- /dev/null +++ b/test/period-stat-helpers.js @@ -0,0 +1,424 @@ +const fp = require('fastify-plugin'); + +const mockPeriodStatService = async (fastify, options) => { + const servicePlugin = require('../libs/services/period-stat'); + await fp(servicePlugin)(fastify, options); + await fastify[options.name || 'statistics'].services.periodStat.init(); +}; + +const createMockFastify = () => { + const findAllResults = []; + const bulkCreateCalls = []; + const destroyCalls = []; + const periodStatRows = []; + + const mockTransaction = { + commit: async () => {}, + rollback: async () => {} + }; + + const mockModel = { + dataRecord: { + findAll: async (opts) => findAllResults.splice(0, findAllResults.length), + destroy: async (opts) => { + destroyCalls.push(opts); + }, + findOne: async () => null + }, + periodStat: { + bulkCreate: async (records, opts) => { + bulkCreateCalls.push({ records: [...records], opts: opts || {} }); + return records; + }, + findAll: async () => periodStatRows.splice(0, periodStatRows.length), + findOne: async () => null + }, + aggregationWatermark: { + findOne: async () => null, + upsert: async () => {}, + create: async (data) => data + }, + channelMeta: { + findAll: async () => [] + } + }; + + const fastify = require('fastify')(); + + fastify.decorate('sequelize', { + Sequelize: { Op: { between: 'between', gte: 'gte', lt: 'lt' }, fn: (name, col) => `${name}(${col})`, col: name => name }, + instance: { transaction: async () => mockTransaction } + }); + + fastify.decorate('statistics', { + models: mockModel, + services: {} + }); + + return { fastify, findAllResults, bulkCreateCalls, destroyCalls, periodStatRows, mockModel }; +}; + +const createQueryMockFastify = () => { + const periodStatRows = []; + const dataRecordFindAllResult = []; + const findAllCalls = []; + const channelMetaRows = []; + + const mockTransaction = { + commit: async () => {}, + rollback: async () => {} + }; + + const mockModel = { + dataRecord: { + findAll: async (opts) => { + findAllCalls.push({ model: 'dataRecord', opts }); + return dataRecordFindAllResult.splice(0); + }, + destroy: async () => {}, + findOne: async () => null + }, + periodStat: { + bulkCreate: async () => {}, + findAll: async (opts) => { + findAllCalls.push({ model: 'periodStat', opts }); + const period = opts.where && opts.where.period; + if (period && period.in) { + return periodStatRows.filter(row => period.in.includes(row.period)); + } + return periodStatRows.filter(row => !period || row.period === period); + }, + findOne: async () => null + }, + channelMeta: { + findAll: async ({ where }) => { + if (where && where.channel && where.channel.in) { + return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); + } + return channelMetaRows; + } + }, + aggregationWatermark: { + findOne: async () => null, + upsert: async () => {}, + create: async (data) => data + } + }; + + const fastify = require('fastify')(); + + fastify.decorate('sequelize', { + Sequelize: { + Op: { between: 'between', like: 'like', or: 'or', in: 'in', gte: 'gte', lt: 'lt' }, + fn: (name, col) => `${name}(${col})`, + col: name => name + }, + instance: { transaction: async () => mockTransaction } + }); + + fastify.decorate('statistics', { + models: mockModel, + services: {} + }); + + return { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls, channelMetaRows }; +}; + +const createCacheTestMockFastify = () => { + const periodStatRows = []; + const findAllCalls = []; + const channelMetaRows = []; + + const mockTransaction = { + commit: async () => {}, + rollback: async () => {} + }; + + const mockModel = { + dataRecord: { + findAll: async (opts) => { + findAllCalls.push({ model: 'dataRecord', opts }); + return []; + }, + destroy: async () => {}, + findOne: async () => null + }, + periodStat: { + bulkCreate: async () => {}, + findAll: async (opts) => { + findAllCalls.push({ model: 'periodStat', opts }); + const period = opts.where && opts.where.period; + if (period && period.in) { + return periodStatRows.filter(row => period.in.includes(row.period)); + } + return periodStatRows.filter(row => !period || row.period === period); + }, + findOne: async () => null + }, + channelMeta: { + findAll: async ({ where }) => { + if (where && where.channel && where.channel.in) { + return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); + } + return channelMetaRows; + } + }, + aggregationWatermark: { + findOne: async () => null, + upsert: async () => {}, + create: async (data) => data + } + }; + + const fastify = require('fastify')(); + + fastify.decorate('sequelize', { + Sequelize: { + Op: { between: 'between', like: 'like', or: 'or', in: 'in', gte: 'gte', lt: 'lt' }, + fn: (name, col) => `${name}(${col})`, + col: name => name + }, + instance: { transaction: async () => mockTransaction } + }); + + fastify.decorate('statistics', { + models: mockModel, + services: {} + }); + + return { fastify, periodStatRows, findAllCalls, channelMetaRows }; +}; + +const createExternalCacheMockFastify = () => { + const cacheStore = {}; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { cacheStore[key] = value; } + }; + + const periodStatRows = []; + const findAllCalls = []; + const channelMetaRows = []; + + const mockTransaction = { + commit: async () => {}, + rollback: async () => {} + }; + + const mockModel = { + dataRecord: { + findAll: async (opts) => { + findAllCalls.push({ model: 'dataRecord', opts }); + return []; + }, + destroy: async () => {}, + findOne: async () => null + }, + periodStat: { + bulkCreate: async () => {}, + findAll: async (opts) => { + findAllCalls.push({ model: 'periodStat', opts }); + const period = opts.where && opts.where.period; + if (period && period.in) { + return periodStatRows.filter(row => period.in.includes(row.period)); + } + return periodStatRows; + }, + findOne: async () => null + }, + channelMeta: { + findAll: async ({ where }) => { + if (where && where.channel && where.channel.in) { + return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); + } + return channelMetaRows; + } + }, + aggregationWatermark: { + findOne: async () => null, + upsert: async () => {}, + create: async (data) => data + } + }; + + const fastify = require('fastify')(); + + fastify.decorate('sequelize', { + Sequelize: { + Op: { between: 'between', like: 'like', or: 'or', in: 'in', gte: 'gte', lt: 'lt' }, + fn: (name, col) => `${name}(${col})`, + col: name => name + }, + instance: { transaction: async () => mockTransaction } + }); + + fastify.decorate('statistics', { + models: mockModel, + services: {} + }); + + return { fastify, periodStatRows, findAllCalls, channelMetaRows, cacheStore, externalCache }; +}; + +const createCompensateMockFastify = () => { + const findAllResults = []; + const periodStatRows = []; + const bulkCreateCalls = []; + const watermarkStore = {}; + let destroyCount = 0; + + const mockTransaction = { + commit: async () => {}, + rollback: async () => {} + }; + + const mockModel = { + dataRecord: { + findAll: async () => findAllResults.splice(0), + findOne: async () => null, + destroy: async () => { destroyCount++; return destroyCount; } + }, + periodStat: { + bulkCreate: async (records, opts) => { + bulkCreateCalls.push({ records: [...records], opts: opts || {} }); + return records; + }, + findAll: async (opts) => { + const period = opts.where && opts.where.period; + if (period) { + return periodStatRows.filter(r => r.period === period); + } + return periodStatRows.splice(0); + }, + findOne: async () => null + }, + channelMeta: { + findAll: async () => [] + }, + aggregationWatermark: { + findOne: async ({ where }) => { + const entry = watermarkStore[where.period]; + if (!entry) return null; + return { + ...entry, + update: async (values) => { Object.assign(entry, values); } + }; + }, + upsert: async (data) => { watermarkStore[data.period] = data; }, + create: async (data) => { watermarkStore[data.period] = data; return data; } + } + }; + + const fastify = require('fastify')(); + fastify.decorate('sequelize', { + Sequelize: { Op: { between: 'between', gte: 'gte', lt: 'lt' }, fn: (name, col) => `${name}(${col})`, col: name => name }, + instance: { transaction: async () => mockTransaction } + }); + fastify.decorate('statistics', { models: mockModel, services: {} }); + + return { fastify, findAllResults, periodStatRows, bulkCreateCalls, watermarkStore, mockModel }; +}; + +const createFullMockFastify = () => { + const periodStatRows = []; + const findAllResults = []; + const findAllCalls = []; + const channelMetaRows = []; + const bulkCreateCalls = []; + const watermarkStore = {}; + const logCalls = { error: [], warn: [], info: [] }; + + const mockTransaction = { + commit: async () => {}, + rollback: async () => {} + }; + + const mockModel = { + dataRecord: { + findAll: async (opts) => { + findAllCalls.push({ model: 'dataRecord', opts }); + return [...findAllResults]; + }, + findOne: async () => null, + destroy: async () => {} + }, + periodStat: { + bulkCreate: async (records, opts) => { + bulkCreateCalls.push({ records: [...records], opts: opts || {} }); + return records; + }, + findAll: async (opts) => { + findAllCalls.push({ model: 'periodStat', opts }); + const period = opts.where && opts.where.period; + if (period && typeof period === 'object' && period.in) { + return periodStatRows.filter(row => period.in.includes(row.period)); + } + if (typeof period === 'string') { + return periodStatRows.filter(row => row.period === period); + } + return [...periodStatRows]; + }, + findOne: async () => null + }, + channelMeta: { + findAll: async ({ where }) => { + if (where && where.channel && where.channel.in) { + return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); + } + return channelMetaRows; + } + }, + aggregationWatermark: { + findOne: async ({ where }) => { + const entry = watermarkStore[where.period]; + if (!entry) return null; + return { + ...entry, + update: async (values) => { Object.assign(entry, values); } + }; + }, + upsert: async (data) => { watermarkStore[data.period] = data; }, + create: async (data) => { watermarkStore[data.period] = data; return data; } + } + }; + + const fastify = require('fastify')(); + + ['error', 'warn', 'info'].forEach(level => { + const orig = fastify.log[level]; + fastify.log[level] = function (...args) { + logCalls[level].push(args.join(' ')); + return orig ? orig.apply(this, args) : undefined; + }; + }); + + fastify.decorate('sequelize', { + Sequelize: { + Op: { between: 'between', like: 'like', or: 'or', in: 'in', gte: 'gte', lt: 'lt' }, + fn: (name, col) => `${name}(${col})`, + col: name => name + }, + instance: { transaction: async () => mockTransaction } + }); + + fastify.decorate('statistics', { + models: mockModel, + services: {} + }); + + return { + fastify, periodStatRows, findAllResults, findAllCalls, + channelMetaRows, bulkCreateCalls, watermarkStore, logCalls, mockModel, + mockTransaction + }; +}; + +module.exports = { + mockPeriodStatService, + createMockFastify, + createQueryMockFastify, + createCacheTestMockFastify, + createExternalCacheMockFastify, + createCompensateMockFastify, + createFullMockFastify +}; diff --git a/test/period-stat-init-cascade.test.js b/test/period-stat-init-cascade.test.js new file mode 100644 index 0000000..f27d31d --- /dev/null +++ b/test/period-stat-init-cascade.test.js @@ -0,0 +1,527 @@ +const { expect } = require('chai'); +const { Sequelize, DataTypes, Op } = require('sequelize'); +const fp = require('fastify-plugin'); +const dayjs = require('dayjs'); +const utc = require('dayjs/plugin/utc'); +const timezone = require('dayjs/plugin/timezone'); + +dayjs.extend(utc); +dayjs.extend(timezone); + +/** + * 创建基于 SQLite 内存数据库的测试环境 + */ +const createSqliteTestEnv = async () => { + const sequelize = new Sequelize('sqlite::memory:', { logging: false }); + + const channelMeta = sequelize.define('cascadeTestChannelMeta', { + channel: { type: DataTypes.STRING, allowNull: false }, + title: { type: DataTypes.STRING, allowNull: false }, + description: { type: DataTypes.TEXT } + }, { + underscored: true, + tableName: 'cascade_test_channel_meta', + indexes: [{ name: 'idx_cascade_test_channel_meta_channel', unique: true, fields: ['channel'] }] + }); + + const dataRecord = sequelize.define('cascadeTestDataRecord', { + channel: { type: DataTypes.STRING, allowNull: false }, + attributeName: { type: DataTypes.STRING, defaultValue: 'default' }, + data: { type: DataTypes.DECIMAL(16, 2), allowNull: false, defaultValue: 0 }, + time: { type: DataTypes.DATE, allowNull: false }, + unit: { type: DataTypes.STRING }, + channelMetaId: { type: DataTypes.INTEGER } + }, { + underscored: true, + tableName: 'cascade_test_data_record', + indexes: [ + { name: 'idx_cascade_test_data_record_channel', fields: ['channel'] }, + { name: 'idx_cascade_test_data_record_time', fields: ['time'] }, + { name: 'idx_cascade_test_data_record_channel_attr_time', fields: ['channel', 'attribute_name', 'time'] } + ] + }); + + const periodStat = sequelize.define('cascadeTestPeriodStat', { + period: { type: DataTypes.STRING, allowNull: false }, + time: { type: DataTypes.DATE, allowNull: false }, + channel: { type: DataTypes.STRING, allowNull: false }, + attributeName: { type: DataTypes.STRING, defaultValue: 'default' }, + aggregate: { type: DataTypes.STRING, allowNull: false }, + data: { type: DataTypes.DECIMAL(16, 2), allowNull: false, defaultValue: 0 }, + unit: { type: DataTypes.STRING }, + channelMetaId: { type: DataTypes.INTEGER } + }, { + underscored: true, + tableName: 'cascade_test_period_stat', + indexes: [ + { name: 'idx_cascade_test_period_stat_unique', unique: true, fields: ['period', 'channel', 'attribute_name', 'aggregate', 'time'] }, + { name: 'idx_cascade_test_period_stat_period_time', fields: ['period', 'time'] } + ] + }); + + const aggregationWatermark = sequelize.define('cascadeTestAggregationWatermark', { + period: { type: DataTypes.STRING, allowNull: false }, + nextTime: { type: DataTypes.DATE, allowNull: false } + }, { + underscored: true, + tableName: 'cascade_test_aggregation_watermark', + indexes: [ + { name: 'idx_cascade_test_aggregation_watermark_period', unique: true, fields: ['period'] } + ] + }); + + dataRecord.belongsTo(channelMeta, { foreignKey: 'channelMetaId' }); + periodStat.belongsTo(channelMeta, { foreignKey: 'channelMetaId' }); + + await sequelize.sync({ force: true }); + + const fastify = require('fastify')(); + fastify.decorate('sequelize', { + Sequelize: { Op, fn: Sequelize.fn, col: Sequelize.col }, + instance: sequelize + }); + fastify.decorate('statistics', { + models: { dataRecord, periodStat, aggregationWatermark, channelMeta }, + services: {} + }); + + return { fastify, sequelize, models: { dataRecord, periodStat, aggregationWatermark, channelMeta } }; +}; + +const loadPeriodStatService = async (fastify, options = {}) => { + const servicePlugin = require('../libs/services/period-stat'); + await fp(servicePlugin)(fastify, { + name: 'statistics', + ...options + }); +}; + +const isSameTime = (a, b) => new Date(a).getTime() === new Date(b).getTime(); + +/** + * 辅助：将本地时间日期字符串转为 dayjs 在当前时区下的 Date 对象 + * 因为 period-stat 服务使用本地时区计算 truncateTime + */ +const localDate = (str) => dayjs(str).toDate(); + +describe('@kne/fastify-statistics', function () { + this.timeout(60000); + + describe('init 级联聚合集成测试', () => { + describe('跨年数据：init 从 data-record 自动级联聚合所有周期', () => { + it('应在 init 时从 data-record 自动完成 h→d→w→m→q→y 的全链路聚合', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify, { compensationEnabled: true }); + + // ===== 构造跨年数据（使用本地时区时间） ===== + // dayjs 默认用本地时区，我们用本地时区的时间字符串 + // 数据跨越 2025-Q3 ~ 2026-Q2，覆盖两个年份 + + const records = []; + + // --- 2025-09-15: Q3 --- + records.push( + { channel: 'task', attributeName: 'total', data: 10, time: localDate('2025-09-15T08:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 5, time: localDate('2025-09-15T09:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 8, time: localDate('2025-09-15T08:30:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 4, time: localDate('2025-09-15T09:30:00'), unit: 'count' } + ); + + // --- 2025-12-10: Q4 --- + records.push( + { channel: 'task', attributeName: 'total', data: 20, time: localDate('2025-12-10T10:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 15, time: localDate('2025-12-10T11:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 18, time: localDate('2025-12-10T10:30:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 12, time: localDate('2025-12-10T11:30:00'), unit: 'count' } + ); + + // --- 2026-01-05: Q1 --- + records.push( + { channel: 'task', attributeName: 'total', data: 30, time: localDate('2026-01-05T14:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 25, time: localDate('2026-01-05T15:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 28, time: localDate('2026-01-05T14:30:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 22, time: localDate('2026-01-05T15:30:00'), unit: 'count' } + ); + + // --- 2026-03-20: Q1 另一天 --- + records.push( + { channel: 'task', attributeName: 'total', data: 40, time: localDate('2026-03-20T16:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 35, time: localDate('2026-03-20T17:00:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 38, time: localDate('2026-03-20T16:30:00'), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 30, time: localDate('2026-03-20T17:30:00'), unit: 'count' } + ); + + // --- 昨天 (Q2) --- + const yesterday = dayjs().subtract(1, 'day').format('YYYY-MM-DD'); + records.push( + { channel: 'task', attributeName: 'total', data: 50, time: localDate(`${yesterday}T08:00:00`), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 45, time: localDate(`${yesterday}T09:00:00`), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 48, time: localDate(`${yesterday}T08:30:00`), unit: 'count' }, + { channel: 'task', attributeName: 'success', data: 40, time: localDate(`${yesterday}T09:30:00`), unit: 'count' } + ); + + await models.dataRecord.bulkCreate(records); + + // 验证初始数据量 + expect(await models.dataRecord.count()).to.equal(20); + expect(await models.periodStat.count()).to.equal(0); + expect(await models.aggregationWatermark.count()).to.equal(0); + + // ===== 执行 init ===== + await fastify.statistics.services.periodStat.init(); + + // ===== 验证 h 周期聚合 ===== + const hRecords = await models.periodStat.findAll({ where: { period: 'h' }, raw: true }); + const hTimes = [...new Set(hRecords.map(r => new Date(r.time).toISOString()))].sort(); + + // 5 个不同日期，每个有不同小时 + // 注意：init 会从最早数据的小时开始逐小时补偿到当前小时 + // 有数据的小时是 5 个 + expect(hTimes.length).to.be.at.least(5); + + // 验证具体小时数据 + // 2025-09-15 08:00 h: total sum=10, success sum=8 + const sep15_08 = dayjs('2025-09-15T08:00:00').startOf('hour').toDate(); + const h08TotalSum = hRecords.find( + r => isSameTime(r.time, sep15_08) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(h08TotalSum).to.exist; + expect(parseFloat(h08TotalSum.data)).to.equal(10); + + // data-record 应已被删除（h 聚合后删除源数据） + expect(await models.dataRecord.count()).to.equal(0); + + // ===== 验证 d 周期聚合 ===== + const dRecords = await models.periodStat.findAll({ where: { period: 'd' }, raw: true }); + const dTimes = [...new Set(dRecords.map(r => new Date(r.time).toISOString()))].sort(); + + // 至少 5 天的数据 + expect(dTimes.length).to.be.at.least(5); + + // 2025-09-15 d: total sum=10+5=15, success sum=8+4=12 + const sep15 = dayjs('2025-09-15').startOf('day').toDate(); + const d0915TotalSum = dRecords.find( + r => isSameTime(r.time, sep15) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(d0915TotalSum).to.exist; + expect(parseFloat(d0915TotalSum.data)).to.equal(15); + + // 昨天 d: total sum=50+45=95, success sum=48+40=88 + const yesterdayStart = dayjs().subtract(1, 'day').startOf('day').toDate(); + const dYesterdayTotalSum = dRecords.find( + r => isSameTime(r.time, yesterdayStart) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(dYesterdayTotalSum).to.exist; + expect(parseFloat(dYesterdayTotalSum.data)).to.equal(95); + + // ===== 验证 w 周期聚合 ===== + const wRecords = await models.periodStat.findAll({ where: { period: 'w' }, raw: true }); + expect(wRecords.length).to.be.at.least(3); // 至少 3 个不同的周 + + // ===== 验证 m 周期聚合 ===== + const mRecords = await models.periodStat.findAll({ where: { period: 'm' }, raw: true }); + const mTimes = [...new Set(mRecords.map(r => new Date(r.time).toISOString()))].sort(); + + // init 会补偿到当前月，已完成的月至少有：2025-09, 2025-12, 2026-01, 2026-03 + // 当前月 2026-05 的 m 不会被聚合（月份未完成） + expect(mTimes.length).to.be.at.least(4); + + // 2025-09 m: total sum=15, success sum=12 + const sep1 = dayjs('2025-09-01').startOf('month').toDate(); + const m09TotalSum = mRecords.find( + r => isSameTime(r.time, sep1) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(m09TotalSum).to.exist; + expect(parseFloat(m09TotalSum.data)).to.equal(15); + + // ===== 验证 q 周期聚合 ===== + const qRecords = await models.periodStat.findAll({ where: { period: 'q' }, raw: true }); + const qTimes = [...new Set(qRecords.map(r => new Date(r.time).toISOString()))].sort(); + + // 已完成的季度：2025-Q3, 2025-Q4, 2026-Q1 + // 当前季度 2026-Q2 未完成，不会被聚合 + expect(qTimes.length).to.be.at.least(3); + + // 2025-Q3: total sum=15, success sum=12 + const q3Start = dayjs('2025-07-01').startOf('month').toDate(); + const q3TotalSum = qRecords.find( + r => isSameTime(r.time, q3Start) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(q3TotalSum).to.exist; + expect(parseFloat(q3TotalSum.data)).to.equal(15); + + // 2025-Q4: total sum=20+15=35, success sum=18+12=30 + const q4Start = dayjs('2025-10-01').startOf('month').toDate(); + const q4TotalSum = qRecords.find( + r => isSameTime(r.time, q4Start) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(q4TotalSum).to.exist; + expect(parseFloat(q4TotalSum.data)).to.equal(35); + + // 2026-Q1: total sum=30+25+40+35=130, success sum=28+22+38+30=118 + const q1Start = dayjs('2026-01-01').startOf('month').toDate(); + const q1TotalSum = qRecords.find( + r => isSameTime(r.time, q1Start) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(q1TotalSum).to.exist; + expect(parseFloat(q1TotalSum.data)).to.equal(130); + + // ===== 验证 y 周期聚合 ===== + const yRecords = await models.periodStat.findAll({ where: { period: 'y' }, raw: true }); + const yTimes = [...new Set(yRecords.map(r => new Date(r.time).toISOString()))].sort(); + + // 已完成的年：2025 + // 当前年 2026 未完成，不会被聚合 + expect(yTimes.length).to.be.at.least(1); + + // 2025: total sum=15+35=50, success sum=12+30=42 + const y2025Start = dayjs('2025-01-01').startOf('year').toDate(); + const y2025TotalSum = yRecords.find( + r => isSameTime(r.time, y2025Start) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(y2025TotalSum).to.exist; + expect(parseFloat(y2025TotalSum.data)).to.equal(50); + + const y2025SuccessSum = yRecords.find( + r => isSameTime(r.time, y2025Start) && r.attributeName === 'success' && r.aggregate === 'sum' + ); + expect(y2025SuccessSum).to.exist; + expect(parseFloat(y2025SuccessSum.data)).to.equal(42); + + // ===== 验证水位线 ===== + const hWm = await models.aggregationWatermark.findOne({ where: { period: 'h' }, raw: true }); + const dWm = await models.aggregationWatermark.findOne({ where: { period: 'd' }, raw: true }); + const mWm = await models.aggregationWatermark.findOne({ where: { period: 'm' }, raw: true }); + const qWm = await models.aggregationWatermark.findOne({ where: { period: 'q' }, raw: true }); + const yWm = await models.aggregationWatermark.findOne({ where: { period: 'y' }, raw: true }); + + // h 水位线应推进到当前小时 + const nowTruncatedH = dayjs().startOf('hour').toDate(); + expect(new Date(hWm.nextTime).getTime()).to.equal(nowTruncatedH.getTime()); + + // d 水位线应推进到今天 + const nowTruncatedD = dayjs().startOf('day').toDate(); + expect(new Date(dWm.nextTime).getTime()).to.equal(nowTruncatedD.getTime()); + + // m 水位线应推进到当前月 + const nowTruncatedM = dayjs().startOf('month').toDate(); + expect(new Date(mWm.nextTime).getTime()).to.equal(nowTruncatedM.getTime()); + + // q 水位线应推进到当前季度 + const nowQ = dayjs(); + const nowTruncatedQ = nowQ.month(Math.floor(nowQ.month() / 3) * 3).startOf('month').toDate(); + expect(new Date(qWm.nextTime).getTime()).to.equal(nowTruncatedQ.getTime()); + + // y 水位线应推进到当前年 + const nowTruncatedY = dayjs().startOf('year').toDate(); + expect(new Date(yWm.nextTime).getTime()).to.equal(nowTruncatedY.getTime()); + + await fastify.close(); + }); + + it('应正确处理跨月多天的 d→m→q→y 聚合', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify, { compensationEnabled: true }); + + // 构造 2025 年的完整数据（已完成的年、季、月） + // 2025-04 和 2025-05 (都在 Q2) + const records = []; + + // 2025-04-10 (Q2) + records.push( + { channel: 'order', attributeName: 'amount', data: 100, time: localDate('2025-04-10T10:00:00'), unit: 'yuan' }, + { channel: 'order', attributeName: 'amount', data: 200, time: localDate('2025-04-10T14:00:00'), unit: 'yuan' } + ); + + // 2025-04-15 (Q2) + records.push( + { channel: 'order', attributeName: 'amount', data: 150, time: localDate('2025-04-15T09:00:00'), unit: 'yuan' }, + { channel: 'order', attributeName: 'amount', data: 250, time: localDate('2025-04-15T16:00:00'), unit: 'yuan' } + ); + + // 2025-05-01 (Q2) + records.push( + { channel: 'order', attributeName: 'amount', data: 400, time: localDate('2025-05-01T08:00:00'), unit: 'yuan' }, + { channel: 'order', attributeName: 'amount', data: 500, time: localDate('2025-05-01T12:00:00'), unit: 'yuan' } + ); + + // 2025-07-15 (Q3, 不同季度) + records.push( + { channel: 'order', attributeName: 'amount', data: 300, time: localDate('2025-07-15T11:00:00'), unit: 'yuan' } + ); + + await models.dataRecord.bulkCreate(records); + + await fastify.statistics.services.periodStat.init(); + + // ===== 验证 d 聚合 ===== + const dRecords = await models.periodStat.findAll({ where: { period: 'd', attributeName: 'amount' }, raw: true }); + const dTimes = [...new Set(dRecords.map(r => new Date(r.time).toISOString()))].sort(); + expect(dTimes.length).to.be.at.least(4); // 4 天 + + // 2025-04-10: sum=100+200=300 + const d0410 = dayjs('2025-04-10').startOf('day').toDate(); + const d0410Sum = dRecords.find(r => isSameTime(r.time, d0410) && r.aggregate === 'sum'); + expect(d0410Sum).to.exist; + expect(parseFloat(d0410Sum.data)).to.equal(300); + + // 2025-05-01: sum=400+500=900 + const d0501 = dayjs('2025-05-01').startOf('day').toDate(); + const d0501Sum = dRecords.find(r => isSameTime(r.time, d0501) && r.aggregate === 'sum'); + expect(d0501Sum).to.exist; + expect(parseFloat(d0501Sum.data)).to.equal(900); + + // ===== 验证 m 聚合 ===== + const mRecords = await models.periodStat.findAll({ where: { period: 'm', attributeName: 'amount' }, raw: true }); + + // 2025-04: sum=100+200+150+250=700 + const mApr = dayjs('2025-04-01').startOf('month').toDate(); + const mAprSum = mRecords.find(r => isSameTime(r.time, mApr) && r.aggregate === 'sum'); + expect(mAprSum).to.exist; + expect(parseFloat(mAprSum.data)).to.equal(700); + + // 2025-05: sum=400+500=900 + const mMay = dayjs('2025-05-01').startOf('month').toDate(); + const mMaySum = mRecords.find(r => isSameTime(r.time, mMay) && r.aggregate === 'sum'); + expect(mMaySum).to.exist; + expect(parseFloat(mMaySum.data)).to.equal(900); + + // 2025-07: sum=300 + const mJul = dayjs('2025-07-01').startOf('month').toDate(); + const mJulSum = mRecords.find(r => isSameTime(r.time, mJul) && r.aggregate === 'sum'); + expect(mJulSum).to.exist; + expect(parseFloat(mJulSum.data)).to.equal(300); + + // ===== 验证 q 聚合 ===== + const qRecords = await models.periodStat.findAll({ where: { period: 'q', attributeName: 'amount' }, raw: true }); + + // 2025-Q2 (04-01): sum=700+900=1600 + const qQ2 = dayjs('2025-04-01').startOf('month').toDate(); + const qQ2Sum = qRecords.find(r => isSameTime(r.time, qQ2) && r.aggregate === 'sum'); + expect(qQ2Sum).to.exist; + expect(parseFloat(qQ2Sum.data)).to.equal(1600); + + // 2025-Q3 (07-01): sum=300 + const qQ3 = dayjs('2025-07-01').startOf('month').toDate(); + const qQ3Sum = qRecords.find(r => isSameTime(r.time, qQ3) && r.aggregate === 'sum'); + expect(qQ3Sum).to.exist; + expect(parseFloat(qQ3Sum.data)).to.equal(300); + + // ===== 验证 y 聚合 ===== + const yRecords = await models.periodStat.findAll({ where: { period: 'y', attributeName: 'amount' }, raw: true }); + + // 2025: sum=1600+300=1900 + const y2025 = dayjs('2025-01-01').startOf('year').toDate(); + const y2025Sum = yRecords.find(r => isSameTime(r.time, y2025) && r.aggregate === 'sum'); + expect(y2025Sum).to.exist; + expect(parseFloat(y2025Sum.data)).to.equal(1900); + + await fastify.close(); + }); + + it('应正确聚合多通道层级数据', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify, { compensationEnabled: true }); + + // 使用历史日期确保聚合完成 + const records = []; + const yesterday = dayjs().subtract(1, 'day').format('YYYY-MM-DD'); + + records.push( + { channel: 'record-result', attributeName: 'total', data: 5, time: localDate(`${yesterday}T08:00:00`), unit: 'count' }, + { channel: 'record-result:system', attributeName: 'total', data: 3, time: localDate(`${yesterday}T08:00:00`), unit: 'count' }, + { channel: 'record-result:system:7', attributeName: 'total', data: 1, time: localDate(`${yesterday}T08:00:00`), unit: 'count' } + ); + + await models.dataRecord.bulkCreate(records); + + await fastify.statistics.services.periodStat.init(); + + // 验证 d 聚合 - 每个通道各自聚合 + const dRecords = await models.periodStat.findAll({ where: { period: 'd', aggregate: 'sum' }, raw: true }); + + const dRoot = dRecords.find(r => r.channel === 'record-result' && r.attributeName === 'total'); + expect(dRoot).to.exist; + expect(parseFloat(dRoot.data)).to.equal(5); + + const dSystem = dRecords.find(r => r.channel === 'record-result:system' && r.attributeName === 'total'); + expect(dSystem).to.exist; + expect(parseFloat(dSystem.data)).to.equal(3); + + const dSystem7 = dRecords.find(r => r.channel === 'record-result:system:7' && r.attributeName === 'total'); + expect(dSystem7).to.exist; + expect(parseFloat(dSystem7.data)).to.equal(1); + + await fastify.close(); + }); + + it('应在 init 后再次 init 时不重复聚合已有数据', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify, { compensationEnabled: true }); + + const yesterday = dayjs().subtract(1, 'day').format('YYYY-MM-DD'); + await models.dataRecord.bulkCreate([ + { channel: 'task', attributeName: 'total', data: 10, time: localDate(`${yesterday}T08:00:00`), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 20, time: localDate(`${yesterday}T09:00:00`), unit: 'count' } + ]); + + // 第一次 init + await fastify.statistics.services.periodStat.init(); + + const hCount1 = await models.periodStat.count({ where: { period: 'h' } }); + const dCount1 = await models.periodStat.count({ where: { period: 'd' } }); + + // 第二次 init - 水位线已是最新的，不应重复聚合 + await fastify.statistics.services.periodStat.init(); + + const hCount2 = await models.periodStat.count({ where: { period: 'h' } }); + const dCount2 = await models.periodStat.count({ where: { period: 'd' } }); + + expect(hCount2).to.equal(hCount1); + expect(dCount2).to.equal(dCount1); + + await fastify.close(); + }); + + it('应正确处理昨天数据生成 d 级别聚合（用户报告的 bug）', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify, { compensationEnabled: true }); + + // 模拟用户报告的场景：data-record 中有今天和昨天的数据 + // init 后应该有昨天的 d 级别聚合 + const yesterday = dayjs().subtract(1, 'day'); + const yesterdayStr = yesterday.format('YYYY-MM-DD'); + const todayStr = dayjs().format('YYYY-MM-DD'); + + await models.dataRecord.bulkCreate([ + // 昨天的数据 + { channel: 'task', attributeName: 'total', data: 10, time: localDate(`${yesterdayStr}T08:00:00`), unit: 'count' }, + { channel: 'task', attributeName: 'total', data: 20, time: localDate(`${yesterdayStr}T14:00:00`), unit: 'count' }, + // 今天的数据 + { channel: 'task', attributeName: 'total', data: 30, time: localDate(`${todayStr}T08:00:00`), unit: 'count' } + ]); + + await fastify.statistics.services.periodStat.init(); + + // 验证昨天的 h 聚合存在 + const hRecords = await models.periodStat.findAll({ where: { period: 'h' }, raw: true }); + const yesterdayH = hRecords.filter(r => { + const t = new Date(r.time); + return dayjs(t).format('YYYY-MM-DD') === yesterdayStr; + }); + expect(yesterdayH.length).to.be.at.least(2); // 至少 2 个小时窗口 + + // 关键验证：昨天的 d 聚合必须存在！ + const dRecords = await models.periodStat.findAll({ where: { period: 'd' }, raw: true }); + const yesterdayDStart = yesterday.startOf('day').toDate(); + const yesterdayDSum = dRecords.find( + r => isSameTime(r.time, yesterdayDStart) && r.attributeName === 'total' && r.aggregate === 'sum' + ); + expect(yesterdayDSum).to.exist; + expect(parseFloat(yesterdayDSum.data)).to.equal(30); // 10 + 20 + + await fastify.close(); + }); + }); + }); +}); diff --git a/test/period-stat-query.test.js b/test/period-stat-query.test.js new file mode 100644 index 0000000..52d1c79 --- /dev/null +++ b/test/period-stat-query.test.js @@ -0,0 +1,1435 @@ +const { expect } = require('chai'); +const { + mockPeriodStatService, createQueryMockFastify, createCacheTestMockFastify, + createExternalCacheMockFastify, createFullMockFastify +} = require('./period-stat-helpers'); + +describe('@kne/fastify-statistics', function () { + describe('query 查询测试', () => { + describe('query 方法', () => { + const { createQueryMockFastify: localCreateQueryMockFastify } = { createQueryMockFastify }; + + it('should return attribute-keyed object when single aggregate and single default attribute with no filter', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push({ + period: 'h', channel: 'sensor', attributeName: 'default', + aggregate: 'sum', data: 100, time: startTime + }); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].channel).to.equal('sensor'); + expect(results[0].period).to.equal('h'); + expect(results[0].data).to.deep.equal({ default: 100 }); + + await fastify.close(); + }); + + it('should return attribute-keyed object when single aggregate and multiple attributes', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ temperature: 100, humidity: 200 }); + + await fastify.close(); + }); + + it('should return nested object when multiple aggregates and single default attribute', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'avg', data: 25, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ sum: { default: 100 }, avg: { default: 25 } }); + + await fastify.close(); + }); + + it('should return nested object when multiple aggregates and multiple attributes', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'avg', data: 25, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'avg', data: 50, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ + sum: { temperature: 100, humidity: 200 }, + avg: { temperature: 25, humidity: 50 } + }); + + await fastify.close(); + }); + + it('should not flatten default attribute when attributeNames filter is provided', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, attributeNames: ['default'], aggregates: ['sum'] + }); + + expect(results[0].data).to.deep.equal({ default: 100 }); + + await fastify.close(); + }); + + it('should query all child channels', async () => { + const { fastify, periodStatRows, channelMetaRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor:room1', attributeName: 'default', aggregate: 'sum', data: 50, time: startTime }, + { period: 'h', channel: 'sensor:room2', attributeName: 'default', aggregate: 'sum', data: 30, time: startTime } + ); + channelMetaRows.push({ channel: 'sensor', title: '传感器', description: '温度' }); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true + }); + + expect(results.length).to.equal(1); + const root = results[0]; + expect(root.channel).to.equal('sensor'); + expect(root.items[0].data).to.deep.equal({ default: 100 }); + expect(root.children.length).to.equal(2); + const childChannels = root.children.map(c => c.channel); + expect(childChannels).to.include('sensor:room1'); + expect(childChannels).to.include('sensor:room2'); + const room1 = root.children.find(c => c.channel === 'sensor:room1'); + expect(room1.items[0].data).to.deep.equal({ default: 50 }); + + expect(channelMetas).to.have.property('sensor'); + expect(channelMetas.sensor.title).to.equal('传感器'); + expect(Object.keys(channelMetas).length).to.equal(1); + + await fastify.close(); + }); + + it('should query data from all period types in single query', async () => { + const { fastify, periodStatRows, findAllCalls } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 10, time: startTime }, + { period: 'd', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 240, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const periodTypes = results.map(r => r.period); + expect(periodTypes).to.include('h'); + expect(periodTypes).to.include('d'); + + const psCalls = findAllCalls.filter(c => c.model === 'periodStat'); + expect(psCalls.length).to.equal(1); + expect(psCalls[0].opts.where.period).to.deep.equal({ in: ['h', 'd', 'w', 'm', 'q', 'y'] }); + + await fastify.close(); + }); + + it('should return empty array when no data found', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['nonexistent'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results).to.deep.equal([]); + expect(channelMetas).to.deep.equal({}); + + await fastify.close(); + }); + + it('should filter by attributeNames when provided', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, attributeNames: ['temperature'], aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ temperature: 100 }); + + await fastify.close(); + }); + + it('should query data-record for current hour and format correctly', async () => { + const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + dataRecordFindAllResult.push({ + channel: 'sensor', attributeName: 'temperature', + sum: 50, avg: 25, count: 2, min: 10, max: 40 + }); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const drCalls = findAllCalls.filter(c => c.model === 'dataRecord'); + expect(drCalls.length).to.be.greaterThan(0); + + const hourResult = results.find(r => r.period === 'h' && r.channel === 'sensor'); + if (hourResult) { + expect(hourResult.data).to.deep.equal({ temperature: 50 }); + } + + await fastify.close(); + }); + + it('should return all aggregates when aggregates not specified', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'avg', data: 25, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'count', data: 4, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'min', data: 10, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'max', data: 40, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ sum: { default: 100 }, avg: { default: 25 }, count: { default: 4 }, min: { default: 10 }, max: { default: 40 } }); + + await fastify.close(); + }); + + it('should sort results by time', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T03:00:00.000Z'); + + const time2 = new Date('2026-05-01T02:00:00.000Z'); + const time0 = new Date('2026-05-01T00:00:00.000Z'); + const time1 = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 30, time: time2 }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 10, time: time0 }, + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 20, time: time1 } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results[0].data).to.deep.equal({ default: 10 }); + expect(results[1].data).to.deep.equal({ default: 20 }); + expect(results[2].data).to.deep.equal({ default: 30 }); + + await fastify.close(); + }); + + it('should use client timezone to determine current hour when timezone is provided', async () => { + const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + dataRecordFindAllResult.push({ + channel: 'sensor', attributeName: 'temperature', + sum: 50, avg: null, count: null, min: null, max: null + }); + + const resultsNoTz = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const resultsWithTz = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], timezone: 'Asia/Shanghai' + }); + + const drCallsNoTz = findAllCalls.filter(c => c.model === 'dataRecord'); + expect(drCallsNoTz.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should calculate different current hour boundaries for different timezones', async () => { + const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 7200000); + const endTime = new Date(now.getTime() + 3600000); + + dataRecordFindAllResult.push({ + channel: 'sensor', attributeName: 'default', + sum: 100, avg: null, count: null, min: null, max: null + }); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], timezone: 'America/New_York' + }); + + const drCalls = findAllCalls.filter(c => c.model === 'dataRecord'); + expect(drCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should throw error for invalid timezone', async () => { + const { fastify } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + try { + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], timezone: 'Invalid/Timezone' + }); + expect.fail('should have thrown'); + } catch (e) { + expect(e.message).to.include('Invalid timezone'); + } + + await fastify.close(); + }); + + it('should query without channel filter and return channelMetas', async () => { + const { fastify, periodStatRows, channelMetaRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor1', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor2', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } + ); + channelMetaRows.push( + { channel: 'sensor1', title: '传感器1', description: null }, + { channel: 'sensor2', title: '传感器2', description: '温度' } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(2); + expect(Object.keys(channelMetas).length).to.equal(2); + expect(channelMetas.sensor1.title).to.equal('传感器1'); + expect(channelMetas.sensor2.title).to.equal('传感器2'); + + await fastify.close(); + }); + + it('should handle null attributeName with single aggregate and no filter', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ default: 100 }); + + await fastify.close(); + }); + + it('should handle null attributeName with single aggregate and attributeNames filter', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, attributeNames: ['value'], aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ default: 100 }); + + await fastify.close(); + }); + + it('should handle null attributeName with multiple aggregates and attributeNames filter', async () => { + const { fastify, periodStatRows } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'avg', data: 25, time: startTime } + ); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, attributeNames: ['value'], aggregates: ['sum', 'avg'] + }); + + expect(results.length).to.equal(1); + expect(results[0].data).to.deep.equal({ + sum: { default: 100 }, + avg: { default: 25 } + }); + + await fastify.close(); + }); + + it('should skip undefined aggregate values from data-record in query', async () => { + const { fastify, dataRecordFindAllResult } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + dataRecordFindAllResult.push({ + channel: 'sensor', attributeName: 'temperature', + sum: 50, avg: undefined, count: undefined, min: undefined, max: undefined + }); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] + }); + + const hourResult = results.find(r => r.period === 'h'); + if (hourResult) { + expect(hourResult.data).to.deep.equal({ temperature: 50 }); + } + + await fastify.close(); + }); + + it('should use startTime as drStartTime when startTime is after currentHourStart', async () => { + const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const now = new Date(); + const currentHourStart = new Date(now); + currentHourStart.setMinutes(0, 0, 0); + const startTime = new Date(currentHourStart.getTime() + 30 * 60 * 1000); + const endTime = new Date(currentHourStart.getTime() + 60 * 60 * 1000); + + dataRecordFindAllResult.push({ + channel: 'sensor', attributeName: 'default', + sum: 100, avg: null, count: null, min: null, max: null + }); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should handle null attributeName in data-record query results', async () => { + const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = localCreateQueryMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + dataRecordFindAllResult.push({ + channel: 'sensor', attributeName: null, + sum: 100, avg: null, count: null, min: null, max: null + }); + + const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const drCalls = findAllCalls.filter(c => c.model === 'dataRecord'); + expect(drCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + }); + + describe('查询缓存测试（内存模式）', () => { + it('should cache query result and return from cache on second call', async () => { + const { fastify, periodStatRows, findAllCalls } = createCacheTestMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const result1 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + expect(result1.list.length).to.be.greaterThan(0); + + const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; + + const result2 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterSecond = findAllCalls.filter(c => c.model === 'periodStat').length; + expect(callsAfterSecond).to.equal(callsAfterFirst); + expect(result2).to.deep.equal(result1); + + await fastify.close(); + }); + + it('should invalidate cache when invalidateQueryCache is called', async () => { + const { fastify, periodStatRows, findAllCalls } = createCacheTestMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics' }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.length; + + fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'avg', data: 50, time: startTime } + ); + + const result = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] + }); + expect(findAllCalls.length).to.be.greaterThan(callsAfterFirst); + + await fastify.close(); + }); + + it('should not cache when queryCacheEnabled is false', async () => { + const { fastify, periodStatRows, findAllCalls } = createCacheTestMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', queryCacheEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.length; + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.length).to.be.greaterThan(callsAfterFirst); + + await fastify.close(); + }); + + it('should evict oldest entry when memory cache exceeds maxEntries', async () => { + const { fastify, periodStatRows } = createCacheTestMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', queryCacheMaxEntries: 2 }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'default', aggregate: 'sum', data: 10, time: startTime }, + { period: 'h', channel: 'ch2', attributeName: 'default', aggregate: 'sum', data: 20, time: startTime }, + { period: 'h', channel: 'ch3', attributeName: 'default', aggregate: 'sum', data: 30, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ channels: ['ch1'], startTime, endTime, aggregates: ['sum'] }); + await fastify.statistics.services.periodStat.query({ channels: ['ch2'], startTime, endTime, aggregates: ['sum'] }); + await fastify.statistics.services.periodStat.query({ channels: ['ch3'], startTime, endTime, aggregates: ['sum'] }); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'default', aggregate: 'sum', data: 11, time: startTime } + ); + + const result = await fastify.statistics.services.periodStat.query({ channels: ['ch1'], startTime, endTime, aggregates: ['sum'] }); + expect(result.list[0].data.default).to.equal(11); + + await fastify.close(); + }); + + it('should use historyTTL for non-realtime queries', async () => { + const { fastify, periodStatRows } = createCacheTestMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', queryCacheTTL: 1, queryCacheHistoryTTL: 3600 }); + + const startTime = new Date('2020-05-01T00:00:00.000Z'); + const endTime = new Date('2020-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + await new Promise(resolve => setTimeout(resolve, 1100)); + + const result = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + expect(result.list[0].data.default).to.equal(100); + + await fastify.close(); + }); + + it('should not cache when isCompensating is true', async () => { + const { fastify, periodStatRows } = createCacheTestMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(fastify.statistics.services.periodStat.isCompensating()).to.be.false; + + await fastify.close(); + }); + }); + + describe('查询缓存测试（外部缓存模式）', () => { + it('should cache query result in external cache and return from cache on second call', async () => { + const { fastify, periodStatRows, externalCache } = createExternalCacheMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const result1 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + expect(result1.list.length).to.be.greaterThan(0); + + fastify.statistics.models.periodStat.findAll = async () => []; + + const result2 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(result2.list.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should return null from external cache when payload is invalid', async () => { + const cacheStore = {}; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { cacheStore[key] = value; } + }; + + const periodStatRows = []; + const findAllCalls = []; + const mockTransaction = { commit: async () => {}, rollback: async () => {} }; + const mockModel = { + dataRecord: { findAll: async (opts) => { findAllCalls.push({ model: 'dataRecord', opts }); return []; }, findOne: async () => null, destroy: async () => {} }, + periodStat: { bulkCreate: async () => {}, findAll: async (opts) => { findAllCalls.push({ model: 'periodStat', opts }); return periodStatRows.splice(0); }, findOne: async () => null }, + channelMeta: { findAll: async () => [] }, + aggregationWatermark: { findOne: async () => null, upsert: async () => {}, create: async (d) => d } + }; + + const fastify = require('fastify')(); + fastify.decorate('sequelize', { + Sequelize: { Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, fn: (n, c) => `${n}(${c})`, col: n => n }, + instance: { transaction: async () => mockTransaction } + }); + fastify.decorate('statistics', { models: mockModel, services: {} }); + + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + const cacheKey = 'statistics:query:' + JSON.stringify({ channels: ['sensor'], startTime: startTime.toISOString(), endTime: endTime.toISOString(), attributeNames: [], aggregates: ['sum'], timezone: '', includeChildren: false }); + cacheStore[cacheKey] = 'not-an-object'; + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const result = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should invalidate external cache when channel version changes', async () => { + const { fastify, periodStatRows, cacheStore, externalCache } = createExternalCacheMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } + ); + + const result = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + expect(result.list[0].data.default).to.equal(200); + + await fastify.close(); + }); + + it('should handle external cache with 3-argument set (TTL support)', async () => { + const cacheStore = {}; + const setCalls = []; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { + setCalls.push({ key, value, ttl }); + cacheStore[key] = value; + } + }; + + const periodStatRows = []; + const mockTransaction = { commit: async () => {}, rollback: async () => {} }; + const mockModel = { + dataRecord: { findAll: async () => [], findOne: async () => null, destroy: async () => {} }, + periodStat: { bulkCreate: async () => {}, findAll: async () => periodStatRows.splice(0), findOne: async () => null }, + channelMeta: { findAll: async () => [] }, + aggregationWatermark: { findOne: async () => null, upsert: async () => {}, create: async (d) => d } + }; + + const fastify = require('fastify')(); + fastify.decorate('sequelize', { + Sequelize: { Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, fn: (n, c) => `${n}(${c})`, col: n => n }, + instance: { transaction: async () => mockTransaction } + }); + fastify.decorate('statistics', { models: mockModel, services: {} }); + + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(setCalls.length).to.be.greaterThan(0); + const queryCacheSet = setCalls.find(c => c.key.includes('statistics:query:')); + expect(queryCacheSet).to.exist; + expect(queryCacheSet.ttl).to.exist; + + await fastify.close(); + }); + }); + + describe('queryCache 版本失效测试（内存模式）', () => { + it('should invalidate memory cache when globalVersion changes', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; + + fastify.statistics.services.periodStat.invalidateQueryCache([]); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); + + await fastify.close(); + }); + + it('should invalidate memory cache when channelVersion changes', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; + + fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); + + await fastify.close(); + }); + + it('should not return expired memory cache entries', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, queryCacheTTL: 1 }); + + const now = new Date(); + const rtStart = new Date(now.getTime() - 3600000); + const rtEnd = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: rtStart } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime: rtStart, endTime: rtEnd, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; + + await new Promise(resolve => setTimeout(resolve, 1100)); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: rtStart } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime: rtStart, endTime: rtEnd, aggregates: ['sum'] + }); + + expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); + + await fastify.close(); + }); + + it('should set globalVersion in memory cache for realtime query with no channels', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; + + fastify.statistics.services.periodStat.invalidateQueryCache([]); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); + + await fastify.close(); + }); + + it('should hit memory cache for realtime query with matching channelVersions', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; + + const result2 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.equal(callsAfterFirst); + expect(result2.list[0].data.default).to.equal(100); + + await fastify.close(); + }); + }); + + describe('queryCache 版本失效测试（外部缓存模式）', () => { + it('should invalidate external cache when globalVersion changes', async () => { + const cacheStore = {}; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { cacheStore[key] = value; } + }; + + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + fastify.statistics.services.periodStat.invalidateQueryCache([]); + + fastify.statistics.models.periodStat.findAll = async () => []; + + const result = await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + expect(result.list.length).to.equal(0); + + await fastify.close(); + }); + + it('should invalidate external cache when channelVersion changes for specific channel', async () => { + const cacheStore = {}; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { cacheStore[key] = value; } + }; + + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const result1 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + expect(result1.list.length).to.be.greaterThan(0); + + fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); + + fastify.statistics.models.periodStat.findAll = async () => []; + + const result2 = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + expect(result2.list.length).to.equal(0); + + await fastify.close(); + }); + + it('should set globalVersion in external cache for realtime query with no channels', async () => { + const cacheStore = {}; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { cacheStore[key] = value; } + }; + + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const now = new Date(); + const startTime = new Date(now.getTime() - 3600000); + const endTime = new Date(now.getTime() + 3600000); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + startTime, endTime, aggregates: ['sum'] + }); + + const cacheKeys = Object.keys(cacheStore).filter(k => k.includes('query')); + expect(cacheKeys.length).to.be.greaterThan(0); + expect(cacheStore[cacheKeys[0]].globalVersion).to.exist; + + await fastify.close(); + }); + + it('should handle external cache set without TTL support', async () => { + const cacheStore = {}; + const setCalls = []; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value) => { + setCalls.push({ key, value }); + cacheStore[key] = value; + } + }; + + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(setCalls.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should return null from external cache when payload has no value property', async () => { + const cacheStore = {}; + const externalCache = { + get: async (key) => cacheStore[key] || null, + set: async (key, value, ttl) => { cacheStore[key] = value; } + }; + + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + const cacheKey = 'statistics:query:' + JSON.stringify({ + aggregates: ['sum'], attributeNames: [], channels: ['sensor'], + endTime: endTime.toISOString(), includeChildren: false, + startTime: startTime.toISOString(), timezone: '' + }); + cacheStore[cacheKey] = { notValue: true }; + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(0); + + await fastify.close(); + }); + }); + + describe('formatGroupData 边界测试', () => { + it('should not include unit when all items have null or undefined unit', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime, unit: null }, + { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime, unit: undefined } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].unit).to.be.undefined; + + await fastify.close(); + }); + + it('should include unit only for attributes with non-null unit', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime, unit: '°C' }, + { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime, unit: null } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].unit).to.deep.equal({ temp: '°C' }); + + await fastify.close(); + }); + }); + + describe('query 边界测试', () => { + it('should handle channels as a string instead of array', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: 'sensor', startTime, endTime, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + expect(results[0].channel).to.equal('sensor'); + + await fastify.close(); + }); + + it('should escape special characters in channel names for includeChildren query', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor%test', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor%test'], startTime, endTime, aggregates: ['sum'], includeChildren: true + }); + + expect(results.length).to.be.greaterThan(0); + + await fastify.close(); + }); + + it('should build channel tree with parent having no items but children having items', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor:room1', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true + }); + + expect(results.length).to.equal(1); + expect(results[0].channel).to.equal('sensor'); + expect(results[0].children.length).to.be.greaterThan(0); + expect(results[0].children[0].channel).to.equal('sensor:room1'); + + await fastify.close(); + }); + + it('should skip unit assignment when attributeName already in unitMap', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime, unit: '°C' }, + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'avg', data: 50, time: startTime, unit: '°F' } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] + }); + + expect(results.length).to.equal(1); + expect(results[0].unit.temp).to.equal('°C'); + + await fastify.close(); + }); + + it('should not include unit in item entries when unit is undefined', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T02:00:00.000Z'); + + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true + }); + + expect(results.length).to.equal(1); + if (results[0].items) { + expect(results[0].items[0].unit).to.be.undefined; + } + + await fastify.close(); + }); + + it('should return null node when channel has no items and no children in tree', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T01:00:00.000Z'); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['nonexistent'], startTime, endTime, aggregates: ['sum'], includeChildren: true + }); + + expect(results.length).to.equal(0); + + await fastify.close(); + }); + + it('should handle multiple items for same channel in channelGroups', async () => { + const { fastify, periodStatRows } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const startTime = new Date('2026-05-01T00:00:00.000Z'); + const endTime = new Date('2026-05-01T02:00:00.000Z'); + + const secondHour = new Date('2026-05-01T01:00:00.000Z'); + periodStatRows.push( + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime }, + { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 200, time: secondHour } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true + }); + + expect(results.length).to.equal(1); + expect(results[0].items.length).to.equal(2); + + await fastify.close(); + }); + + it('should use default prefix when name option is not provided', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); + + const now = new Date(); + const oneHourAgo = new Date(now.getTime() - 3600000); + + periodStatRows.push( + { period: 'h', channel: 'ch1', attributeName: 'temp', aggregate: 'sum', data: 100, time: oneHourAgo } + ); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['ch1'], startTime: oneHourAgo, endTime: now, aggregates: ['sum'] + }); + + expect(results.length).to.equal(1); + await fastify.close(); + }); + + it('should hit external cache when channelVersions all match', async () => { + const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); + + const cachedList = [{ channel: 'ch1', data: { temp: 42 } }]; + const cachedValue = { channelMetas: {}, list: cachedList }; + + const externalCache = { + get: async (key) => { + if (key.includes('query:')) { + return { + value: cachedValue, + channelVersions: { ch1: 1 } + }; + } + return null; + }, + set: async (key, value, ttl) => {} + }; + + await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); + + const now = new Date(); + const oneHourAgo = new Date(now.getTime() - 3600000); + + fastify.statistics.services.periodStat.invalidateQueryCache(['ch1']); + + const { list: results } = await fastify.statistics.services.periodStat.query({ + channels: ['ch1'], startTime: oneHourAgo, endTime: now, aggregates: ['sum'] + }); + + const dbCalls = findAllCalls.filter(c => c.model === 'periodStat'); + expect(dbCalls.length).to.equal(0); + expect(results).to.deep.equal(cachedList); + + await fastify.close(); + }); + }); + }); +}); diff --git a/test/period-stat-watermark.test.js b/test/period-stat-watermark.test.js new file mode 100644 index 0000000..0a3ceb8 --- /dev/null +++ b/test/period-stat-watermark.test.js @@ -0,0 +1,585 @@ +const { expect } = require('chai'); +const { Sequelize, DataTypes, Op } = require('sequelize'); +const fp = require('fastify-plugin'); +const dayjs = require('dayjs'); +const utc = require('dayjs/plugin/utc'); +const timezone = require('dayjs/plugin/timezone'); + +dayjs.extend(utc); +dayjs.extend(timezone); + +/** + * 创建基于 SQLite 内存数据库的测试环境 + * 包含所有真实 Sequelize 模型，可以直接执行 SQL 查询 + */ +const createSqliteTestEnv = async () => { + const sequelize = new Sequelize('sqlite::memory:', { logging: false }); + + const channelMeta = sequelize.define('watermarkTestChannelMeta', { + channel: { type: DataTypes.STRING, allowNull: false }, + title: { type: DataTypes.STRING, allowNull: false }, + description: { type: DataTypes.TEXT } + }, { + underscored: true, + tableName: 'wm_test_channel_meta', + indexes: [{ name: 'idx_wm_test_channel_meta_channel', unique: true, fields: ['channel'] }] + }); + + const dataRecord = sequelize.define('watermarkTestDataRecord', { + channel: { type: DataTypes.STRING, allowNull: false }, + attributeName: { type: DataTypes.STRING, defaultValue: 'default' }, + data: { type: DataTypes.DECIMAL(16, 2), allowNull: false, defaultValue: 0 }, + time: { type: DataTypes.DATE, allowNull: false }, + unit: { type: DataTypes.STRING }, + channelMetaId: { type: DataTypes.INTEGER } + }, { + underscored: true, + tableName: 'wm_test_data_record', + indexes: [ + { name: 'idx_wm_test_data_record_channel', fields: ['channel'] }, + { name: 'idx_wm_test_data_record_time', fields: ['time'] }, + { name: 'idx_wm_test_data_record_channel_time', fields: ['channel', 'time'] }, + { name: 'idx_wm_test_data_record_channel_attr_time', fields: ['channel', 'attribute_name', 'time'] } + ] + }); + + const periodStat = sequelize.define('watermarkTestPeriodStat', { + period: { type: DataTypes.STRING, allowNull: false }, + time: { type: DataTypes.DATE, allowNull: false }, + channel: { type: DataTypes.STRING, allowNull: false }, + attributeName: { type: DataTypes.STRING, defaultValue: 'default' }, + aggregate: { type: DataTypes.STRING, allowNull: false }, + data: { type: DataTypes.DECIMAL(16, 2), allowNull: false, defaultValue: 0 }, + unit: { type: DataTypes.STRING }, + channelMetaId: { type: DataTypes.INTEGER } + }, { + underscored: true, + tableName: 'wm_test_period_stat', + indexes: [ + { name: 'idx_wm_test_period_stat_unique', unique: true, fields: ['period', 'channel', 'attribute_name', 'aggregate', 'time'] }, + { name: 'idx_wm_test_period_stat_period_time', fields: ['period', 'time'] }, + { name: 'idx_wm_test_period_stat_channel_attr_time', fields: ['channel', 'attribute_name', 'time'] } + ] + }); + + const aggregationWatermark = sequelize.define('watermarkTestAggregationWatermark', { + period: { type: DataTypes.STRING, allowNull: false }, + nextTime: { type: DataTypes.DATE, allowNull: false } + }, { + underscored: true, + tableName: 'wm_test_aggregation_watermark', + indexes: [ + { name: 'idx_wm_test_aggregation_watermark_period', unique: true, fields: ['period'] } + ] + }); + + dataRecord.belongsTo(channelMeta, { foreignKey: 'channelMetaId' }); + periodStat.belongsTo(channelMeta, { foreignKey: 'channelMetaId' }); + + await sequelize.sync({ force: true }); + + const fastify = require('fastify')(); + fastify.decorate('sequelize', { + Sequelize: { Op, fn: Sequelize.fn, col: Sequelize.col }, + instance: sequelize + }); + fastify.decorate('statistics', { + models: { dataRecord, periodStat, aggregationWatermark, channelMeta }, + services: {} + }); + + return { fastify, sequelize, models: { dataRecord, periodStat, aggregationWatermark, channelMeta } }; +}; + +/** + * 加载 period-stat 服务插件 + * 默认不启用自动补偿，由测试手动控制聚合流程 + */ +const loadPeriodStatService = async (fastify, options = {}) => { + const servicePlugin = require('../libs/services/period-stat'); + await fp(servicePlugin)(fastify, { + name: 'statistics', + compensationEnabled: false, + ...options + }); +}; + +/** + * 辅助：比较时间（兼容 raw:true 返回的字符串和 Date 对象） + */ +const isSameTime = (a, b) => { + return new Date(a).getTime() === new Date(b).getTime(); +}; + +/** + * 辅助：查询指定周期的所有 period-stat 记录 + */ +const findPeriodStatRecords = async (models, { period, channel, aggregate: aggType } = {}) => { + const where = {}; + if (period) where.period = period; + if (channel) where.channel = channel; + if (aggType) where.aggregate = aggType; + return models.periodStat.findAll({ where, raw: true }); +}; + +/** + * 辅助：查询指定周期的水位线 + */ +const getWatermarkFromDb = async (models, period) => { + const row = await models.aggregationWatermark.findOne({ where: { period }, raw: true }); + return row ? row.nextTime : null; +}; + +/** + * 辅助：设置指定周期的水位线 + */ +const setWatermarkToDb = async (models, period, nextTime) => { + const existing = await models.aggregationWatermark.findOne({ where: { period } }); + if (existing) { + await existing.update({ nextTime }); + } else { + await models.aggregationWatermark.create({ period, nextTime }); + } +}; + +describe('@kne/fastify-statistics', function () { + this.timeout(30000); + + describe('水位线聚合集成测试（SQLite）', () => { + describe('场景一：无水位线和period-stat，从data-record聚合并设置水位线', () => { + it('应从data-record聚合h数据，级联聚合d/w/m/q/y，并设置水位线', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify); + + // ===== Step 1: 插入 data-record 打点数据（3个小时，每小时2条）===== + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 20, time: new Date('2026-05-01T00:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 22, time: new Date('2026-05-01T00:45:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 21, time: new Date('2026-05-01T01:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 23, time: new Date('2026-05-01T01:45:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 19, time: new Date('2026-05-01T02:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 24, time: new Date('2026-05-01T02:45:00Z'), unit: '°C' } + ]); + + // 验证初始状态：data-record 有数据，period-stat 为空，水位线为空 + expect(await models.dataRecord.count()).to.equal(6); + expect(await models.periodStat.count()).to.equal(0); + expect(await getWatermarkFromDb(models, 'h')).to.be.null; + + // ===== Step 2: 聚合 h 数据（从 data-record）===== + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-01T01:00:00Z') + }); + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T01:00:00Z'), + endTime: new Date('2026-05-01T02:00:00Z') + }); + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T02:00:00Z'), + endTime: new Date('2026-05-01T03:00:00Z') + }); + + // 验证 h 记录：5种聚合 × 3小时 = 15条 + const hRecords = await findPeriodStatRecords(models, { period: 'h' }); + expect(hRecords.length).to.equal(15); + + // 验证 h:00 的聚合值（sum=42, count=2, avg=21, min=20, max=22） + const h00Sum = hRecords.find(r => isSameTime(r.time, '2026-05-01T00:00:00Z') && r.aggregate === 'sum'); + expect(h00Sum).to.exist; + expect(parseFloat(h00Sum.data)).to.equal(42); + + const h00Count = hRecords.find(r => isSameTime(r.time, '2026-05-01T00:00:00Z') && r.aggregate === 'count'); + expect(parseFloat(h00Count.data)).to.equal(2); + + const h00Avg = hRecords.find(r => isSameTime(r.time, '2026-05-01T00:00:00Z') && r.aggregate === 'avg'); + expect(parseFloat(h00Avg.data)).to.equal(21); + + const h00Min = hRecords.find(r => isSameTime(r.time, '2026-05-01T00:00:00Z') && r.aggregate === 'min'); + expect(parseFloat(h00Min.data)).to.equal(20); + + const h00Max = hRecords.find(r => isSameTime(r.time, '2026-05-01T00:00:00Z') && r.aggregate === 'max'); + expect(parseFloat(h00Max.data)).to.equal(22); + + // 验证 data-record 已被删除（h聚合后删除源数据） + expect(await models.dataRecord.count()).to.equal(0); + + // ===== Step 3: 级联聚合 d（从 h 数据）===== + await fastify.statistics.services.periodStat.aggregate('d', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-02T00:00:00Z') + }); + + const dRecords = await findPeriodStatRecords(models, { period: 'd' }); + expect(dRecords.length).to.equal(5); + + // d: sum=42+44+43=129, count=2+2+2=6, avg=129/6=21.5, min=19, max=24 + const dSum = dRecords.find(r => r.aggregate === 'sum'); + expect(parseFloat(dSum.data)).to.equal(129); + + const dCount = dRecords.find(r => r.aggregate === 'count'); + expect(parseFloat(dCount.data)).to.equal(6); + + const dAvg = dRecords.find(r => r.aggregate === 'avg'); + expect(parseFloat(dAvg.data)).to.equal(21.5); + + const dMin = dRecords.find(r => r.aggregate === 'min'); + expect(parseFloat(dMin.data)).to.equal(19); + + const dMax = dRecords.find(r => r.aggregate === 'max'); + expect(parseFloat(dMax.data)).to.equal(24); + + // ===== Step 4: 级联聚合 w（从 d 数据）===== + // 2026-05-01 是周五，所在周从 2026-04-27（周一）开始 + await fastify.statistics.services.periodStat.aggregate('w', { + startTime: new Date('2026-04-27T00:00:00Z'), + endTime: new Date('2026-05-04T00:00:00Z') + }); + + const wRecords = await findPeriodStatRecords(models, { period: 'w' }); + expect(wRecords.length).to.equal(5); + + const wSum = wRecords.find(r => r.aggregate === 'sum'); + expect(parseFloat(wSum.data)).to.equal(129); + + // ===== Step 5: 级联聚合 m（从 d 数据）===== + await fastify.statistics.services.periodStat.aggregate('m', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-06-01T00:00:00Z') + }); + + const mRecords = await findPeriodStatRecords(models, { period: 'm' }); + expect(mRecords.length).to.equal(5); + + const mSum = mRecords.find(r => r.aggregate === 'sum'); + expect(parseFloat(mSum.data)).to.equal(129); + + // ===== Step 6: 级联聚合 q（从 m 数据）===== + await fastify.statistics.services.periodStat.aggregate('q', { + startTime: new Date('2026-04-01T00:00:00Z'), + endTime: new Date('2026-07-01T00:00:00Z') + }); + + const qRecords = await findPeriodStatRecords(models, { period: 'q' }); + expect(qRecords.length).to.equal(5); + + // ===== Step 7: 级联聚合 y（从 q 数据）===== + await fastify.statistics.services.periodStat.aggregate('y', { + startTime: new Date('2026-01-01T00:00:00Z'), + endTime: new Date('2027-01-01T00:00:00Z') + }); + + const yRecords = await findPeriodStatRecords(models, { period: 'y' }); + expect(yRecords.length).to.equal(5); + + // ===== Step 8: 设置水位线 ===== + await setWatermarkToDb(models, 'h', new Date('2026-05-01T03:00:00Z')); + await setWatermarkToDb(models, 'd', new Date('2026-05-02T00:00:00Z')); + await setWatermarkToDb(models, 'w', new Date('2026-05-04T00:00:00Z')); + await setWatermarkToDb(models, 'm', new Date('2026-06-01T00:00:00Z')); + await setWatermarkToDb(models, 'q', new Date('2026-07-01T00:00:00Z')); + await setWatermarkToDb(models, 'y', new Date('2027-01-01T00:00:00Z')); + + // ===== Step 9: 验证水位线 ===== + expect(new Date(await getWatermarkFromDb(models, 'h')).toISOString()).to.equal('2026-05-01T03:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'd')).toISOString()).to.equal('2026-05-02T00:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'w')).toISOString()).to.equal('2026-05-04T00:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'm')).toISOString()).to.equal('2026-06-01T00:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'q')).toISOString()).to.equal('2026-07-01T00:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'y')).toISOString()).to.equal('2027-01-01T00:00:00.000Z'); + + // 汇总验证：period-stat 中所有周期的记录数 + const allRecords = await models.periodStat.count(); + // h:15 + d:5 + w:5 + m:5 + q:5 + y:5 = 40 + expect(allRecords).to.equal(40); + + await fastify.close(); + }); + + it('应支持多通道多属性名的聚合', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify); + + // 插入两个通道的数据 + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 20, time: new Date('2026-05-01T00:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 24, time: new Date('2026-05-01T00:45:00Z'), unit: '°C' }, + { channel: 'sensor:humidity', attributeName: 'value', data: 60, time: new Date('2026-05-01T00:20:00Z'), unit: '%' }, + { channel: 'sensor:humidity', attributeName: 'value', data: 65, time: new Date('2026-05-01T00:50:00Z'), unit: '%' } + ]); + + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-01T01:00:00Z') + }); + + // 2个通道 × 5种聚合 = 10条 h 记录 + const hRecords = await findPeriodStatRecords(models, { period: 'h' }); + expect(hRecords.length).to.equal(10); + + const tempSum = hRecords.find(r => r.channel === 'sensor:temp' && r.aggregate === 'sum'); + expect(parseFloat(tempSum.data)).to.equal(44); + + const humiditySum = hRecords.find(r => r.channel === 'sensor:humidity' && r.aggregate === 'sum'); + expect(parseFloat(humiditySum.data)).to.equal(125); + + await fastify.close(); + }); + }); + + describe('场景二：水位线过期，从水位线时间补偿聚合并更新水位线', () => { + it('应从过期水位线位置开始补偿h聚合，级联更新d/w/m，然后更新水位线', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify); + + // ===== Step 1: 初始聚合 - 小时 00、01 ===== + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 20, time: new Date('2026-05-01T00:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 22, time: new Date('2026-05-01T00:45:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 21, time: new Date('2026-05-01T01:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 23, time: new Date('2026-05-01T01:45:00Z'), unit: '°C' } + ]); + + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-01T01:00:00Z') + }); + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T01:00:00Z'), + endTime: new Date('2026-05-01T02:00:00Z') + }); + + // 聚合 d + await fastify.statistics.services.periodStat.aggregate('d', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-02T00:00:00Z') + }); + + // 记录初始 d 的 sum 值 + const dRecordsBefore = await findPeriodStatRecords(models, { period: 'd', aggregate: 'sum' }); + expect(dRecordsBefore.length).to.equal(1); + const dSumBefore = parseFloat(dRecordsBefore[0].data); + // 20+22 + 21+23 = 86 + expect(dSumBefore).to.equal(86); + + // ===== Step 2: 设置过期水位线 ===== + // h 水位线在 02:00（实际应该推进到 03:00），表示小时 02 未聚合 + await setWatermarkToDb(models, 'h', new Date('2026-05-01T02:00:00Z')); + // d 水位线在当天 00:00（实际应该推进到次日），表示当天 d 需要重新聚合 + await setWatermarkToDb(models, 'd', new Date('2026-05-01T00:00:00Z')); + + // ===== Step 3: 插入新的 data-record 打点数据（小时 02）===== + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 19, time: new Date('2026-05-01T02:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 24, time: new Date('2026-05-01T02:45:00Z'), unit: '°C' } + ]); + + // 验证新数据已入库 + expect(await models.dataRecord.count()).to.equal(2); + + // ===== Step 4: 从水位线位置开始补偿 h 聚合 ===== + // 模拟补偿：从 h 水位线时间 02:00 聚合到 03:00 + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T02:00:00Z'), + endTime: new Date('2026-05-01T03:00:00Z') + }); + + // 验证新的 h 记录 + const hRecords = await findPeriodStatRecords(models, { period: 'h' }); + // 5种聚合 × 3小时 = 15条 + expect(hRecords.length).to.equal(15); + + const h02Sum = hRecords.find( + r => isSameTime(r.time, '2026-05-01T02:00:00Z') && r.aggregate === 'sum' + ); + expect(parseFloat(h02Sum.data)).to.equal(43); // 19 + 24 + + const h02Count = hRecords.find( + r => isSameTime(r.time, '2026-05-01T02:00:00Z') && r.aggregate === 'count' + ); + expect(parseFloat(h02Count.data)).to.equal(2); + + // 验证 data-record 中新的打点数据已被删除 + expect(await models.dataRecord.count()).to.equal(0); + + // ===== Step 5: 重新聚合 d（包含新的 h 数据）===== + await fastify.statistics.services.periodStat.aggregate('d', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-02T00:00:00Z') + }); + + // 验证 d 的 sum 已更新（86 + 43 = 129） + const dRecordsAfter = await findPeriodStatRecords(models, { period: 'd', aggregate: 'sum' }); + expect(dRecordsAfter.length).to.equal(1); + expect(parseFloat(dRecordsAfter[0].data)).to.equal(129); + + // 验证 d 的 count 已更新（4 + 2 = 6） + const dCountAfter = await findPeriodStatRecords(models, { period: 'd', aggregate: 'count' }); + expect(parseFloat(dCountAfter[0].data)).to.equal(6); + + // 验证 d 的 min 已更新（之前 min=20，现在 min=19） + const dMinAfter = await findPeriodStatRecords(models, { period: 'd', aggregate: 'min' }); + expect(parseFloat(dMinAfter[0].data)).to.equal(19); + + // 验证 d 的 max 已更新（之前 max=23，现在 max=24） + const dMaxAfter = await findPeriodStatRecords(models, { period: 'd', aggregate: 'max' }); + expect(parseFloat(dMaxAfter[0].data)).to.equal(24); + + // ===== Step 6: 重新聚合 w/m/q/y（级联更新）===== + await fastify.statistics.services.periodStat.aggregate('w', { + startTime: new Date('2026-04-27T00:00:00Z'), + endTime: new Date('2026-05-04T00:00:00Z') + }); + const wSumAfter = await findPeriodStatRecords(models, { period: 'w', aggregate: 'sum' }); + expect(parseFloat(wSumAfter[0].data)).to.equal(129); + + await fastify.statistics.services.periodStat.aggregate('m', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-06-01T00:00:00Z') + }); + const mSumAfter = await findPeriodStatRecords(models, { period: 'm', aggregate: 'sum' }); + expect(parseFloat(mSumAfter[0].data)).to.equal(129); + + // ===== Step 7: 更新水位线到最新 ===== + await setWatermarkToDb(models, 'h', new Date('2026-05-01T03:00:00Z')); + await setWatermarkToDb(models, 'd', new Date('2026-05-02T00:00:00Z')); + await setWatermarkToDb(models, 'w', new Date('2026-05-04T00:00:00Z')); + await setWatermarkToDb(models, 'm', new Date('2026-06-01T00:00:00Z')); + + // ===== Step 8: 验证水位线已更新 ===== + expect(new Date(await getWatermarkFromDb(models, 'h')).toISOString()).to.equal('2026-05-01T03:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'd')).toISOString()).to.equal('2026-05-02T00:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'w')).toISOString()).to.equal('2026-05-04T00:00:00.000Z'); + expect(new Date(await getWatermarkFromDb(models, 'm')).toISOString()).to.equal('2026-06-01T00:00:00.000Z'); + + await fastify.close(); + }); + + it('应正确处理多窗口补偿：水位线落后多个小时', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify); + + // 初始：聚合小时 00 + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 20, time: new Date('2026-05-01T00:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 22, time: new Date('2026-05-01T00:45:00Z'), unit: '°C' } + ]); + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-01T01:00:00Z') + }); + + // 设置 h 水位线在 01:00（落后3个小时：01、02、03 未聚合） + await setWatermarkToDb(models, 'h', new Date('2026-05-01T01:00:00Z')); + + // 插入 01、02、03 小时的打点数据 + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 10, time: new Date('2026-05-01T01:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 11, time: new Date('2026-05-01T01:45:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 12, time: new Date('2026-05-01T02:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 13, time: new Date('2026-05-01T02:45:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 14, time: new Date('2026-05-01T03:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 15, time: new Date('2026-05-01T03:45:00Z'), unit: '°C' } + ]); + + // 模拟补偿循环：从水位线 01:00 开始逐窗口聚合到 04:00 + const watermarkTime = new Date('2026-05-01T01:00:00Z'); + const targetTime = new Date('2026-05-01T04:00:00Z'); + let nextTime = watermarkTime; + + while (nextTime < targetTime) { + const endTime = new Date(nextTime.getTime() + 3600000); // +1小时 + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: nextTime, + endTime + }); + nextTime = endTime; + await setWatermarkToDb(models, 'h', nextTime); + } + + // 验证：h 水位线已推进到 04:00 + expect(new Date(await getWatermarkFromDb(models, 'h')).toISOString()).to.equal('2026-05-01T04:00:00.000Z'); + + // 验证：所有 4 个小时的 h 记录都存在 + const hRecords = await findPeriodStatRecords(models, { period: 'h' }); + expect(hRecords.length).to.equal(20); // 5种聚合 × 4小时 + + // 验证：data-record 已全部删除 + expect(await models.dataRecord.count()).to.equal(0); + + // 验证：每小时的数据正确 + const h01Sum = hRecords.find( + r => isSameTime(r.time, '2026-05-01T01:00:00Z') && r.aggregate === 'sum' + ); + expect(parseFloat(h01Sum.data)).to.equal(21); // 10+11 + + const h02Sum = hRecords.find( + r => isSameTime(r.time, '2026-05-01T02:00:00Z') && r.aggregate === 'sum' + ); + expect(parseFloat(h02Sum.data)).to.equal(25); // 12+13 + + const h03Sum = hRecords.find( + r => isSameTime(r.time, '2026-05-01T03:00:00Z') && r.aggregate === 'sum' + ); + expect(parseFloat(h03Sum.data)).to.equal(29); // 14+15 + + await fastify.close(); + }); + + it('应正确处理无新数据的补偿窗口', async () => { + const { fastify, models } = await createSqliteTestEnv(); + await loadPeriodStatService(fastify); + + // 初始：聚合小时 00 + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 20, time: new Date('2026-05-01T00:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 22, time: new Date('2026-05-01T00:45:00Z'), unit: '°C' } + ]); + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T00:00:00Z'), + endTime: new Date('2026-05-01T01:00:00Z') + }); + + // 设置 h 水位线在 01:00 + await setWatermarkToDb(models, 'h', new Date('2026-05-01T01:00:00Z')); + + // 小时 01 没有数据（空窗口），小时 02 有数据 + await models.dataRecord.bulkCreate([ + { channel: 'sensor:temp', attributeName: 'value', data: 30, time: new Date('2026-05-01T02:15:00Z'), unit: '°C' }, + { channel: 'sensor:temp', attributeName: 'value', data: 32, time: new Date('2026-05-01T02:45:00Z'), unit: '°C' } + ]); + + // 补偿小时 01（空窗口） + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T01:00:00Z'), + endTime: new Date('2026-05-01T02:00:00Z') + }); + + // 空 window 不应产生 period-stat 记录 + const hRecordsAfterEmpty = await findPeriodStatRecords(models, { period: 'h' }); + const h01Records = hRecordsAfterEmpty.filter( + r => isSameTime(r.time, '2026-05-01T01:00:00Z') + ); + expect(h01Records.length).to.equal(0); + + // 补偿小时 02（有数据） + await fastify.statistics.services.periodStat.aggregate('h', { + startTime: new Date('2026-05-01T02:00:00Z'), + endTime: new Date('2026-05-01T03:00:00Z') + }); + + const hRecordsAfter02 = await findPeriodStatRecords(models, { period: 'h' }); + const h02Records = hRecordsAfter02.filter( + r => isSameTime(r.time, '2026-05-01T02:00:00Z') + ); + expect(h02Records.length).to.equal(5); // 5种聚合 + + // 推进水位线跳过空窗口 + await setWatermarkToDb(models, 'h', new Date('2026-05-01T03:00:00Z')); + expect(new Date(await getWatermarkFromDb(models, 'h')).toISOString()).to.equal('2026-05-01T03:00:00.000Z'); + + await fastify.close(); + }); + }); + }); +}); diff --git a/test/period-stat.test.js b/test/period-stat.test.js deleted file mode 100644 index 5da06cd..0000000 --- a/test/period-stat.test.js +++ /dev/null @@ -1,2844 +0,0 @@ -const { expect } = require('chai'); -const fp = require('fastify-plugin'); - -const mockPeriodStatService = (fastify, options) => { - const servicePlugin = require('../libs/services/period-stat'); - return fp(servicePlugin)(fastify, options); -}; - -const createMockFastify = () => { - const findAllResults = []; - const bulkCreateCalls = []; - const destroyCalls = []; - const periodStatRows = []; - - const mockTransaction = { - commit: async () => {}, - rollback: async () => {} - }; - - const mockModel = { - dataRecord: { - findAll: async () => findAllResults.splice(0, findAllResults.length), - destroy: async (opts) => { - destroyCalls.push(opts); - } - }, - periodStat: { - bulkCreate: async (records, opts) => { - bulkCreateCalls.push({ records: [...records], opts: opts || {} }); - return records; - }, - findAll: async () => periodStatRows.splice(0, periodStatRows.length) - } - }; - - const fastify = require('fastify')(); - - fastify.decorate('sequelize', { - Sequelize: { Op: { between: 'between' }, fn: (name, col) => `${name}(${col})`, col: name => name }, - instance: { transaction: async () => mockTransaction } - }); - - fastify.decorate('statistics', { - models: mockModel, - services: {} - }); - - return { fastify, findAllResults, bulkCreateCalls, destroyCalls, periodStatRows, mockModel }; -}; - -describe('@kne/fastify-statistics', function () { - describe('周期统计接口测试', () => { - describe('aggregate 方法 - 从 data-record 聚合 (period=h)', () => { - it('should throw error for unsupported period', async () => { - const { fastify } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - try { - await fastify.statistics.services.periodStat.aggregate('x'); - expect.fail('should have thrown'); - } catch (e) { - expect(e.message).to.include('Unsupported period: x'); - } - - await fastify.close(); - }); - - it('should generate records for all aggregate types from data-record when period=h', async () => { - const { fastify, findAllResults, bulkCreateCalls } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - findAllResults.push({ - channel: 'temperature', - attributeName: 'value', - sum: 100, - avg: 25, - count: 4, - min: 10, - max: 40 - }); - - const records = await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); - - expect(records.length).to.equal(5); - expect(bulkCreateCalls.length).to.equal(1); - expect(bulkCreateCalls[0].records.length).to.equal(5); - expect(bulkCreateCalls[0].opts.updateOnDuplicate).to.deep.equal(['data', 'unit']); - - const aggregates = records.map(r => r.aggregate); - expect(aggregates).to.have.members(['sum', 'avg', 'count', 'min', 'max']); - - const sumRecord = records.find(r => r.aggregate === 'sum'); - expect(sumRecord.period).to.equal('h'); - expect(sumRecord.channel).to.equal('temperature'); - expect(sumRecord.attributeName).to.equal('value'); - expect(sumRecord.data).to.equal(100); - expect(sumRecord.title).to.be.undefined; - expect(sumRecord.description).to.be.undefined; - expect(sumRecord.unit).to.be.undefined; - - await fastify.close(); - }); - - it('should not delete data-record immediately after successful h aggregation', async () => { - const { fastify, findAllResults, destroyCalls } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); - - expect(destroyCalls.length).to.equal(0); - - await fastify.close(); - }); - - it('should not delete data-record when no records to aggregate', async () => { - const { fastify, destroyCalls } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); - - expect(destroyCalls.length).to.equal(0); - - await fastify.close(); - }); - - it('should skip null aggregate values from data-record', async () => { - const { fastify, findAllResults } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 100, avg: null, count: 5, min: null, max: 50 - }); - - const records = await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); - - expect(records.length).to.equal(3); - const aggregates = records.map(r => r.aggregate); - expect(aggregates).to.have.members(['sum', 'count', 'max']); - - await fastify.close(); - }); - }); - - describe('aggregate 方法 - 从 period-stat 聚合 (period>d/w/m/q/y)', () => { - it('should aggregate from period-stat(h) when period=d', async () => { - const { fastify, periodStatRows, bulkCreateCalls, destroyCalls } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'sum', data: 30, time: new Date('2026-05-01T00:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'sum', data: 20, time: new Date('2026-05-01T01:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'count', data: 3, time: new Date('2026-05-01T00:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'count', data: 2, time: new Date('2026-05-01T01:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'min', data: 8, time: new Date('2026-05-01T00:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'min', data: 12, time: new Date('2026-05-01T01:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'max', data: 22, time: new Date('2026-05-01T00:00:00.000Z') }, - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'max', data: 28, time: new Date('2026-05-01T01:00:00.000Z') } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - - expect(records.length).to.equal(5); - - const sumRecord = records.find(r => r.aggregate === 'sum'); - expect(sumRecord.data).to.equal(50); - - const countRecord = records.find(r => r.aggregate === 'count'); - expect(countRecord.data).to.equal(5); - - const avgRecord = records.find(r => r.aggregate === 'avg'); - expect(avgRecord.data).to.equal(50 / 5); - - const minRecord = records.find(r => r.aggregate === 'min'); - expect(minRecord.data).to.equal(8); - - const maxRecord = records.find(r => r.aggregate === 'max'); - expect(maxRecord.data).to.equal(28); - - expect(destroyCalls.length).to.equal(0); - - // Verify bulkCreate used updateOnDuplicate for idempotency - expect(bulkCreateCalls.length).to.equal(1); - expect(bulkCreateCalls[0].opts.updateOnDuplicate).to.deep.equal(['data', 'unit']); - - await fastify.close(); - }); - - it('should aggregate multiple channels separately from period-stat', async () => { - const { fastify, periodStatRows } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'temperature', attributeName: 'value', aggregate: 'sum', data: 50, time: startTime }, - { period: 'h', channel: 'humidity', attributeName: 'value', aggregate: 'sum', data: 200, time: startTime } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - - expect(records.filter(r => r.channel === 'temperature').length).to.equal(1); - expect(records.filter(r => r.channel === 'humidity').length).to.equal(1); - - await fastify.close(); - }); - - it('should compute avg from sum and count of lower period', async () => { - const { fastify, periodStatRows } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 120, time: startTime }, - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'count', data: 3, time: startTime } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - - const avgRecord = records.find(r => r.aggregate === 'avg'); - expect(avgRecord.data).to.equal(40); - - await fastify.close(); - }); - - it('should not call bulkCreate when no period-stat rows to aggregate', async () => { - const { fastify, bulkCreateCalls } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - - expect(records.length).to.equal(0); - expect(bulkCreateCalls.length).to.equal(0); - - await fastify.close(); - }); - - it('should group by channel and attributeName', async () => { - const { fastify, periodStatRows } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'd', channel: 'temperature', attributeName: 'high', aggregate: 'sum', data: 50, time: startTime }, - { period: 'd', channel: 'temperature', attributeName: 'low', aggregate: 'sum', data: 30, time: startTime } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('m', { startTime, endTime }); - - expect(records.length).to.equal(2); - const highRecord = records.find(r => r.attributeName === 'high'); - const lowRecord = records.find(r => r.attributeName === 'low'); - expect(highRecord.data).to.equal(50); - expect(lowRecord.data).to.equal(30); - - await fastify.close(); - }); - - it('should handle null attributeName in aggregateFromPeriodStat', async () => { - const { fastify, periodStatRows } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: null, aggregate: 'sum', data: 50, time: startTime } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - expect(records.length).to.equal(1); - expect(records[0].attributeName).to.equal(null); - - await fastify.close(); - }); - - it('should handle only count aggregate without sum in aggregateFromPeriodStat', async () => { - const { fastify, periodStatRows } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'count', data: 5, time: startTime }, - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'min', data: 1, time: startTime }, - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'max', data: 10, time: startTime } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - const aggregates = records.map(r => r.aggregate); - expect(aggregates).to.include('count'); - expect(aggregates).to.include('min'); - expect(aggregates).to.include('max'); - expect(aggregates).to.not.include('sum'); - expect(aggregates).to.not.include('avg'); - - await fastify.close(); - }); - }); - - describe('cron 任务注册测试', () => { - it('should register cron jobs when fastify.cron is available', async () => { - const { fastify } = createMockFastify(); - const createdJobs = []; - - fastify.decorate('cron', { - createJob: (jobConfig) => { - createdJobs.push(jobConfig); - } - }); - - await mockPeriodStatService(fastify, { name: 'statistics' }); - - expect(createdJobs.length).to.equal(6); - - const jobNames = createdJobs.map(j => j.name); - expect(jobNames).to.include('statistics-period-stat-h'); - expect(jobNames).to.include('statistics-period-stat-d'); - expect(jobNames).to.include('statistics-period-stat-w'); - expect(jobNames).to.include('statistics-period-stat-m'); - expect(jobNames).to.include('statistics-period-stat-q'); - expect(jobNames).to.include('statistics-period-stat-y'); - - expect(createdJobs[0].cronTime).to.exist; - expect(createdJobs[0].onTick).to.be.a('function'); - expect(createdJobs[0].startWhenReady).to.be.true; - - await fastify.close(); - }); - - it('should not register cron jobs when fastify.cron is not available', async () => { - const { fastify } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - expect(fastify.statistics.services.periodStat.aggregate).to.be.a('function'); - - await fastify.close(); - }); - }); - - describe('query 方法', () => { - const createQueryMockFastify = () => { - const periodStatRows = []; - const dataRecordFindAllResult = []; - const findAllCalls = []; - const channelMetaRows = []; - - const mockTransaction = { - commit: async () => {}, - rollback: async () => {} - }; - - const mockModel = { - dataRecord: { - findAll: async (opts) => { - findAllCalls.push({ model: 'dataRecord', opts }); - return dataRecordFindAllResult.splice(0); - }, - destroy: async () => {} - }, - periodStat: { - bulkCreate: async () => {}, - findAll: async (opts) => { - findAllCalls.push({ model: 'periodStat', opts }); - const period = opts.where && opts.where.period; - if (period && period.in) { - return periodStatRows.filter(row => period.in.includes(row.period)); - } - return periodStatRows.filter(row => !period || row.period === period); - } - }, - channelMeta: { - findAll: async ({ where }) => { - if (where && where.channel && where.channel.in) { - return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); - } - return channelMetaRows; - } - } - }; - - const fastify = require('fastify')(); - - fastify.decorate('sequelize', { - Sequelize: { - Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, - fn: (name, col) => `${name}(${col})`, - col: name => name - }, - instance: { transaction: async () => mockTransaction } - }); - - fastify.decorate('statistics', { - models: mockModel, - services: {} - }); - - return { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls, channelMetaRows }; - }; - - it('should return attribute-keyed object when single aggregate and single default attribute with no filter', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push({ - period: 'h', channel: 'sensor', attributeName: 'default', - aggregate: 'sum', data: 100, time: startTime - }); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].channel).to.equal('sensor'); - expect(results[0].period).to.equal('h'); - expect(results[0].data).to.deep.equal({ default: 100 }); - - await fastify.close(); - }); - - it('should return attribute-keyed object when single aggregate and multiple attributes', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ temperature: 100, humidity: 200 }); - - await fastify.close(); - }); - - it('should return nested object when multiple aggregates and single default attribute', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'avg', data: 25, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ sum: { default: 100 }, avg: { default: 25 } }); - - await fastify.close(); - }); - - it('should return nested object when multiple aggregates and multiple attributes', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'avg', data: 25, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'avg', data: 50, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ - sum: { temperature: 100, humidity: 200 }, - avg: { temperature: 25, humidity: 50 } - }); - - await fastify.close(); - }); - - it('should not flatten default attribute when attributeNames filter is provided', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, attributeNames: ['default'], aggregates: ['sum'] - }); - - expect(results[0].data).to.deep.equal({ default: 100 }); - - await fastify.close(); - }); - - it('should query all child channels', async () => { - const { fastify, periodStatRows, channelMetaRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor:room1', attributeName: 'default', aggregate: 'sum', data: 50, time: startTime }, - { period: 'h', channel: 'sensor:room2', attributeName: 'default', aggregate: 'sum', data: 30, time: startTime } - ); - channelMetaRows.push({ channel: 'sensor', title: '传感器', description: '温度' }); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true - }); - - expect(results.length).to.equal(1); - const root = results[0]; - expect(root.channel).to.equal('sensor'); - expect(root.items[0].data).to.deep.equal({ default: 100 }); - expect(root.children.length).to.equal(2); - const childChannels = root.children.map(c => c.channel); - expect(childChannels).to.include('sensor:room1'); - expect(childChannels).to.include('sensor:room2'); - const room1 = root.children.find(c => c.channel === 'sensor:room1'); - expect(room1.items[0].data).to.deep.equal({ default: 50 }); - - expect(channelMetas).to.have.property('sensor'); - expect(channelMetas.sensor.title).to.equal('传感器'); - expect(Object.keys(channelMetas).length).to.equal(1); - - await fastify.close(); - }); - - it('should query data from all period types in single query', async () => { - const { fastify, periodStatRows, findAllCalls } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 10, time: startTime }, - { period: 'd', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 240, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const periodTypes = results.map(r => r.period); - expect(periodTypes).to.include('h'); - expect(periodTypes).to.include('d'); - - const psCalls = findAllCalls.filter(c => c.model === 'periodStat'); - expect(psCalls.length).to.equal(1); - expect(psCalls[0].opts.where.period).to.deep.equal({ in: ['h', 'd', 'w', 'm', 'q', 'y'] }); - - await fastify.close(); - }); - - it('should return empty array when no data found', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['nonexistent'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results).to.deep.equal([]); - expect(channelMetas).to.deep.equal({}); - - await fastify.close(); - }); - - it('should filter by attributeNames when provided', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temperature', aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, attributeNames: ['temperature'], aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ temperature: 100 }); - - await fastify.close(); - }); - - it('should query data-record for current hour and format correctly', async () => { - const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - dataRecordFindAllResult.push({ - channel: 'sensor', attributeName: 'temperature', - sum: 50, avg: 25, count: 2, min: 10, max: 40 - }); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const drCalls = findAllCalls.filter(c => c.model === 'dataRecord'); - expect(drCalls.length).to.be.greaterThan(0); - - const hourResult = results.find(r => r.period === 'h' && r.channel === 'sensor'); - if (hourResult) { - expect(hourResult.data).to.deep.equal({ temperature: 50 }); - } - - await fastify.close(); - }); - - it('should return all aggregates when aggregates not specified', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'avg', data: 25, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'count', data: 4, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'min', data: 10, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'max', data: 40, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ sum: { default: 100 }, avg: { default: 25 }, count: { default: 4 }, min: { default: 10 }, max: { default: 40 } }); - - await fastify.close(); - }); - - it('should sort results by time', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T03:00:00.000Z'); - - const time2 = new Date('2026-05-01T02:00:00.000Z'); - const time0 = new Date('2026-05-01T00:00:00.000Z'); - const time1 = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 30, time: time2 }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 10, time: time0 }, - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 20, time: time1 } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results[0].data).to.deep.equal({ default: 10 }); - expect(results[1].data).to.deep.equal({ default: 20 }); - expect(results[2].data).to.deep.equal({ default: 30 }); - - await fastify.close(); - }); - - it('should use client timezone to determine current hour when timezone is provided', async () => { - const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - dataRecordFindAllResult.push({ - channel: 'sensor', attributeName: 'temperature', - sum: 50, avg: null, count: null, min: null, max: null - }); - - const resultsNoTz = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const resultsWithTz = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], timezone: 'Asia/Shanghai' - }); - - const drCallsNoTz = findAllCalls.filter(c => c.model === 'dataRecord'); - expect(drCallsNoTz.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should calculate different current hour boundaries for different timezones', async () => { - const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 7200000); - const endTime = new Date(now.getTime() + 3600000); - - dataRecordFindAllResult.push({ - channel: 'sensor', attributeName: 'default', - sum: 100, avg: null, count: null, min: null, max: null - }); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], timezone: 'America/New_York' - }); - - const drCalls = findAllCalls.filter(c => c.model === 'dataRecord'); - expect(drCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should throw error for invalid timezone', async () => { - const { fastify } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - try { - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], timezone: 'Invalid/Timezone' - }); - expect.fail('should have thrown'); - } catch (e) { - expect(e.message).to.include('Invalid timezone'); - } - - await fastify.close(); - }); - - it('should query without channel filter and return channelMetas', async () => { - const { fastify, periodStatRows, channelMetaRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor1', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor2', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } - ); - channelMetaRows.push( - { channel: 'sensor1', title: '传感器1', description: null }, - { channel: 'sensor2', title: '传感器2', description: '温度' } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(2); - expect(Object.keys(channelMetas).length).to.equal(2); - expect(channelMetas.sensor1.title).to.equal('传感器1'); - expect(channelMetas.sensor2.title).to.equal('传感器2'); - - await fastify.close(); - }); - - it('should handle null attributeName with single aggregate and no filter', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ default: 100 }); - - await fastify.close(); - }); - - it('should handle null attributeName with single aggregate and attributeNames filter', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, attributeNames: ['value'], aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ default: 100 }); - - await fastify.close(); - }); - - it('should handle null attributeName with multiple aggregates and attributeNames filter', async () => { - const { fastify, periodStatRows } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: null, aggregate: 'avg', data: 25, time: startTime } - ); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, attributeNames: ['value'], aggregates: ['sum', 'avg'] - }); - - expect(results.length).to.equal(1); - expect(results[0].data).to.deep.equal({ - sum: { default: 100 }, - avg: { default: 25 } - }); - - await fastify.close(); - }); - - it('should skip undefined aggregate values from data-record in query', async () => { - const { fastify, dataRecordFindAllResult } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - dataRecordFindAllResult.push({ - channel: 'sensor', attributeName: 'temperature', - sum: 50, avg: undefined, count: undefined, min: undefined, max: undefined - }); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] - }); - - const hourResult = results.find(r => r.period === 'h'); - if (hourResult) { - expect(hourResult.data).to.deep.equal({ temperature: 50 }); - } - - await fastify.close(); - }); - - it('should use startTime as drStartTime when startTime is after currentHourStart', async () => { - const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const now = new Date(); - const currentHourStart = new Date(now); - currentHourStart.setMinutes(0, 0, 0); - const startTime = new Date(currentHourStart.getTime() + 30 * 60 * 1000); - const endTime = new Date(currentHourStart.getTime() + 60 * 60 * 1000); - - dataRecordFindAllResult.push({ - channel: 'sensor', attributeName: 'default', - sum: 100, avg: null, count: null, min: null, max: null - }); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should handle null attributeName in data-record query results', async () => { - const { fastify, periodStatRows, dataRecordFindAllResult, findAllCalls } = createQueryMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - dataRecordFindAllResult.push({ - channel: 'sensor', attributeName: null, - sum: 100, avg: null, count: null, min: null, max: null - }); - - const { list: results, channelMetas } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const drCalls = findAllCalls.filter(c => c.model === 'dataRecord'); - expect(drCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - }); - - describe('事务回滚测试', () => { - it('should rollback transaction when bulkCreate fails in aggregateFromDataRecord', async () => { - const { fastify, findAllResults, mockModel } = createMockFastify(); - let rollbackCalled = false; - const mockTransaction = { - commit: async () => {}, - rollback: async () => { rollbackCalled = true; } - }; - fastify.sequelize.instance.transaction = async () => mockTransaction; - - mockModel.periodStat.bulkCreate = async () => { - throw new Error('bulkCreate error'); - }; - - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - try { - await fastify.statistics.services.periodStat.aggregate('h', { startTime, endTime }); - expect.fail('should have thrown'); - } catch (e) { - expect(e.message).to.equal('bulkCreate error'); - expect(rollbackCalled).to.be.true; - } - - await fastify.close(); - }); - - it('should rollback transaction when bulkCreate fails in aggregateFromPeriodStat', async () => { - const { fastify, periodStatRows, mockModel } = createMockFastify(); - let rollbackCalled = false; - const mockTransaction = { - commit: async () => {}, - rollback: async () => { rollbackCalled = true; } - }; - fastify.sequelize.instance.transaction = async () => mockTransaction; - - mockModel.periodStat.bulkCreate = async () => { - throw new Error('bulkCreate error'); - }; - - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: startTime } - ); - - try { - await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - expect.fail('should have thrown'); - } catch (e) { - expect(e.message).to.equal('bulkCreate error'); - expect(rollbackCalled).to.be.true; - } - - await fastify.close(); - }); - }); - - describe('aggregate 自动时间计算', () => { - it('should auto-calculate startTime and endTime when not provided', async () => { - const { fastify, findAllResults } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - const records = await fastify.statistics.services.periodStat.aggregate('h'); - - expect(records.length).to.equal(1); - expect(records[0].aggregate).to.equal('sum'); - expect(records[0].data).to.equal(10); - - await fastify.close(); - }); - - it('should auto-calculate time for all period types', async () => { - const periods = ['h', 'd', 'w', 'm', 'q', 'y']; - for (const period of periods) { - const { fastify, findAllResults, periodStatRows } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - if (period === 'h') { - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - } else { - periodStatRows.push( - { period: period === 'd' ? 'h' : period === 'w' || period === 'm' ? 'd' : period === 'q' ? 'm' : 'q', - channel: 'ch1', attributeName: 'val', - aggregate: 'sum', data: 10, time: new Date() } - ); - } - - const records = await fastify.statistics.services.periodStat.aggregate(period); - // Should not throw, auto time calculation works for all periods - expect(records).to.be.an('array'); - - await fastify.close(); - } - }); - }); - - describe('cron onTick 错误处理', () => { - it('should catch and log error when aggregate fails in cron onTick', async () => { - const { fastify } = createMockFastify(); - const createdJobs = []; - let logErrorCalled = false; - - const originalLogError = fastify.log.error; - fastify.log.error = (msg) => { - logErrorCalled = true; - return originalLogError ? originalLogError.call(fastify.log, msg) : undefined; - }; - - fastify.decorate('cron', { - createJob: (jobConfig) => { - createdJobs.push(jobConfig); - } - }); - - // Register service first - await mockPeriodStatService(fastify, { name: 'statistics' }); - - // Now manually trigger onTick with a failing aggregate - fastify.statistics.models.dataRecord.findAll = async () => { - throw new Error('Cron aggregate error'); - }; - - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - expect(hJob).to.exist; - - await hJob.onTick(); - - expect(logErrorCalled).to.be.true; - - await fastify.close(); - }); - }); - - describe('查询缓存测试（内存模式）', () => { - const createCacheTestMockFastify = () => { - const periodStatRows = []; - const findAllCalls = []; - const channelMetaRows = []; - - const mockTransaction = { - commit: async () => {}, - rollback: async () => {} - }; - - const mockModel = { - dataRecord: { - findAll: async (opts) => { - findAllCalls.push({ model: 'dataRecord', opts }); - return []; - }, - destroy: async () => {} - }, - periodStat: { - bulkCreate: async () => {}, - findAll: async (opts) => { - findAllCalls.push({ model: 'periodStat', opts }); - const period = opts.where && opts.where.period; - if (period && period.in) { - return periodStatRows.filter(row => period.in.includes(row.period)); - } - return periodStatRows.filter(row => !period || row.period === period); - } - }, - channelMeta: { - findAll: async ({ where }) => { - if (where && where.channel && where.channel.in) { - return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); - } - return channelMetaRows; - } - } - }; - - const fastify = require('fastify')(); - - fastify.decorate('sequelize', { - Sequelize: { - Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, - fn: (name, col) => `${name}(${col})`, - col: name => name - }, - instance: { transaction: async () => mockTransaction } - }); - - fastify.decorate('statistics', { - models: mockModel, - services: {} - }); - - return { fastify, periodStatRows, findAllCalls, channelMetaRows }; - }; - - it('should cache query result and return from cache on second call', async () => { - const { fastify, periodStatRows, findAllCalls } = createCacheTestMockFastify(); - // Disable compensation so it doesn't interfere with cache - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const result1 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - expect(result1.list.length).to.be.greaterThan(0); - - const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; - - const result2 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Second call should hit cache, no additional DB calls - const callsAfterSecond = findAllCalls.filter(c => c.model === 'periodStat').length; - expect(callsAfterSecond).to.equal(callsAfterFirst); - expect(result2).to.deep.equal(result1); - - await fastify.close(); - }); - - it('should invalidate cache when invalidateQueryCache is called', async () => { - const { fastify, periodStatRows, findAllCalls } = createCacheTestMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.length; - - // Invalidate cache for sensor channel - fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); - - // Add new data - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'avg', data: 50, time: startTime } - ); - - // Query again - should miss cache and fetch from DB - const result = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] - }); - expect(findAllCalls.length).to.be.greaterThan(callsAfterFirst); - - await fastify.close(); - }); - - it('should not cache when queryCacheEnabled is false', async () => { - const { fastify, periodStatRows, findAllCalls } = createCacheTestMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', queryCacheEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.length; - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Second call should NOT hit cache, should make new DB calls - expect(findAllCalls.length).to.be.greaterThan(callsAfterFirst); - - await fastify.close(); - }); - - it('should evict oldest entry when memory cache exceeds maxEntries', async () => { - const { fastify, periodStatRows } = createCacheTestMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', queryCacheMaxEntries: 2 }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'default', aggregate: 'sum', data: 10, time: startTime }, - { period: 'h', channel: 'ch2', attributeName: 'default', aggregate: 'sum', data: 20, time: startTime }, - { period: 'h', channel: 'ch3', attributeName: 'default', aggregate: 'sum', data: 30, time: startTime } - ); - - // Fill cache with 3 entries (max is 2, so first should be evicted) - await fastify.statistics.services.periodStat.query({ channels: ['ch1'], startTime, endTime, aggregates: ['sum'] }); - await fastify.statistics.services.periodStat.query({ channels: ['ch2'], startTime, endTime, aggregates: ['sum'] }); - await fastify.statistics.services.periodStat.query({ channels: ['ch3'], startTime, endTime, aggregates: ['sum'] }); - - // Query ch1 again - should miss cache (evicted) - // Since periodStatRows is already spliced out, we need fresh data - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'default', aggregate: 'sum', data: 11, time: startTime } - ); - - const result = await fastify.statistics.services.periodStat.query({ channels: ['ch1'], startTime, endTime, aggregates: ['sum'] }); - // Should have fetched new data (data=11) since ch1 was evicted - expect(result.list[0].data.default).to.equal(11); - - await fastify.close(); - }); - - it('should use historyTTL for non-realtime queries', async () => { - const { fastify, periodStatRows } = createCacheTestMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', queryCacheTTL: 1, queryCacheHistoryTTL: 3600 }); - - const startTime = new Date('2020-05-01T00:00:00.000Z'); - const endTime = new Date('2020-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // First query - populates cache - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Wait for realtime TTL (1s) to expire, but historyTTL is 3600s - await new Promise(resolve => setTimeout(resolve, 1100)); - - // Query again - should still hit cache because historyTTL is long - const result = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - expect(result.list[0].data.default).to.equal(100); - - await fastify.close(); - }); - - it('should not cache when isCompensating is true', async () => { - const { fastify, periodStatRows } = createCacheTestMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // Before compensation, query should be cached - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Simulate compensating state - expect(fastify.statistics.services.periodStat.isCompensating()).to.be.false; - - await fastify.close(); - }); - }); - - describe('查询缓存测试（外部缓存模式）', () => { - const createExternalCacheMockFastify = () => { - const cacheStore = {}; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { cacheStore[key] = value; } - }; - - const periodStatRows = []; - const findAllCalls = []; - const channelMetaRows = []; - - const mockTransaction = { - commit: async () => {}, - rollback: async () => {} - }; - - const mockModel = { - dataRecord: { - findAll: async (opts) => { - findAllCalls.push({ model: 'dataRecord', opts }); - return []; - }, - destroy: async () => {} - }, - periodStat: { - bulkCreate: async () => {}, - findAll: async (opts) => { - findAllCalls.push({ model: 'periodStat', opts }); - const period = opts.where && opts.where.period; - if (period && period.in) { - return periodStatRows.filter(row => period.in.includes(row.period)); - } - return periodStatRows; - } - }, - channelMeta: { - findAll: async ({ where }) => { - if (where && where.channel && where.channel.in) { - return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); - } - return channelMetaRows; - } - } - }; - - const fastify = require('fastify')(); - - fastify.decorate('sequelize', { - Sequelize: { - Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, - fn: (name, col) => `${name}(${col})`, - col: name => name - }, - instance: { transaction: async () => mockTransaction } - }); - - fastify.decorate('statistics', { - models: mockModel, - services: {} - }); - - return { fastify, periodStatRows, findAllCalls, channelMetaRows, cacheStore, externalCache }; - }; - - it('should cache query result in external cache and return from cache on second call', async () => { - const { fastify, periodStatRows, externalCache } = createExternalCacheMockFastify(); - // Disable compensation to avoid interference - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const result1 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - expect(result1.list.length).to.be.greaterThan(0); - - // Now make findAll return empty to prove second query uses cache - fastify.statistics.models.periodStat.findAll = async () => []; - - const result2 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Second call should hit external cache, return same result - expect(result2.list.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should return null from external cache when payload is invalid', async () => { - const cacheStore = {}; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { cacheStore[key] = value; } - }; - - const periodStatRows = []; - const findAllCalls = []; - const mockTransaction = { commit: async () => {}, rollback: async () => {} }; - const mockModel = { - dataRecord: { findAll: async (opts) => { findAllCalls.push({ model: 'dataRecord', opts }); return []; }, destroy: async () => {} }, - periodStat: { bulkCreate: async () => {}, findAll: async (opts) => { findAllCalls.push({ model: 'periodStat', opts }); return periodStatRows.splice(0); } }, - channelMeta: { findAll: async () => [] } - }; - - const fastify = require('fastify')(); - fastify.decorate('sequelize', { - Sequelize: { Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, fn: (n, c) => `${n}(${c})`, col: n => n }, - instance: { transaction: async () => mockTransaction } - }); - fastify.decorate('statistics', { models: mockModel, services: {} }); - - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - // Manually corrupt the cache - const cacheKey = 'statistics:query:' + JSON.stringify({ channels: ['sensor'], startTime: startTime.toISOString(), endTime: endTime.toISOString(), attributeNames: [], aggregates: ['sum'], timezone: '', includeChildren: false }); - cacheStore[cacheKey] = 'not-an-object'; // Invalid payload - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const result = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Should have fetched from DB since cache was invalid - expect(findAllCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should invalidate external cache when channel version changes', async () => { - const { fastify, periodStatRows, cacheStore, externalCache } = createExternalCacheMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // First query - populates cache - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Invalidate cache for sensor channel - fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); - - // Add new data and query again - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } - ); - - const result = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - // Should have fetched fresh data - expect(result.list[0].data.default).to.equal(200); - - await fastify.close(); - }); - - it('should handle external cache with 3-argument set (TTL support)', async () => { - const cacheStore = {}; - const setCalls = []; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { - setCalls.push({ key, value, ttl }); - cacheStore[key] = value; - } - }; - - const periodStatRows = []; - const mockTransaction = { commit: async () => {}, rollback: async () => {} }; - const mockModel = { - dataRecord: { findAll: async () => [], destroy: async () => {} }, - periodStat: { bulkCreate: async () => {}, findAll: async () => periodStatRows.splice(0) }, - channelMeta: { findAll: async () => [] } - }; - - const fastify = require('fastify')(); - fastify.decorate('sequelize', { - Sequelize: { Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, fn: (n, c) => `${n}(${c})`, col: n => n }, - instance: { transaction: async () => mockTransaction } - }); - fastify.decorate('statistics', { models: mockModel, services: {} }); - - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - // Should have called cache.set - expect(setCalls.length).to.be.greaterThan(0); - // The set call for query cache should have TTL - const queryCacheSet = setCalls.find(c => c.key.includes('statistics:query:')); - expect(queryCacheSet).to.exist; - expect(queryCacheSet.ttl).to.exist; - - await fastify.close(); - }); - }); - - describe('compensate 补偿逻辑测试', () => { - const createCompensateMockFastify = () => { - const findAllResults = []; - const periodStatRows = []; - const bulkCreateCalls = []; - const watermarkStore = {}; - let destroyCount = 0; - - const mockTransaction = { - commit: async () => {}, - rollback: async () => {} - }; - - const mockModel = { - dataRecord: { - findAll: async () => findAllResults.splice(0), - destroy: async () => { destroyCount++; return destroyCount; } - }, - periodStat: { - bulkCreate: async (records, opts) => { - bulkCreateCalls.push({ records: [...records], opts: opts || {} }); - return records; - }, - findAll: async (opts) => { - const period = opts.where && opts.where.period; - if (period) { - return periodStatRows.filter(r => r.period === period); - } - return periodStatRows.splice(0); - } - }, - aggregationWatermark: { - findOne: async ({ where }) => watermarkStore[where.period] || null, - upsert: async (data) => { watermarkStore[data.period] = data; } - } - }; - - const fastify = require('fastify')(); - fastify.decorate('sequelize', { - Sequelize: { Op: { between: 'between' }, fn: (name, col) => `${name}(${col})`, col: name => name }, - instance: { transaction: async () => mockTransaction } - }); - fastify.decorate('statistics', { models: mockModel, services: {} }); - - return { fastify, findAllResults, periodStatRows, bulkCreateCalls, watermarkStore, mockModel }; - }; - - it('should initialize watermark from data-record min time when no existing watermark', async () => { - const { fastify, findAllResults, watermarkStore, mockModel } = createCompensateMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - // Mock dataRecord.findOne for initWatermark (called when no watermark exists) - const minTime = new Date('2026-05-01T00:00:00.000Z'); - mockModel.dataRecord.findOne = async () => ({ minTime }); - mockModel.periodStat.findOne = async () => ({ minTime }); - - // No watermark exists yet - expect(watermarkStore['h']).to.be.undefined; - - // The compensate function is called at startup or via cron. - // We can test it by triggering the cron onTick. - // Let's register cron and trigger it manually. - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { - createdJobs.push(jobConfig); - } - }); - - // Re-register to pick up the cron decorator - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - // Provide data for aggregate - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - // Trigger the h period cron job - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - if (hJob) { - await hJob.onTick(); - // After compensation, watermark should be set - expect(watermarkStore['h']).to.exist; - } - - await fastify.close(); - }); - - it('should return existing watermark without reinitializing', async () => { - const { fastify, watermarkStore, mockModel } = createCompensateMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const existingTime = new Date('2026-05-01T00:00:00.000Z'); - watermarkStore['h'] = { period: 'h', nextTime: existingTime }; - - // findOne should not be called since watermark exists - let findOneCalled = false; - mockModel.dataRecord.findOne = async () => { findOneCalled = true; return null; }; - mockModel.periodStat.findOne = async () => { findOneCalled = true; return null; }; - - // Trigger compensate via cron - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { - createdJobs.push(jobConfig); - } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - if (hJob) { - await hJob.onTick(); - } - - await fastify.close(); - }); - - it('should initialize watermark from period-stat min time for non-h periods', async () => { - const { fastify, findAllResults, watermarkStore, mockModel } = createCompensateMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - // For period 'd', dependency is from period-stat 'h' - const minTime = new Date('2026-05-01T00:00:00.000Z'); - mockModel.periodStat.findOne = async (opts) => { - if (opts.where && opts.where.period === 'h') return { minTime }; - return null; - }; - mockModel.dataRecord.findOne = async () => ({ minTime }); - - // Pre-set h watermark so d compensation can proceed - watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-27T00:00:00.000Z') }; - - // Provide h period data for d aggregation - fastify.statistics.models.periodStat.findAll = async (opts) => { - const period = opts.where && opts.where.period; - if (period === 'h') { - return [{ period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: minTime }]; - } - return []; - }; - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { - createdJobs.push(jobConfig); - } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const dJob = createdJobs.find(j => j.name === 'statistics-period-stat-d'); - if (dJob) { - await dJob.onTick(); - } - - await fastify.close(); - }); - - it('should return existing watermark without reinitializing', async () => { - const { fastify, watermarkStore } = createCompensateMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const existingTime = new Date('2026-05-01T00:00:00.000Z'); - watermarkStore['h'] = { period: 'h', nextTime: existingTime }; - - // Trigger aggregate which calls initWatermark - const records = await fastify.statistics.services.periodStat.aggregate('h'); - // Should return existing watermark, no re-init - expect(watermarkStore['h'].nextTime).to.deep.equal(existingTime); - - await fastify.close(); - }); - - it('should skip compensate when lock is already held', async () => { - const { fastify, watermarkStore } = createCompensateMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const { compensate } = require('../libs/services/period-stat'); - // The compensate function is internal, test via periodStat.isCompensating - expect(fastify.statistics.services.periodStat.isCompensating()).to.be.false; - - await fastify.close(); - }); - - it('should log warning when compensation is incomplete', async () => { - const { fastify, watermarkStore } = createCompensateMockFastify(); - let warnLogged = false; - const origLogWarn = fastify.log.warn; - fastify.log.warn = function (msg) { - warnLogged = true; - return origLogWarn ? origLogWarn.call(this, msg) : undefined; - }; - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 1 }); - - // Set watermark far in the past so there's a lot to compensate - watermarkStore['h'] = { period: 'h', nextTime: new Date('2020-01-01T00:00:00.000Z') }; - - // Manually trigger compensate for period h - // Since there's no data, it will advance watermark without aggregating - // But with batch size 1, it will only do 1 iteration - // This requires us to call the compensate function through the service - // compensate is not exposed directly, but isCompensating is - expect(fastify.statistics.services.periodStat.isCompensating()).to.be.false; - - fastify.log.warn = origLogWarn; - await fastify.close(); - }); - }); - - describe('invalidateQueryCache 版本递增测试', () => { - it('should increment channel versions for multi-level channels', async () => { - const { fastify } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - // Invalidate cache for a multi-level channel - fastify.statistics.services.periodStat.invalidateQueryCache(['device:sensor:temp']); - - // The next query should miss cache for this channel and its parents - // We verify indirectly by ensuring the function doesn't throw - expect(fastify.statistics.services.periodStat.isCompensating).to.be.a('function'); - - await fastify.close(); - }); - - it('should increment globalVersion on every invalidateQueryCache call', async () => { - const { fastify } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - // Multiple invalidations should keep incrementing version - fastify.statistics.services.periodStat.invalidateQueryCache(['ch1']); - fastify.statistics.services.periodStat.invalidateQueryCache(['ch2']); - // No throw means it's working - expect(fastify.statistics.services.periodStat.invalidateQueryCache).to.be.a('function'); - - await fastify.close(); - }); - - it('should handle empty channels array in invalidateQueryCache', async () => { - const { fastify } = createMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics' }); - - // Should not throw - fastify.statistics.services.periodStat.invalidateQueryCache([]); - - await fastify.close(); - }); - }); - - describe('覆盖率补充测试', () => { - const createFullMockFastify = () => { - const periodStatRows = []; - const findAllResults = []; - const findAllCalls = []; - const channelMetaRows = []; - const bulkCreateCalls = []; - const watermarkStore = {}; - const logCalls = { error: [], warn: [], info: [] }; - - const mockTransaction = { - commit: async () => {}, - rollback: async () => {} - }; - - const mockModel = { - dataRecord: { - findAll: async (opts) => { - findAllCalls.push({ model: 'dataRecord', opts }); - return [...findAllResults]; - }, - findOne: async () => null, - destroy: async () => {} - }, - periodStat: { - bulkCreate: async (records, opts) => { - bulkCreateCalls.push({ records: [...records], opts: opts || {} }); - return records; - }, - findAll: async (opts) => { - findAllCalls.push({ model: 'periodStat', opts }); - const period = opts.where && opts.where.period; - if (period && typeof period === 'object' && period.in) { - return periodStatRows.filter(row => period.in.includes(row.period)); - } - if (typeof period === 'string') { - return periodStatRows.filter(row => row.period === period); - } - return [...periodStatRows]; - }, - findOne: async () => null - }, - channelMeta: { - findAll: async ({ where }) => { - if (where && where.channel && where.channel.in) { - return channelMetaRows.filter(row => where.channel.in.includes(row.channel)); - } - return channelMetaRows; - } - }, - aggregationWatermark: { - findOne: async ({ where }) => watermarkStore[where.period] || null, - upsert: async (data) => { watermarkStore[data.period] = data; } - } - }; - - const fastify = require('fastify')(); - - ['error', 'warn', 'info'].forEach(level => { - const orig = fastify.log[level]; - fastify.log[level] = function (...args) { - logCalls[level].push(args.join(' ')); - return orig ? orig.apply(this, args) : undefined; - }; - }); - - fastify.decorate('sequelize', { - Sequelize: { - Op: { between: 'between', like: 'like', or: 'or', in: 'in' }, - fn: (name, col) => `${name}(${col})`, - col: name => name - }, - instance: { transaction: async () => mockTransaction } - }); - - fastify.decorate('statistics', { - models: mockModel, - services: {} - }); - - return { - fastify, periodStatRows, findAllResults, findAllCalls, - channelMetaRows, bulkCreateCalls, watermarkStore, logCalls, mockModel, - mockTransaction - }; - }; - - describe('queryCache 版本失效测试（内存模式）', () => { - it('should invalidate memory cache when globalVersion changes', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // Realtime query with no channels → stores globalVersion in cache entry - await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; - - fastify.statistics.services.periodStat.invalidateQueryCache([]); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } - ); - - const result = await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); - - await fastify.close(); - }); - - it('should invalidate memory cache when channelVersion changes', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // Realtime query with channels → stores channelVersions in cache entry - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; - - fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } - ); - - const result = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); - - await fastify.close(); - }); - - it('should not return expired memory cache entries', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, queryCacheTTL: 1 }); - - const now = new Date(); - const rtStart = new Date(now.getTime() - 3600000); - const rtEnd = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: rtStart } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime: rtStart, endTime: rtEnd, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; - - await new Promise(resolve => setTimeout(resolve, 1100)); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: rtStart } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime: rtStart, endTime: rtEnd, aggregates: ['sum'] - }); - - expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); - - await fastify.close(); - }); - - it('should set globalVersion in memory cache for realtime query with no channels', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; - - fastify.statistics.services.periodStat.invalidateQueryCache([]); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 200, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(callsAfterFirst); - - await fastify.close(); - }); - - it('should hit memory cache for realtime query with matching channelVersions', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - const callsAfterFirst = findAllCalls.filter(c => c.model === 'periodStat').length; - - // Second query with same params → should hit cache (channelVersions match) - const result2 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.equal(callsAfterFirst); - expect(result2.list[0].data.default).to.equal(100); - - await fastify.close(); - }); - }); - - describe('queryCache 版本失效测试（外部缓存模式）', () => { - it('should invalidate external cache when globalVersion changes', async () => { - const cacheStore = {}; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { cacheStore[key] = value; } - }; - - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // Realtime query with no channels → stores globalVersion in cache entry - await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - fastify.statistics.services.periodStat.invalidateQueryCache([]); - - fastify.statistics.models.periodStat.findAll = async () => []; - - const result = await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - expect(result.list.length).to.equal(0); - - await fastify.close(); - }); - - it('should invalidate external cache when channelVersion changes for specific channel', async () => { - const cacheStore = {}; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { cacheStore[key] = value; } - }; - - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - // Realtime query with specific channels → stores channelVersions in cache - const result1 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - expect(result1.list.length).to.be.greaterThan(0); - - // Invalidate sensor channel version - fastify.statistics.services.periodStat.invalidateQueryCache(['sensor']); - - // Replace DB to return empty, proving cache was bypassed - fastify.statistics.models.periodStat.findAll = async () => []; - - const result2 = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - expect(result2.list.length).to.equal(0); - - await fastify.close(); - }); - - it('should set globalVersion in external cache for realtime query with no channels', async () => { - const cacheStore = {}; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { cacheStore[key] = value; } - }; - - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const now = new Date(); - const startTime = new Date(now.getTime() - 3600000); - const endTime = new Date(now.getTime() + 3600000); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - startTime, endTime, aggregates: ['sum'] - }); - - const cacheKeys = Object.keys(cacheStore).filter(k => k.includes('query')); - expect(cacheKeys.length).to.be.greaterThan(0); - expect(cacheStore[cacheKeys[0]].globalVersion).to.exist; - - await fastify.close(); - }); - - it('should handle external cache set without TTL support', async () => { - const cacheStore = {}; - const setCalls = []; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value) => { - setCalls.push({ key, value }); - cacheStore[key] = value; - } - }; - - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(setCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should return null from external cache when payload has no value property', async () => { - const cacheStore = {}; - const externalCache = { - get: async (key) => cacheStore[key] || null, - set: async (key, value, ttl) => { cacheStore[key] = value; } - }; - - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', cache: externalCache, compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - const cacheKey = 'statistics:query:' + JSON.stringify({ - aggregates: ['sum'], attributeNames: [], channels: ['sensor'], - endTime: endTime.toISOString(), includeChildren: false, - startTime: startTime.toISOString(), timezone: '' - }); - cacheStore[cacheKey] = { notValue: true }; - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(findAllCalls.filter(c => c.model === 'periodStat').length).to.be.greaterThan(0); - - await fastify.close(); - }); - }); - - describe('compensate 详细逻辑测试', () => { - it('should log error when aggregate fails during compensation', async () => { - const { fastify, watermarkStore, logCalls, mockModel } = createFullMockFastify(); - - watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-27T00:00:00.000Z') }; - mockModel.dataRecord.findAll = async () => { throw new Error('DB read error'); }; - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - expect(hJob).to.exist; - await hJob.onTick(); - - expect(logCalls.error.some(msg => msg.includes('Failed to compensate period h'))).to.be.true; - - await fastify.close(); - }); - - it('should log warning when compensation is incomplete', async () => { - const { fastify, watermarkStore, logCalls, findAllResults } = createFullMockFastify(); - - watermarkStore['h'] = { period: 'h', nextTime: new Date('2020-01-01T00:00:00.000Z') }; - - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 1 }); - - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - expect(hJob).to.exist; - await hJob.onTick(); - - expect(logCalls.warn.some(msg => msg.includes('补偿未完成'))).to.be.true; - - await fastify.close(); - }); - - it('should log info when compensation completes with multiple windows', async () => { - const { fastify, watermarkStore, logCalls, findAllResults } = createFullMockFastify(); - - const now = new Date(); - const threeHoursAgo = new Date(now.getTime() - 3 * 3600000); - threeHoursAgo.setMinutes(0, 0, 0); - - watermarkStore['h'] = { period: 'h', nextTime: threeHoursAgo }; - - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); - - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - expect(hJob).to.exist; - await hJob.onTick(); - - expect(logCalls.info.some(msg => msg.includes('补偿完成'))).to.be.true; - - await fastify.close(); - }); - - it('should log error when startup compensation fails', async () => { - const { fastify, logCalls, mockModel, watermarkStore } = createFullMockFastify(); - - // Make setWatermark fail → error propagates out of compensate to startup catch - mockModel.aggregationWatermark.upsert = async () => { throw new Error('Watermark write error'); }; - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: true }); - - // Wait for async startup compensation - for (let i = 0; i < 20; i++) { - if (logCalls.error.some(msg => msg.includes('Startup compensation failed'))) break; - await new Promise(resolve => setTimeout(resolve, 100)); - } - - expect(logCalls.error.some(msg => msg.includes('Startup compensation failed'))).to.be.true; - - await fastify.close(); - }); - - it('should trigger upstream compensation when upstream watermark is behind', async () => { - const { fastify, watermarkStore, findAllResults, bulkCreateCalls, periodStatRows, mockModel } = createFullMockFastify(); - - // Set d watermark in the past - watermarkStore['d'] = { period: 'd', nextTime: new Date('2026-05-26T00:00:00.000Z') }; - // Set h watermark BEHIND d's window end → triggers compensate('h') - watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-26T00:00:00.000Z') }; - - // Provide data for h aggregation (data-record) - findAllResults.push({ - channel: 'ch1', attributeName: 'val', - sum: 10, avg: null, count: null, min: null, max: null - }); - - // Provide data for d aggregation (period-stat for h period) - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: new Date('2026-05-26T00:00:00.000Z') } - ); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const dJob = createdJobs.find(j => j.name === 'statistics-period-stat-d'); - expect(dJob).to.exist; - await dJob.onTick(); - - // Should have called bulkCreate (for h compensation then d aggregation) - expect(bulkCreateCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should log error via cron onTick when compensate throws outside aggregate', async () => { - const { fastify, watermarkStore, logCalls, mockModel } = createFullMockFastify(); - - // Set watermark so compensate has work to do - watermarkStore['h'] = { period: 'h', nextTime: new Date('2026-05-27T00:00:00.000Z') }; - - // Make setWatermark fail → error propagates out of compensate to cron onTick catch - mockModel.aggregationWatermark.upsert = async () => { throw new Error('Watermark write error'); }; - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const hJob = createdJobs.find(j => j.name === 'statistics-period-stat-h'); - expect(hJob).to.exist; - await hJob.onTick(); - - expect(logCalls.error.some(msg => msg.includes('Failed to compensate period h'))).to.be.true; - - await fastify.close(); - }); - }); - - describe('formatGroupData 边界测试', () => { - it('should not include unit when all items have null or undefined unit', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime, unit: null }, - { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime, unit: undefined } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].unit).to.be.undefined; - - await fastify.close(); - }); - - it('should include unit only for attributes with non-null unit', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime, unit: '°C' }, - { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime, unit: null } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].unit).to.deep.equal({ temp: '°C' }); - - await fastify.close(); - }); - }); - - describe('query 边界测试', () => { - it('should handle channels as a string instead of array', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: 'sensor', startTime, endTime, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - expect(results[0].channel).to.equal('sensor'); - - await fastify.close(); - }); - - it('should escape special characters in channel names for includeChildren query', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor%test', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor%test'], startTime, endTime, aggregates: ['sum'], includeChildren: true - }); - - expect(results.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should build channel tree with parent having no items but children having items', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'sensor:room1', attributeName: 'default', aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true - }); - - expect(results.length).to.equal(1); - expect(results[0].channel).to.equal('sensor'); - expect(results[0].children.length).to.be.greaterThan(0); - expect(results[0].children[0].channel).to.equal('sensor:room1'); - - await fastify.close(); - }); - }); - - describe('aggregateFromPeriodStat unit 继承测试', () => { - it('should set unit to null when items have no unit field', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-02T00:00:00.000Z'); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 50, time: startTime } - ); - - const records = await fastify.statistics.services.periodStat.aggregate('d', { startTime, endTime }); - - expect(records.length).to.equal(1); - expect(records[0].unit).to.equal(null); - - await fastify.close(); - }); - }); - - describe('w/m/q/y 周期 compensate getNextStart 覆盖测试', () => { - it('should compensate w period using getNextStart (week)', async () => { - const { fastify, watermarkStore, findAllResults, bulkCreateCalls, periodStatRows } = createFullMockFastify(); - - // Set w watermark 2 weeks in the past, and d watermark already caught up - const twoWeeksAgo = new Date(); - twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14); - // Truncate to start of that week (Monday) - const dayjs = require('dayjs'); - const wStart = dayjs(twoWeeksAgo).startOf('week').add(1, 'day').startOf('day').toDate(); - - watermarkStore['w'] = { period: 'w', nextTime: wStart }; - // d watermark must be ahead of w's next endTime - watermarkStore['d'] = { period: 'd', nextTime: new Date() }; - - // Provide d-level periodStat data for aggregation - periodStatRows.push( - { period: 'd', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: wStart } - ); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); - - const wJob = createdJobs.find(j => j.name === 'statistics-period-stat-w'); - expect(wJob).to.exist; - await wJob.onTick(); - - // Should have created aggregated records - expect(bulkCreateCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should compensate m period using getNextStart (month)', async () => { - const { fastify, watermarkStore, bulkCreateCalls, periodStatRows } = createFullMockFastify(); - - // Set m watermark 2 months in the past - const twoMonthsAgo = new Date(); - twoMonthsAgo.setMonth(twoMonthsAgo.getMonth() - 2); - twoMonthsAgo.setDate(1); - twoMonthsAgo.setHours(0, 0, 0, 0); - - watermarkStore['m'] = { period: 'm', nextTime: twoMonthsAgo }; - watermarkStore['d'] = { period: 'd', nextTime: new Date() }; - - periodStatRows.push( - { period: 'd', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: twoMonthsAgo } - ); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); - - const mJob = createdJobs.find(j => j.name === 'statistics-period-stat-m'); - expect(mJob).to.exist; - await mJob.onTick(); - - expect(bulkCreateCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should compensate q period using getNextStart (quarter)', async () => { - const { fastify, watermarkStore, bulkCreateCalls, periodStatRows } = createFullMockFastify(); - - // Set q watermark 6 months in the past - const sixMonthsAgo = new Date(); - sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); - // Truncate to quarter start - const qMonth = Math.floor(sixMonthsAgo.getMonth() / 3) * 3; - sixMonthsAgo.setMonth(qMonth, 1); - sixMonthsAgo.setHours(0, 0, 0, 0); - - watermarkStore['q'] = { period: 'q', nextTime: sixMonthsAgo }; - watermarkStore['m'] = { period: 'm', nextTime: new Date() }; - - periodStatRows.push( - { period: 'm', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: sixMonthsAgo } - ); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); - - const qJob = createdJobs.find(j => j.name === 'statistics-period-stat-q'); - expect(qJob).to.exist; - await qJob.onTick(); - - expect(bulkCreateCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - - it('should compensate y period using getNextStart (year)', async () => { - const { fastify, watermarkStore, bulkCreateCalls, periodStatRows } = createFullMockFastify(); - - // Set y watermark 2 years in the past - const twoYearsAgo = new Date(); - twoYearsAgo.setFullYear(twoYearsAgo.getFullYear() - 2); - twoYearsAgo.setMonth(0, 1); - twoYearsAgo.setHours(0, 0, 0, 0); - - watermarkStore['y'] = { period: 'y', nextTime: twoYearsAgo }; - watermarkStore['q'] = { period: 'q', nextTime: new Date() }; - - periodStatRows.push( - { period: 'q', channel: 'ch1', attributeName: 'val', aggregate: 'sum', data: 10, time: twoYearsAgo } - ); - - const createdJobs = []; - fastify.decorate('cron', { - createJob: (jobConfig) => { createdJobs.push(jobConfig); } - }); - - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false, compensationBatchSize: 100 }); - - const yJob = createdJobs.find(j => j.name === 'statistics-period-stat-y'); - expect(yJob).to.exist; - await yJob.onTick(); - - expect(bulkCreateCalls.length).to.be.greaterThan(0); - - await fastify.close(); - }); - }); - - describe('剩余分支覆盖测试', () => { - it('should use default prefix when name option is not provided', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - // Don't pass name option — covers line 100: options.name || 'statistics' - await mockPeriodStatService(fastify, { compensationEnabled: false }); - - const now = new Date(); - const oneHourAgo = new Date(now.getTime() - 3600000); - - periodStatRows.push( - { period: 'h', channel: 'ch1', attributeName: 'temp', aggregate: 'sum', data: 100, time: oneHourAgo } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['ch1'], startTime: oneHourAgo, endTime: now, aggregates: ['sum'] - }); - - expect(results.length).to.equal(1); - await fastify.close(); - }); - - it('should hit external cache when channelVersions all match', async () => { - const { fastify, periodStatRows, findAllCalls } = createFullMockFastify(); - - const cachedValue = [{ channel: 'ch1', data: { temp: 42 } }]; - let setCalls = []; - - const externalCache = { - get: async (key) => { - if (key.includes('query:')) { - // Return cache payload with matching versions - return { - value: cachedValue, - globalVersion: 0, - channelVersions: { ch1: 1 } - }; - } - return null; - }, - set: async (key, value, ttl) => { - setCalls.push({ key, value, ttl }); - } - }; - - await mockPeriodStatService(fastify, { cache: externalCache, compensationEnabled: false }); - - const now = new Date(); - const oneHourAgo = new Date(now.getTime() - 3600000); - - // First invalidate to set channelVersion for ch1 = 1 - fastify.statistics.services.periodStat.invalidateQueryCache(['ch1']); - - // Now query — cache should hit since channelVersion matches - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['ch1'], startTime: oneHourAgo, endTime: now, aggregates: ['sum'] - }); - - // Cache hit means no DB query - const dbCalls = findAllCalls.filter(c => c.model === 'periodStat'); - expect(dbCalls.length).to.equal(0); - // Results come from cache - expect(results).to.deep.equal(cachedValue); - - await fastify.close(); - }); - - it('should handle invalidateQueryCache with no arguments (default empty array)', async () => { - const { fastify } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - // Call without arguments — covers default parameter branch - fastify.statistics.services.periodStat.invalidateQueryCache(); - - // Should not throw - await fastify.close(); - }); - - it('should skip unit assignment when attributeName already in unitMap', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - // Two items with same attributeName, first sets unit, second is skipped (covers branch at 445-446) - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime, unit: '°C' }, - { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'avg', data: 50, time: startTime, unit: '°F' } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum', 'avg'] - }); - - expect(results.length).to.equal(1); - // Unit should be from the first item (°C), not overwritten by second (°F) - expect(results[0].unit.temp).to.equal('°C'); - - await fastify.close(); - }); - - it('should not include unit in item entries when unit is undefined', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T02:00:00.000Z'); - - // Item with no unit field — covers line 492 branch where unit is undefined - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true - }); - - expect(results.length).to.equal(1); - // Item entries should not have 'unit' key when unit is undefined - if (results[0].items) { - expect(results[0].items[0].unit).to.be.undefined; - } - - await fastify.close(); - }); - - it('should return null node when channel has no items and no children in tree', async () => { - const { fastify, periodStatRows, channelMetaRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T01:00:00.000Z'); - - // Query a channel that has no items and no children — covers line 510 returning null - // The tree should be empty - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['nonexistent'], startTime, endTime, aggregates: ['sum'], includeChildren: true - }); - - // No results since channel has no data and no children - expect(results.length).to.equal(0); - - await fastify.close(); - }); - - it('should handle multiple items for same channel in channelGroups', async () => { - const { fastify, periodStatRows } = createFullMockFastify(); - await mockPeriodStatService(fastify, { name: 'statistics', compensationEnabled: false }); - - const startTime = new Date('2026-05-01T00:00:00.000Z'); - const endTime = new Date('2026-05-01T02:00:00.000Z'); - - // Multiple items for same channel — covers line 479 branch where channelGroups[item.channel] already exists - periodStatRows.push( - { period: 'h', channel: 'sensor', attributeName: 'temp', aggregate: 'sum', data: 100, time: startTime }, - { period: 'h', channel: 'sensor', attributeName: 'humidity', aggregate: 'sum', data: 200, time: startTime } - ); - - const { list: results } = await fastify.statistics.services.periodStat.query({ - channels: ['sensor'], startTime, endTime, aggregates: ['sum'], includeChildren: true - }); - - expect(results.length).to.equal(1); - expect(results[0].items.length).to.equal(2); - - await fastify.close(); - }); - }); - }); - }); -});